diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..34ad7ec3 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,8 @@ +# Auto-detect text files and normalize line endings +* text=auto + +# Enforce LF line endings for TSV files to ensure cross-platform test parity (especially on Windows) +*.tsv text eol=lf + +# Shell scripts must have LF to execute on Unix environments +*.sh text eol=lf diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 00000000..828cfdd5 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1 @@ +github: turtle261 diff --git a/.github/workflows/python-release.yml b/.github/workflows/python-release.yml index b4b3b4bb..fa7c2829 100644 --- a/.github/workflows/python-release.yml +++ b/.github/workflows/python-release.yml @@ -7,6 +7,7 @@ on: env: CARGO_TERM_COLOR: always + PYTHON_RELEASE_FEATURES: python-extension,all-backends,aixi-gameengine permissions: contents: read @@ -58,8 +59,8 @@ jobs: env: RUSTFLAGS: -C target-cpu=x86-64 run: >- - cargo check --release --manifest-path infotheory_py/Cargo.toml - --features python-extension,backend-rosa,backend-mamba,backend-rwkv,backend-zpaq + cargo check --release --manifest-path crates/infotheory_py/Cargo.toml + --features $PYTHON_RELEASE_FEATURES - name: Build wheel (Linux) if: runner.os == 'Linux' @@ -71,14 +72,16 @@ jobs: RUSTFLAGS: -C target-cpu=x86-64 run: | export PATH=".venv/bin:$PATH" + rm -rf target/maturin-linux ln -sf python-zig .venv/bin/zig export ZIG_COMMAND="$(command -v zig)" export AR_x86_64_unknown_linux_gnu="$(command -v llvm-ar)" export RANLIB_x86_64_unknown_linux_gnu="$(command -v llvm-ranlib)" .venv/bin/python -m maturin build --release \ - --manifest-path infotheory_py/Cargo.toml \ + --manifest-path crates/infotheory_py/Cargo.toml \ + --target-dir target/maturin-linux \ --interpreter python3 \ - --features python-extension,backend-rosa,backend-mamba,backend-rwkv,backend-zpaq \ + --features "$PYTHON_RELEASE_FEATURES" \ --out target/wheels \ --compatibility manylinux2014 \ --zig @@ -91,7 +94,8 @@ jobs: RUSTFLAGS: -C target-cpu=generic run: | export PATH="$(dirname "$VENV_PY"):$PATH" - "$VENV_PY" -m maturin build --release --manifest-path infotheory_py/Cargo.toml --features python-extension,backend-rosa,backend-mamba,backend-rwkv,backend-zpaq --out target/wheels + rm -rf "target/maturin-${{ runner.os }}" + "$VENV_PY" -m maturin build --release --manifest-path crates/infotheory_py/Cargo.toml --target-dir "target/maturin-${{ runner.os }}" --features "$PYTHON_RELEASE_FEATURES" --out target/wheels - name: Upload wheel artifacts uses: actions/upload-artifact@v4 @@ -105,7 +109,7 @@ jobs: shell: bash run: | export PATH=".venv/bin:$PATH" - .venv/bin/python -m maturin sdist --manifest-path infotheory_py/Cargo.toml + .venv/bin/python -m maturin sdist --manifest-path crates/infotheory_py/Cargo.toml - name: Upload sdist artifact if: runner.os == 'Linux' diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 17228f25..1a4b4127 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -57,7 +57,8 @@ jobs: RUSTFLAGS: -C link-arg=-fuse-ld=lld -C target-cpu=x86-64 run: | export PATH="$(dirname "$VENV_PY"):$PATH" - "$VENV_PY" -m maturin develop --profile python-release --manifest-path infotheory_py/Cargo.toml + rm -rf target/maturin-pytest-linux + "$VENV_PY" -m maturin develop --profile python-release --manifest-path crates/infotheory_py/Cargo.toml --target-dir target/maturin-pytest-linux - name: Build extension (macOS/Windows) if: runner.os != 'Linux' @@ -67,7 +68,8 @@ jobs: VENV_PY: ${{ runner.os == 'Windows' && '.venv/Scripts/python.exe' || '.venv/bin/python' }} run: | export PATH="$(dirname "$VENV_PY"):$PATH" - "$VENV_PY" -m maturin develop --profile python-release --manifest-path infotheory_py/Cargo.toml + rm -rf "target/maturin-pytest-${{ runner.os }}" + "$VENV_PY" -m maturin develop --profile python-release --manifest-path crates/infotheory_py/Cargo.toml --target-dir "target/maturin-pytest-${{ runner.os }}" - name: Run tests shell: bash @@ -76,6 +78,20 @@ jobs: run: | "$VENV_PY" -m pytest --cov=infotheory_rs --cov-report=term-missing --cov-report=xml:target/python-coverage.xml --cov-fail-under=100 python/tests + - name: Python GameEngine smoke (Linux) + if: runner.os == 'Linux' + shell: bash + env: + VIRTUAL_ENV: .venv + CC: clang + CXX: clang++ + RUSTFLAGS: -C link-arg=-fuse-ld=lld -C target-cpu=x86-64 + run: | + export PATH=".venv/bin:$PATH" + rm -rf target/maturin-gameengine-linux + .venv/bin/python -m maturin develop --profile python-release --manifest-path crates/infotheory_py/Cargo.toml --target-dir target/maturin-gameengine-linux --features python-extension,all-backends,aixi-gameengine + .venv/bin/python -m pytest -q python/tests/test_aixi_gameengine.py + - name: Upload Python coverage artifact uses: actions/upload-artifact@v4 with: @@ -88,7 +104,7 @@ jobs: env: VENV_PY: ${{ runner.os == 'Windows' && '.venv/Scripts/python.exe' || '.venv/bin/python' }} run: | - "$VENV_PY" -m maturin sdist --manifest-path infotheory_py/Cargo.toml + "$VENV_PY" -m maturin sdist --manifest-path crates/infotheory_py/Cargo.toml - name: Build wheel (Linux) if: runner.os == 'Linux' @@ -100,7 +116,8 @@ jobs: RUSTFLAGS: -C link-arg=-fuse-ld=lld -C target-cpu=x86-64 run: | export PATH="$(dirname "$VENV_PY"):$PATH" - "$VENV_PY" -m maturin build --profile python-release --manifest-path infotheory_py/Cargo.toml + rm -rf target/maturin-wheel-linux + "$VENV_PY" -m maturin build --profile python-release --manifest-path crates/infotheory_py/Cargo.toml --target-dir target/maturin-wheel-linux - name: Build wheel (macOS/Windows) if: runner.os != 'Linux' @@ -109,7 +126,8 @@ jobs: VENV_PY: ${{ runner.os == 'Windows' && '.venv/Scripts/python.exe' || '.venv/bin/python' }} run: | export PATH="$(dirname "$VENV_PY"):$PATH" - "$VENV_PY" -m maturin build --profile python-release --manifest-path infotheory_py/Cargo.toml + rm -rf "target/maturin-wheel-${{ runner.os }}" + "$VENV_PY" -m maturin build --profile python-release --manifest-path crates/infotheory_py/Cargo.toml --target-dir "target/maturin-wheel-${{ runner.os }}" - name: Install built wheel and import shell: bash @@ -159,7 +177,8 @@ jobs: RUSTFLAGS: -C link-arg=-fuse-ld=lld -C target-cpu=x86-64 run: | export PATH=".venv/bin:$PATH" - .venv/bin/python -m maturin develop --profile python-release --manifest-path infotheory_py/Cargo.toml --features python-extension,backend-rosa,backend-mamba,backend-rwkv,backend-zpaq,vm + rm -rf target/maturin-vm-linux + .venv/bin/python -m maturin develop --profile python-release --manifest-path crates/infotheory_py/Cargo.toml --target-dir target/maturin-vm-linux --features python-extension,all-backends,vm - name: Run VM-featured Python smoke tests shell: bash diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 73eb8d0e..a1e72227 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -39,28 +39,50 @@ jobs: - name: Cache Rust artifacts uses: Swatinem/rust-cache@v2 + with: + cache-bin: false - name: Rustfmt run: cargo fmt --all --check + - name: Install uv + uses: astral-sh/setup-uv@v4 + + - name: Tuner traceability anchors + run: | + uv run --no-project --with tree-sitter==0.25.2 --with tree-sitter-rust==0.24.2 python scripts/check_tuner_traceability.py + + - name: Dependency integrity + run: | + cargo update --workspace --locked --dry-run + cargo tree --workspace --all-features --locked --duplicates + - name: Clippy (default + CLI) run: | cargo clippy -p zpaq_rs --locked + cargo clippy --manifest-path vendor/gameengine/Cargo.toml --features builtin --locked cargo clippy -p infotheory --lib --bins --features cli --locked + cargo clippy -p infotheory --lib --no-default-features --features aixi-gameengine --locked cargo clippy -p infotheory --lib --bins --no-default-features --features cli --locked + cargo clippy -p infotheory --lib --bins --no-default-features --features "cli tuner backend-ctw" --locked - name: Warnings gate (default + no-default CLI) run: | RUSTFLAGS="-D warnings" cargo check -p zpaq_rs --locked + RUSTFLAGS="-D warnings" cargo check --manifest-path vendor/gameengine/Cargo.toml --features builtin --locked RUSTFLAGS="-D warnings" cargo check -p infotheory --features cli --locked + RUSTFLAGS="-D warnings" cargo check -p infotheory --no-default-features --features aixi-gameengine --locked RUSTFLAGS="-D warnings" cargo check -p infotheory --no-default-features --features cli --locked + RUSTFLAGS="-D warnings" cargo check -p infotheory --no-default-features --features "tuner backend-ctw" --locked + RUSTFLAGS="-D warnings" cargo check -p infotheory --no-default-features --features "cli tuner backend-ctw" --locked + PYO3_BUILD_EXTENSION_MODULE=1 RUSTFLAGS="-D warnings" cargo check -p infotheory_py --no-default-features --features "tuner backend-ctw" --locked - name: Install coverage tooling run: cargo install cargo-llvm-cov --locked - name: Rust line coverage gate (CLI + library parity suite) run: | - cargo llvm-cov -p infotheory --tests --features cli --locked --summary-only --fail-under-lines 50 + cargo llvm-cov -p infotheory --tests --features "cli all-backends" --locked --summary-only --fail-under-lines 85 - name: Rustdoc coverage gate run: | @@ -83,18 +105,13 @@ jobs: cargo install cargo-audit --locked cargo audit - - name: Dependency integrity - run: | - cargo update --workspace --locked --dry-run - cargo tree --workspace --all-features --locked --duplicates - - name: Install Lean (elan) run: | curl -sSf https://raw.githubusercontent.com/leanprover/elan/master/elan-init.sh | sh -s -- -y echo "$HOME/.elan/bin" >> "$GITHUB_PATH" - name: Build infotheory CLI for Lean validation - run: cargo build --release --features cli --bin infotheory --locked + run: cargo build -p infotheory --release --features cli --bin infotheory --locked - name: Run Lean validation (ite-bench) run: ./projman.sh lean_test @@ -177,38 +194,109 @@ jobs: uses: Swatinem/rust-cache@v2 with: shared-key: platform-hosted-${{ matrix.target }} + cache-bin: false - name: Feature checks env: RUSTFLAGS: -D warnings + PYO3_BUILD_EXTENSION_MODULE: "1" run: | - cargo check --locked - cargo check --no-default-features --locked - cargo check --no-default-features --features cli --locked - cargo check --features cli --locked - - - name: VM feature check + cargo check -p infotheory --locked + cargo check -p infotheory --no-default-features --locked + cargo check -p infotheory --no-default-features --features backend-rosa --locked + cargo check -p infotheory --no-default-features --features backend-ctw --locked + cargo check -p infotheory --no-default-features --features backend-match --locked + cargo check -p infotheory --no-default-features --features backend-ppmd --locked + cargo check -p infotheory --no-default-features --features backend-sequitur --locked + cargo check -p infotheory --no-default-features --features backend-mixture --locked + cargo check -p infotheory --no-default-features --features backend-particle --locked + cargo check -p infotheory --no-default-features --features backend-calibrated --locked + cargo check -p infotheory --no-default-features --features backend-zpaq --locked + cargo check -p infotheory --no-default-features --features backend-rwkv --locked + cargo check -p infotheory --no-default-features --features backend-mamba --locked + cargo check -p infotheory --no-default-features --features all-backends --locked + cargo check -p infotheory --no-default-features --features aixi-gameengine --locked + cargo check -p infotheory --no-default-features --features "aixi-gameengine all-backends" --locked + cargo check -p infotheory --no-default-features --features tuner --locked + cargo check -p infotheory --no-default-features --features "tuner backend-ctw" --locked + cargo check -p infotheory --no-default-features --features "cli tuner backend-ctw" --locked + cargo check -p infotheory_py --no-default-features --features "tuner backend-ctw" --locked + cargo check -p infotheory --no-default-features --features cli --locked + cargo check -p infotheory --features cli --locked + cargo check --manifest-path vendor/gameengine/Cargo.toml --features builtin --locked + cargo check -p zpaq_rs --locked + cargo check -p benchman --locked + + - name: Feature-slice test compile checks + run: | + cargo test -p infotheory --no-run --locked + cargo test -p infotheory --no-default-features --no-run --locked + cargo test -p infotheory --no-default-features --features backend-rosa --no-run --locked + cargo test -p infotheory --no-default-features --features backend-ctw --no-run --locked + cargo test -p infotheory --no-default-features --features backend-match --no-run --locked + cargo test -p infotheory --no-default-features --features backend-ppmd --no-run --locked + cargo test -p infotheory --no-default-features --features backend-sequitur --no-run --locked + cargo test -p infotheory --no-default-features --features backend-mixture --no-run --locked + cargo test -p infotheory --no-default-features --features backend-particle --no-run --locked + cargo test -p infotheory --no-default-features --features backend-calibrated --no-run --locked + cargo test -p infotheory --no-default-features --features backend-zpaq --no-run --locked + cargo test -p infotheory --no-default-features --features backend-rwkv --no-run --locked + cargo test -p infotheory --no-default-features --features backend-mamba --no-run --locked + cargo test -p infotheory --no-default-features --features all-backends --no-run --locked + cargo test -p infotheory --no-default-features --features aixi-gameengine --no-run --locked + cargo test -p infotheory --no-default-features --features "aixi-gameengine all-backends" --no-run --locked + cargo test -p infotheory --no-default-features --features tuner --no-run --locked + cargo test -p infotheory --no-default-features --features "tuner backend-ctw" --no-run --locked + cargo test -p infotheory --no-default-features --features "cli tuner backend-ctw" --no-run --locked + cargo test -p infotheory --no-default-features --features cli --no-run --locked + cargo test -p infotheory --features cli --no-run --locked + cargo test -p infotheory --no-default-features --features "cli all-backends" --no-run --locked + cargo test --manifest-path vendor/gameengine/Cargo.toml --features builtin --no-run --locked + + - name: VM feature slices if: ${{ matrix.run_vm }} - run: cargo check --features vm --locked + run: | + cargo test -p infotheory --no-default-features --features vm --no-run --locked + cargo test -p infotheory --no-default-features --features "vm backend-ctw" --no-run --locked - name: Test matrix env: RUSTDOCFLAGS: ${{ matrix.rustdocflags }} + shell: bash run: | - cargo test --release --locked - cargo test --release --no-default-features --locked - cargo test --release --features cli --locked - cargo test --release --no-default-features --features cli --locked + if [[ "${{ startsWith(matrix.target, 'aarch64-') }}" == "true" ]]; then + export ZPAQ_NOJIT=1 + fi + cargo test -p infotheory --release --locked + cargo test -p infotheory --release --no-default-features --features all-backends --locked + cargo test -p infotheory --release --no-default-features rate_backend_aliases_share_registry_resolution_and_feature_errors --locked + cargo test -p infotheory --release --no-default-features compression_backend_aliases_share_registry_resolution_and_feature_errors --locked + cargo test -p infotheory --release --no-default-features --features backend-ctw --test api_surface --locked + cargo test -p infotheory --release --no-default-features --features backend-rosa --test compression_validation --locked + cargo test -p infotheory --release --no-default-features --features backend-rosa --test api_surface --locked + cargo test -p infotheory --release --no-default-features --features backend-zpaq --test api_surface --locked + cargo test -p infotheory --release --no-default-features --features backend-rwkv parse_compression_backend_name_method_wraps_rwkv_cfg_methods_as_rate_backend --locked + cargo test -p infotheory --release --no-default-features --features backend-rwkv parse_compression_backend_json_wraps_rwkv_cfg_methods_as_rate_backend --locked + cargo test -p infotheory --release --no-default-features --features backend-mamba parse_compression_backend_json_wraps_mamba_cfg_rate_backend --locked + cargo test -p infotheory --release --no-default-features --features "tuner backend-ctw" --test tuner_integration --locked + cargo test -p infotheory --release --no-default-features --features "cli tuner backend-ctw" --test tuner_integration tune_cli_accepts_executor_flags_and_writes_report --locked + cargo test -p infotheory --release --features cli --locked + cargo test -p infotheory --release --no-default-features --features cli --bin infotheory --locked + cargo test -p infotheory --release --no-default-features --features "cli all-backends" --locked + cargo test -p infotheory --release --no-default-features --features aixi-gameengine --locked + cargo test --manifest-path vendor/gameengine/Cargo.toml --release --features builtin --locked + cargo test -p zpaq_rs --release --locked + cargo test -p benchman --release --locked - name: VM tests if: ${{ matrix.run_vm }} env: RUSTDOCFLAGS: ${{ matrix.rustdocflags }} - run: cargo test --release --features vm --locked + run: cargo test -p infotheory --release --no-default-features --features "vm backend-ctw" --locked - name: Build release CLI binary run: | - cargo rustc --release --features cli --bin infotheory --target ${{ matrix.target }} --locked -- ${{ matrix.release_rustc_flags }} + cargo rustc -p infotheory --release --features cli --bin infotheory --target ${{ matrix.target }} --locked -- ${{ matrix.release_rustc_flags }} - name: Package binary artifact shell: bash @@ -318,6 +406,7 @@ jobs: case "$OS" in NetBSD) export CARGO_FEATURE_NOJIT=1 + export ZPAQ_NOJIT=1 export CARGO_PROFILE_RELEASE_LTO=off if ! command -v clang >/dev/null 2>&1 && command -v cc >/dev/null 2>&1; then export CARGO_TARGET_X86_64_UNKNOWN_NETBSD_LINKER=cc @@ -325,6 +414,7 @@ jobs: ;; OpenBSD) export CARGO_FEATURE_NOJIT=1 + export ZPAQ_NOJIT=1 ;; FreeBSD) : @@ -332,18 +422,44 @@ jobs: esac export RUSTDOCFLAGS="${{ matrix.rustdocflags }}" - - RUSTFLAGS="-D warnings" cargo check --locked - RUSTFLAGS="-D warnings" cargo check --locked --no-default-features - RUSTFLAGS="-D warnings" cargo check --locked --features cli - RUSTFLAGS="-D warnings" cargo check --locked --no-default-features --features cli - - cargo test --release --locked - cargo test --release --no-default-features --locked - cargo test --release --features cli --locked - cargo test --release --no-default-features --features cli --locked - - cargo rustc --release --features cli --bin infotheory --locked -- -C target-cpu=x86-64 + RUSTFLAGS="-D warnings" cargo check -p infotheory --locked + RUSTFLAGS="-D warnings" cargo check -p infotheory --locked --no-default-features + RUSTFLAGS="-D warnings" cargo check -p infotheory --locked --no-default-features --features backend-rosa + RUSTFLAGS="-D warnings" cargo check -p infotheory --locked --no-default-features --features backend-ctw + RUSTFLAGS="-D warnings" cargo check -p infotheory --locked --no-default-features --features backend-match + RUSTFLAGS="-D warnings" cargo check -p infotheory --locked --no-default-features --features backend-ppmd + RUSTFLAGS="-D warnings" cargo check -p infotheory --locked --no-default-features --features backend-sequitur + RUSTFLAGS="-D warnings" cargo check -p infotheory --locked --no-default-features --features backend-mixture + RUSTFLAGS="-D warnings" cargo check -p infotheory --locked --no-default-features --features backend-particle + RUSTFLAGS="-D warnings" cargo check -p infotheory --locked --no-default-features --features backend-calibrated + RUSTFLAGS="-D warnings" cargo check -p infotheory --locked --no-default-features --features backend-zpaq + RUSTFLAGS="-D warnings" cargo check -p infotheory --locked --no-default-features --features backend-rwkv + RUSTFLAGS="-D warnings" cargo check -p infotheory --locked --no-default-features --features backend-mamba + RUSTFLAGS="-D warnings" cargo check -p infotheory --locked --no-default-features --features all-backends + RUSTFLAGS="-D warnings" cargo check -p infotheory --locked --features cli + RUSTFLAGS="-D warnings" cargo check -p infotheory --locked --no-default-features --features cli + RUSTFLAGS="-D warnings" cargo check -p infotheory --locked --no-default-features --features "cli all-backends" + PYO3_BUILD_EXTENSION_MODULE=1 RUSTFLAGS="-D warnings" cargo check -p infotheory_py --locked --no-default-features --features "tuner backend-ctw" + RUSTFLAGS="-D warnings" cargo check -p zpaq_rs --locked + RUSTFLAGS="-D warnings" cargo check -p benchman --locked + + cargo test -p infotheory --release --locked + cargo test -p infotheory --release --no-default-features --features all-backends --locked + cargo test -p infotheory --release --no-default-features rate_backend_aliases_share_registry_resolution_and_feature_errors --locked + cargo test -p infotheory --release --no-default-features compression_backend_aliases_share_registry_resolution_and_feature_errors --locked + cargo test -p infotheory --release --no-default-features --features backend-ctw --test api_surface --locked + cargo test -p infotheory --release --no-default-features --features backend-rosa --test compression_validation --locked + cargo test -p infotheory --release --no-default-features --features backend-rosa --test api_surface --locked + cargo test -p infotheory --release --no-default-features --features backend-zpaq --test api_surface --locked + cargo test -p infotheory --release --no-default-features --features backend-rwkv parse_compression_backend_name_method_wraps_rwkv_cfg_methods_as_rate_backend --locked + cargo test -p infotheory --release --no-default-features --features backend-rwkv parse_compression_backend_json_wraps_rwkv_cfg_methods_as_rate_backend --locked + cargo test -p infotheory --release --no-default-features --features backend-mamba parse_compression_backend_json_wraps_mamba_cfg_rate_backend --locked + cargo test -p infotheory --release --features cli --locked + cargo test -p infotheory --release --no-default-features --features "cli all-backends" --locked + cargo test -p zpaq_rs --release --locked + cargo test -p benchman --release --locked + + cargo rustc -p infotheory --release --features cli --bin infotheory --locked -- -C target-cpu=x86-64 mkdir -p dist cp target/release/infotheory dist/infotheory @@ -379,6 +495,8 @@ jobs: - name: Cache Rust artifacts uses: Swatinem/rust-cache@v2 + with: + cache-bin: false - name: Cross-target compile checks (RWKV-only portable profile) env: @@ -463,18 +581,47 @@ jobs: eval "export CARGO_TARGET_${TARGET_ENV_UPPER}_LINKER=clang" export MUSL_LD_RUSTFLAGS="-C link-arg=-fuse-ld=lld" export MUSL_RUSTDOCFLAGS="-C target-cpu=generic -C link-arg=-fuse-ld=lld" + if [ "$TARGET" = "aarch64-unknown-linux-musl" ]; then + export ZPAQ_NOJIT=1 + fi - RUSTFLAGS="-D warnings ${MUSL_LD_RUSTFLAGS}" cargo check --target "$TARGET" --locked - RUSTFLAGS="-D warnings ${MUSL_LD_RUSTFLAGS}" cargo check --target "$TARGET" --no-default-features --locked - RUSTFLAGS="-D warnings ${MUSL_LD_RUSTFLAGS}" cargo check --target "$TARGET" --features cli --locked - RUSTFLAGS="-D warnings ${MUSL_LD_RUSTFLAGS}" cargo check --target "$TARGET" --no-default-features --features cli --locked - - RUSTFLAGS="${MUSL_LD_RUSTFLAGS}" RUSTDOCFLAGS="${MUSL_RUSTDOCFLAGS}" cargo test --release --target "$TARGET" --locked - RUSTFLAGS="${MUSL_LD_RUSTFLAGS}" RUSTDOCFLAGS="${MUSL_RUSTDOCFLAGS}" cargo test --release --target "$TARGET" --no-default-features --locked - RUSTFLAGS="${MUSL_LD_RUSTFLAGS}" RUSTDOCFLAGS="${MUSL_RUSTDOCFLAGS}" cargo test --release --target "$TARGET" --features cli --locked - RUSTFLAGS="${MUSL_LD_RUSTFLAGS}" RUSTDOCFLAGS="${MUSL_RUSTDOCFLAGS}" cargo test --release --target "$TARGET" --no-default-features --features cli --locked - - RUSTFLAGS="${MUSL_LD_RUSTFLAGS}" cargo rustc --release --features cli --bin infotheory --target "$TARGET" --locked -- ${RELEASE_RUSTC_FLAGS} + RUSTFLAGS="-D warnings ${MUSL_LD_RUSTFLAGS}" cargo check -p infotheory --target "$TARGET" --locked + RUSTFLAGS="-D warnings ${MUSL_LD_RUSTFLAGS}" cargo check -p infotheory --target "$TARGET" --no-default-features --locked + RUSTFLAGS="-D warnings ${MUSL_LD_RUSTFLAGS}" cargo check -p infotheory --target "$TARGET" --no-default-features --features backend-rosa --locked + RUSTFLAGS="-D warnings ${MUSL_LD_RUSTFLAGS}" cargo check -p infotheory --target "$TARGET" --no-default-features --features backend-ctw --locked + RUSTFLAGS="-D warnings ${MUSL_LD_RUSTFLAGS}" cargo check -p infotheory --target "$TARGET" --no-default-features --features backend-match --locked + RUSTFLAGS="-D warnings ${MUSL_LD_RUSTFLAGS}" cargo check -p infotheory --target "$TARGET" --no-default-features --features backend-ppmd --locked + RUSTFLAGS="-D warnings ${MUSL_LD_RUSTFLAGS}" cargo check -p infotheory --target "$TARGET" --no-default-features --features backend-sequitur --locked + RUSTFLAGS="-D warnings ${MUSL_LD_RUSTFLAGS}" cargo check -p infotheory --target "$TARGET" --no-default-features --features backend-mixture --locked + RUSTFLAGS="-D warnings ${MUSL_LD_RUSTFLAGS}" cargo check -p infotheory --target "$TARGET" --no-default-features --features backend-particle --locked + RUSTFLAGS="-D warnings ${MUSL_LD_RUSTFLAGS}" cargo check -p infotheory --target "$TARGET" --no-default-features --features backend-calibrated --locked + RUSTFLAGS="-D warnings ${MUSL_LD_RUSTFLAGS}" cargo check -p infotheory --target "$TARGET" --no-default-features --features backend-zpaq --locked + RUSTFLAGS="-D warnings ${MUSL_LD_RUSTFLAGS}" cargo check -p infotheory --target "$TARGET" --no-default-features --features backend-rwkv --locked + RUSTFLAGS="-D warnings ${MUSL_LD_RUSTFLAGS}" cargo check -p infotheory --target "$TARGET" --no-default-features --features backend-mamba --locked + RUSTFLAGS="-D warnings ${MUSL_LD_RUSTFLAGS}" cargo check -p infotheory --target "$TARGET" --no-default-features --features all-backends --locked + RUSTFLAGS="-D warnings ${MUSL_LD_RUSTFLAGS}" cargo check -p infotheory --target "$TARGET" --features cli --locked + RUSTFLAGS="-D warnings ${MUSL_LD_RUSTFLAGS}" cargo check -p infotheory --target "$TARGET" --no-default-features --features cli --locked + RUSTFLAGS="-D warnings ${MUSL_LD_RUSTFLAGS}" cargo check -p infotheory --target "$TARGET" --no-default-features --features "cli all-backends" --locked + RUSTFLAGS="-D warnings ${MUSL_LD_RUSTFLAGS}" cargo check -p zpaq_rs --target "$TARGET" --locked + RUSTFLAGS="-D warnings ${MUSL_LD_RUSTFLAGS}" cargo check -p benchman --target "$TARGET" --locked + + RUSTFLAGS="${MUSL_LD_RUSTFLAGS}" RUSTDOCFLAGS="${MUSL_RUSTDOCFLAGS}" cargo test -p infotheory --release --target "$TARGET" --locked + RUSTFLAGS="${MUSL_LD_RUSTFLAGS}" RUSTDOCFLAGS="${MUSL_RUSTDOCFLAGS}" cargo test -p infotheory --release --target "$TARGET" --no-default-features --features all-backends --locked + RUSTFLAGS="${MUSL_LD_RUSTFLAGS}" RUSTDOCFLAGS="${MUSL_RUSTDOCFLAGS}" cargo test -p infotheory --release --target "$TARGET" --no-default-features rate_backend_aliases_share_registry_resolution_and_feature_errors --locked + RUSTFLAGS="${MUSL_LD_RUSTFLAGS}" RUSTDOCFLAGS="${MUSL_RUSTDOCFLAGS}" cargo test -p infotheory --release --target "$TARGET" --no-default-features compression_backend_aliases_share_registry_resolution_and_feature_errors --locked + RUSTFLAGS="${MUSL_LD_RUSTFLAGS}" RUSTDOCFLAGS="${MUSL_RUSTDOCFLAGS}" cargo test -p infotheory --release --target "$TARGET" --no-default-features --features backend-ctw --test api_surface --locked + RUSTFLAGS="${MUSL_LD_RUSTFLAGS}" RUSTDOCFLAGS="${MUSL_RUSTDOCFLAGS}" cargo test -p infotheory --release --target "$TARGET" --no-default-features --features backend-rosa --test compression_validation --locked + RUSTFLAGS="${MUSL_LD_RUSTFLAGS}" RUSTDOCFLAGS="${MUSL_RUSTDOCFLAGS}" cargo test -p infotheory --release --target "$TARGET" --no-default-features --features backend-rosa --test api_surface --locked + RUSTFLAGS="${MUSL_LD_RUSTFLAGS}" RUSTDOCFLAGS="${MUSL_RUSTDOCFLAGS}" cargo test -p infotheory --release --target "$TARGET" --no-default-features --features backend-zpaq --test api_surface --locked + RUSTFLAGS="${MUSL_LD_RUSTFLAGS}" RUSTDOCFLAGS="${MUSL_RUSTDOCFLAGS}" cargo test -p infotheory --release --target "$TARGET" --no-default-features --features backend-rwkv parse_compression_backend_name_method_wraps_rwkv_cfg_methods_as_rate_backend --locked + RUSTFLAGS="${MUSL_LD_RUSTFLAGS}" RUSTDOCFLAGS="${MUSL_RUSTDOCFLAGS}" cargo test -p infotheory --release --target "$TARGET" --no-default-features --features backend-rwkv parse_compression_backend_json_wraps_rwkv_cfg_methods_as_rate_backend --locked + RUSTFLAGS="${MUSL_LD_RUSTFLAGS}" RUSTDOCFLAGS="${MUSL_RUSTDOCFLAGS}" cargo test -p infotheory --release --target "$TARGET" --no-default-features --features backend-mamba parse_compression_backend_json_wraps_mamba_cfg_rate_backend --locked + RUSTFLAGS="${MUSL_LD_RUSTFLAGS}" RUSTDOCFLAGS="${MUSL_RUSTDOCFLAGS}" cargo test -p infotheory --release --target "$TARGET" --features cli --locked + RUSTFLAGS="${MUSL_LD_RUSTFLAGS}" RUSTDOCFLAGS="${MUSL_RUSTDOCFLAGS}" cargo test -p infotheory --release --target "$TARGET" --no-default-features --features "cli all-backends" --locked + RUSTFLAGS="${MUSL_LD_RUSTFLAGS}" RUSTDOCFLAGS="${MUSL_RUSTDOCFLAGS}" cargo test -p zpaq_rs --release --target "$TARGET" --locked + RUSTFLAGS="${MUSL_LD_RUSTFLAGS}" RUSTDOCFLAGS="${MUSL_RUSTDOCFLAGS}" cargo test -p benchman --release --target "$TARGET" --locked + + RUSTFLAGS="${MUSL_LD_RUSTFLAGS}" cargo rustc -p infotheory --release --features cli --bin infotheory --target "$TARGET" --locked -- ${RELEASE_RUSTC_FLAGS} ' - name: Package binary artifact diff --git a/.gitignore b/.gitignore index 82ebbb9a..ff489acc 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,9 @@ build/ # Local compiled extension dropped into source package by maturin develop python/infotheory_rs/_core*.so python/infotheory_rs/_core*.pyd + + +# Hutter's Papers +docs/hutter + +docs/misc diff --git a/.gitmodules b/.gitmodules index 9ce8a971..88d6a77f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,9 @@ [submodule "nyx-lite"] - path = nyx-lite + path = vendor/nyx-lite url = https://github.com/turtle261/nyx-lite.git [submodule "zpaq_rs"] - path = zpaq_rs + path = vendor/zpaq_rs url = https://github.com/turtle261/zpaq-rs +[submodule "gameengine"] + path = vendor/gameengine + url = https://github.com/turtle261/gameengine.git diff --git a/Cargo.lock b/Cargo.lock index c6ff435f..4c0ad3fc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,22 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "ab_glyph" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01c0457472c38ea5bd1c3b5ada5e368271cb550be7a4ca4a0b4634e9913f6cc2" +dependencies = [ + "ab_glyph_rasterizer", + "owned_ttf_parser", +] + +[[package]] +name = "ab_glyph_rasterizer" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "366ffbaa4442f4684d91e2cd7c5ea7c4ed8add41959a31447066e279e432b618" + [[package]] name = "acpi_tables" version = "0.1.0" @@ -19,6 +35,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ "cfg-if", + "getrandom 0.3.4", "once_cell", "version_check", "zerocopy", @@ -39,6 +56,40 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" +[[package]] +name = "android-activity" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f2a1bb052857d5dd49572219344a7332b31b76405648eabac5bc68978251bcd" +dependencies = [ + "android-properties", + "bitflags 2.11.0", + "cc", + "jni", + "libc", + "log", + "ndk", + "ndk-context", + "ndk-sys", + "num_enum", + "thiserror 2.0.18", +] + +[[package]] +name = "android-properties" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7eb209b1518d6bb87b283c20095f5228ecda460da70b44f0802523dea6da04" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "anes" version = "0.1.6" @@ -47,9 +98,9 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" [[package]] name = "anstream" -version = "0.6.21" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" dependencies = [ "anstyle", "anstyle-parse", @@ -62,15 +113,15 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.13" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" [[package]] name = "anstyle-parse" -version = "0.2.7" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" dependencies = [ "utf8parse", ] @@ -101,12 +152,39 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + [[package]] name = "arrayvec" version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" +[[package]] +name = "as-raw-xcb-connection" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175571dd1d178ced59193a6fc02dde1b972eb0bc56c892cde9beeceac5bf0f6b" + +[[package]] +name = "ash" +version = "0.38.0+1.3.281" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bb44936d800fea8f016d7f2311c6a4f97aebd5dc86f09906139ec848cf3a46f" +dependencies = [ + "libloading", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "autocfg" version = "1.5.0" @@ -115,9 +193,9 @@ checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] name = "aws-lc-fips-sys" -version = "0.13.13" +version = "0.13.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8bce4948d2520386c6d92a6ea2d472300257702242e5a1d01d6add52bd2e7c1" +checksum = "d3d619165468401dec3caa3366ebffbcb83f2f31883e5b3932f8e2dec2ddc568" dependencies = [ "bindgen 0.72.1", "cc", @@ -141,9 +219,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.39.0" +version = "0.39.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fa7e52a4c5c547c741610a2c6f123f3881e409b714cd27e6798ef020c514f0a" +checksum = "83a25cf98105baa966497416dbd42565ce3a8cf8dbfd59803ec9ad46f3126399" dependencies = [ "bindgen 0.72.1", "cc", @@ -218,17 +296,32 @@ dependencies = [ "bitflags 2.11.0", "cexpr", "clang-sys", - "itertools 0.12.1", + "itertools 0.13.0", "log", "prettyplease", "proc-macro2", "quote", "regex", - "rustc-hash 2.1.1", + "rustc-hash 2.1.2", "shlex", "syn", ] +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" + [[package]] name = "bitflags" version = "1.3.2" @@ -254,6 +347,12 @@ dependencies = [ "wyz", ] +[[package]] +name = "block" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d8c1fef690941d3e7788d328517591fecc684c084084702d6ff1641e993699a" + [[package]] name = "block-buffer" version = "0.10.4" @@ -263,6 +362,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block2" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c132eebf10f5cad5289222520a4a058514204aed6d791f1cf4fe8088b82d15f" +dependencies = [ + "objc2", +] + [[package]] name = "bumpalo" version = "3.20.2" @@ -274,6 +382,20 @@ name = "bytemuck" version = "1.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9abbd1bc6865053c427f7198e6af43bfdedc55ab791faed4fbd361d789575ff" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] [[package]] name = "byteorder" @@ -282,13 +404,35 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] -name = "cargo_toml" -version = "0.22.3" +name = "bytes" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" + +[[package]] +name = "calloop" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "374b7c592d9c00c1f4972ea58390ac6b18cbb6ab79011f3bdc90a0b82ca06b77" +checksum = "b99da2f8558ca23c71f4fd15dc57c906239752dd27ff3c00a1d56b685b7cbfec" dependencies = [ - "serde", - "toml", + "bitflags 2.11.0", + "log", + "polling", + "rustix 0.38.44", + "slab", + "thiserror 1.0.69", +] + +[[package]] +name = "calloop-wayland-source" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95a66a987056935f7efce4ab5668920b5d0dac4a7c99991a67395f13702ddd20" +dependencies = [ + "calloop", + "rustix 0.38.44", + "wayland-backend", + "wayland-client", ] [[package]] @@ -314,9 +458,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.56" +version = "1.2.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aebf35691d1bfb0ac386a69bac2fde4dd276fb618cf8bf4f5318fe285e821bb2" +checksum = "b7a4d3ec6524d28a329fc53654bbadc9bdd7b0431f5d65f1a56ffb28a1ee5283" dependencies = [ "find-msvc-tools", "jobserver", @@ -339,6 +483,23 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "chacha20" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core", +] + [[package]] name = "ciborium" version = "0.2.2" @@ -379,9 +540,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.60" +version = "4.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2797f34da339ce31042b27d23607e051786132987f595b02ba4f6a6dffb7030a" +checksum = "b193af5b67834b676abd72466a96c1024e6a6ad978a1f484bd90b85c94041351" dependencies = [ "clap_builder", "clap_derive", @@ -389,9 +550,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.60" +version = "4.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24a241312cea5059b13574bb9b3861cabf758b879c15190b37b6d6fd63ab6876" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" dependencies = [ "anstream", "anstyle", @@ -401,9 +562,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.55" +version = "4.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a92793da1a46a5f2a02a6f4c46c6496b28c43638adea8306fcb0caa1634f24e5" +checksum = "1110bd8a634a1ab8cb04345d8d878267d57c3cf1b38d91b71af6686408bbca6a" dependencies = [ "heck", "proc-macro2", @@ -413,24 +574,45 @@ dependencies = [ [[package]] name = "clap_lex" -version = "1.0.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a822ea5bc7590f9d40f1ba12c0dc3c2760f3482c6984db1573ad11031420831" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" [[package]] name = "cmake" -version = "0.1.57" +version = "0.1.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75443c44cd6b379beb8c5b45d85d0773baf31cce901fe7bb252f4eff3008ef7d" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" dependencies = [ "cc", ] +[[package]] +name = "codespan-reporting" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe6d2e5af09e8c8ad56c969f2157a3d4238cebc7c55f0a517728c38f7b200f81" +dependencies = [ + "serde", + "termcolor", + "unicode-width 0.2.0", +] + [[package]] name = "colorchoice" -version = "1.0.4" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "combine" +version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] [[package]] name = "compact_str" @@ -446,6 +628,15 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "convert_case" version = "0.10.0" @@ -455,6 +646,100 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "core-graphics" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c07782be35f9e1140080c6b96f0d44b739e2278479f64e02fdab4e32dfd8b081" +dependencies = [ + "bitflags 1.3.2", + "core-foundation 0.9.4", + "core-graphics-types 0.1.3", + "foreign-types", + "libc", +] + +[[package]] +name = "core-graphics-types" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45390e6114f68f718cc7a830514a96f903cccd70d02a8f6d9f643ac4ba45afaf" +dependencies = [ + "bitflags 1.3.2", + "core-foundation 0.9.4", + "libc", +] + +[[package]] +name = "core-graphics-types" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d44a101f213f6c4cdc1853d4b78aef6db6bdfa3468798cc1d9912f4735013eb" +dependencies = [ + "bitflags 2.11.0", + "core-foundation 0.10.1", + "libc", +] + +[[package]] +name = "core_maths" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77745e017f5edba1a9c1d854f6f3a52dac8a12dd5af5d2f54aecf61e43d80d30" +dependencies = [ + "libm", +] + +[[package]] +name = "cosmic-text" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "173852283a9a57a3cbe365d86e74dc428a09c50421477d5ad6fe9d9509e37737" +dependencies = [ + "bitflags 2.11.0", + "fontdb", + "harfrust", + "linebender_resource_handle", + "log", + "rangemap", + "rustc-hash 1.1.0", + "self_cell", + "skrifa 0.37.0", + "smol_str", + "swash", + "sys-locale", + "unicode-bidi", + "unicode-linebreak", + "unicode-script", + "unicode-segmentation", +] + [[package]] name = "cpufeatures" version = "0.2.17" @@ -464,6 +749,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -489,7 +783,7 @@ dependencies = [ "cast", "ciborium", "clap", - "criterion-plot 0.5.0", + "criterion-plot", "is-terminal", "itertools 0.10.5", "num-traits", @@ -503,27 +797,6 @@ dependencies = [ "walkdir", ] -[[package]] -name = "criterion" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1c047a62b0cc3e145fa84415a3191f628e980b194c2755aa12300a4e6cbd928" -dependencies = [ - "anes", - "cast", - "ciborium", - "clap", - "criterion-plot 0.6.0", - "itertools 0.13.0", - "num-traits", - "oorandom", - "regex", - "serde", - "serde_json", - "tinytemplate", - "walkdir", -] - [[package]] name = "criterion-plot" version = "0.5.0" @@ -534,16 +807,6 @@ dependencies = [ "itertools 0.10.5", ] -[[package]] -name = "criterion-plot" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b1bcc0dc7dfae599d84ad0b1a55f80cde8af3725da8313b528da95ef783e338" -dependencies = [ - "cast", - "itertools 0.13.0", -] - [[package]] name = "crossbeam-deque" version = "0.8.6" @@ -649,6 +912,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "cursor-icon" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f27ae1dd37df86211c42e150270f82743308803d90a6f6e6651cd730d5e1732f" + [[package]] name = "darling" version = "0.23.0" @@ -706,12 +975,6 @@ dependencies = [ "unicode-xid", ] -[[package]] -name = "device_tree" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f18f717c5c7c2e3483feb64cccebd077245ad6d19007c2db0fd341d38595353c" - [[package]] name = "digest" version = "0.10.7" @@ -722,6 +985,12 @@ dependencies = [ "crypto-common", ] +[[package]] +name = "dispatch" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd0c93bb4b0c6d9b77f4435b0ae98c24d17f1c45b2ff844c6151a07256ca923b" + [[package]] name = "displaydoc" version = "0.2.5" @@ -733,6 +1002,15 @@ dependencies = [ "syn", ] +[[package]] +name = "dlib" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab8ecd87370524b461f8557c119c405552c396ed91fc0a8eec68679eab26f94a" +dependencies = [ + "libloading", +] + [[package]] name = "document-features" version = "0.2.12" @@ -742,6 +1020,18 @@ dependencies = [ "litrs", ] +[[package]] +name = "downcast-rs" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2" + +[[package]] +name = "dpi" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8b14ccef22fc6f5a8f4d7d768562a182c04ce9a3b3157b91390b52ddfdf1a76" + [[package]] name = "dunce" version = "1.0.5" @@ -755,42 +1045,38 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" [[package]] -name = "env_filter" -version = "1.0.0" +name = "equivalent" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a1c3cc8e57274ec99de65301228b537f1e4eedc1b8e0f9411c6caac8ae7308f" -dependencies = [ - "log", - "regex", -] +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" [[package]] -name = "env_logger" -version = "0.11.9" +name = "errno" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2daee4ea451f429a58296525ddf28b45a3b64f1acf6587e2067437bb11e218d" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ - "anstream", - "anstyle", - "env_filter", - "jiff", - "log", + "libc", + "windows-sys 0.61.2", ] [[package]] -name = "equivalent" -version = "1.0.2" +name = "etagere" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +checksum = "fc89bf99e5dc15954a60f707c1e09d7540e5cd9af85fa75caa0b510bc08c5342" +dependencies = [ + "euclid", + "svg_fmt", +] [[package]] -name = "errno" -version = "0.3.14" +name = "euclid" +version = "0.22.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +checksum = "f1a05365e3b1c6d1650318537c7460c6923f1abdd272ad6842baa2b509957a06" dependencies = [ - "libc", - "windows-sys 0.61.2", + "num-traits", ] [[package]] @@ -813,20 +1099,16 @@ checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" name = "firecracker" version = "1.14.0" dependencies = [ - "cargo_toml", "displaydoc", "event-manager", "libc", - "log-instrument", "micro_http", - "regex", "seccompiler", "serde", "serde_derive", "serde_json", "thiserror 2.0.18", "timerfd", - "userfaultfd", "utils", "vmm", "vmm-sys-util", @@ -838,6 +1120,80 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "font-types" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39a654f404bbcbd48ea58c617c2993ee91d1cb63727a37bf2323a4edeed1b8c5" +dependencies = [ + "bytemuck", +] + +[[package]] +name = "font-types" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b38ad915f6dadd993ced50848a8291a543bd41ca62bc10740d5e64e2ab4cfd7" +dependencies = [ + "bytemuck", +] + +[[package]] +name = "fontconfig-parser" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbc773e24e02d4ddd8395fd30dc147524273a83e54e0f312d986ea30de5f5646" +dependencies = [ + "roxmltree", +] + +[[package]] +name = "fontdb" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "457e789b3d1202543297a350643cf459f836cade38934e7a4cf6a39e7cde2905" +dependencies = [ + "fontconfig-parser", + "log", + "memmap2", + "slotmap", + "tinyvec", + "ttf-parser", +] + +[[package]] +name = "foreign-types" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d737d9aa519fb7b749cbc3b962edcf310a8dd1f4b67c91c4f83975dbdd17d965" +dependencies = [ + "foreign-types-macros", + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-macros" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a5c6c585bc94aaf2c7b51dd4c2ba22680844aba4c687be581871a6f518c5742" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "foreign-types-shared" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa9a19cbb55df58761df49b23516a86d432839add4af60fc256da840f66ed35b" + [[package]] name = "fs_extra" version = "1.3.0" @@ -851,27 +1207,43 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" [[package]] -name = "gdbstub" -version = "0.7.9" +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bf845b08f7c2ef3b5ad19f80779d43ae20d278652b91bb80adda65baf2d8ed6" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" dependencies = [ - "bitflags 2.11.0", - "cfg-if", - "log", - "managed", - "num-traits", - "paste", + "futures-core", + "futures-task", + "pin-project-lite", + "slab", ] [[package]] -name = "gdbstub_arch" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22dde0e1b68787036ccedd0b1ff6f953527a0e807e571fbe898975203027278f" +name = "gameengine" +version = "0.3.1" dependencies = [ - "gdbstub", - "num-traits", + "bytemuck", + "criterion", + "glyphon", + "pollster", + "rayon", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "wgpu", + "winit", ] [[package]] @@ -884,6 +1256,16 @@ dependencies = [ "version_check", ] +[[package]] +name = "gethostname" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bd49230192a3797a9a4d6abe9b3eed6f7fa4c8a8a4947977c6f80025f92cbd8" +dependencies = [ + "rustix 1.1.4", + "windows-link", +] + [[package]] name = "getrandom" version = "0.3.4" @@ -892,29 +1274,109 @@ checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", "libc", - "r-efi", + "r-efi 5.3.0", "wasip2", ] [[package]] name = "getrandom" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "139ef39800118c7683f2fd3c98c1b23c09ae076556b435f8e9064ae108aaeeec" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" dependencies = [ "cfg-if", "libc", - "r-efi", + "r-efi 6.0.0", + "rand_core", "wasip2", "wasip3", ] +[[package]] +name = "gl_generator" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a95dfc23a2b4a9a2f5ab41d194f8bfda3cabec42af4e39f08c339eb2a0c124d" +dependencies = [ + "khronos_api", + "log", + "xml-rs", +] + [[package]] name = "glob" version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" +[[package]] +name = "glow" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5e5ea60d70410161c8bf5da3fdfeaa1c72ed2c15f8bbb9d19fe3a4fad085f08" +dependencies = [ + "js-sys", + "slotmap", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "glutin_wgl_sys" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c4ee00b289aba7a9e5306d57c2d05499b2e5dc427f84ac708bd2c090212cf3e" +dependencies = [ + "gl_generator", +] + +[[package]] +name = "glyphon" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "201b40693830fc7f5e53159003e694d40fa56d32352224439ca93f7654ce3566" +dependencies = [ + "cosmic-text", + "etagere", + "lru 0.16.4", + "rustc-hash 2.1.2", + "wgpu", +] + +[[package]] +name = "gpu-allocator" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51255ea7cfaadb6c5f1528d43e92a82acb2b96c43365989a28b2d44ee38f8795" +dependencies = [ + "ash", + "hashbrown 0.16.1", + "log", + "presser", + "thiserror 2.0.18", + "windows", +] + +[[package]] +name = "gpu-descriptor" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b89c83349105e3732062a895becfc71a8f921bb71ecbbdd8ff99263e3b53a0ca" +dependencies = [ + "bitflags 2.11.0", + "gpu-descriptor-types", + "hashbrown 0.15.5", +] + +[[package]] +name = "gpu-descriptor-types" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdf242682df893b86f33a73828fb09ca4b2d3bb6cc95249707fc684d27484b91" +dependencies = [ + "bitflags 2.11.0", +] + [[package]] name = "half" version = "2.7.1" @@ -923,9 +1385,23 @@ checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" dependencies = [ "cfg-if", "crunchy", + "num-traits", "zerocopy", ] +[[package]] +name = "harfrust" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92c020db12c71d8a12a3fe7607873cade3a01a6287e29d540c8723276221b9d8" +dependencies = [ + "bitflags 2.11.0", + "bytemuck", + "core_maths", + "read-fonts 0.35.0", + "smallvec", +] + [[package]] name = "hashbrown" version = "0.15.5" @@ -934,7 +1410,7 @@ checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ "allocator-api2", "equivalent", - "foldhash", + "foldhash 0.1.5", ] [[package]] @@ -942,6 +1418,11 @@ name = "hashbrown" version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] [[package]] name = "heck" @@ -961,6 +1442,12 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hexf-parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfa686283ad6dd069f105e5ab091b04c62850d3e4cf5d67debad1933f55023df" + [[package]] name = "iced-x86" version = "1.21.0" @@ -984,9 +1471,9 @@ checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" [[package]] name = "indexmap" -version = "2.13.0" +version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +checksum = "45a8a2b9cb3e0b0c1803dbb0758ffac5de2f425b23c28f518faabd9d805342ff" dependencies = [ "equivalent", "hashbrown 0.16.1", @@ -1005,12 +1492,14 @@ dependencies = [ [[package]] name = "infotheory" -version = "1.1.1" +version = "1.2.0" dependencies = [ "ahash", "anyhow", "crc32fast", - "criterion 0.5.1", + "criterion", + "gameengine", + "libc", "num_cpus", "nyx-lite", "once_cell", @@ -1019,12 +1508,12 @@ dependencies = [ "serde_json", "sha2", "wide", - "zpaq_rs 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", + "zpaq_rs", ] [[package]] name = "infotheory_py" -version = "1.1.1" +version = "1.2.0" dependencies = [ "anyhow", "infotheory", @@ -1035,9 +1524,9 @@ dependencies = [ [[package]] name = "instability" -version = "0.3.11" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "357b7205c6cd18dd2c86ed312d1e70add149aea98e7ef72b9fdf0270e555c11d" +checksum = "5eb2d60ef19920a3a9193c3e371f726ec1dafc045dac788d0fb3704272458971" dependencies = [ "darling", "indoc", @@ -1091,40 +1580,65 @@ dependencies = [ ] [[package]] -name = "itertools" -version = "0.14.0" +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jni" +version = "0.22.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" dependencies = [ - "either", + "cfg-if", + "combine", + "jni-macros", + "jni-sys 0.4.1", + "log", + "simd_cesu8", + "thiserror 2.0.18", + "walkdir", + "windows-link", ] [[package]] -name = "itoa" -version = "1.0.17" +name = "jni-macros" +version = "0.22.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn", +] [[package]] -name = "jiff" -version = "0.2.21" +name = "jni-sys" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3e3d65f018c6ae946ab16e80944b97096ed73c35b221d1c478a6c81d8f57940" +checksum = "41a652e1f9b6e0275df1f15b32661cf0d4b78d4d87ddec5e0c3c20f097433258" dependencies = [ - "jiff-static", - "log", - "portable-atomic", - "portable-atomic-util", - "serde_core", + "jni-sys 0.4.1", ] [[package]] -name = "jiff-static" -version = "0.2.21" +name = "jni-sys" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a17c2b211d863c7fde02cbea8a3c1a439b98e109286554f2860bdded7ff83818" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" dependencies = [ - "proc-macro2", "quote", "syn", ] @@ -1141,14 +1655,33 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.90" +version = "0.3.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14dc6f6450b3f6d4ed5b16327f38fed626d375a886159ca555bd7822c0c3a5a6" +checksum = "2964e92d1d9dc3364cae4d718d93f227e3abb088e747d92e0395bfdedf1c12ca" dependencies = [ + "cfg-if", + "futures-util", "once_cell", "wasm-bindgen", ] +[[package]] +name = "khronos-egl" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6aae1df220ece3c0ada96b8153459b67eebe9ae9212258bb0134ae60416fdf76" +dependencies = [ + "libc", + "libloading", + "pkg-config", +] + +[[package]] +name = "khronos_api" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2db585e1d738fc771bf08a151420d3ed193d9d895a36df7f6f8a9456b911ddc" + [[package]] name = "kvm-bindings" version = "0.14.0" @@ -1192,9 +1725,9 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" [[package]] name = "libc" -version = "0.2.182" +version = "0.2.184" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6800badb6cb2082ffd7b6a67e6125bb39f18782f793520caee8cb8846be06112" +checksum = "48f5d2a454e16a5ea0f4ced81bd44e4cfc7bd3a507b61887c99fd3538b28e4af" [[package]] name = "libloading" @@ -1206,6 +1739,30 @@ dependencies = [ "windows-link", ] +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "libredox" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e02f3bb43d335493c96bf3fd3a321600bf6bd07ed34bc64118e9293bdffea46c" +dependencies = [ + "bitflags 2.11.0", + "libc", + "plain", + "redox_syscall 0.7.4", +] + +[[package]] +name = "linebender_resource_handle" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4a5ff6bcca6c4867b1c4fd4ef63e4db7436ef363e0ad7531d1558856bae64f4" + [[package]] name = "linux-loader" version = "0.13.2" @@ -1251,24 +1808,6 @@ dependencies = [ "serde_core", ] -[[package]] -name = "log-instrument" -version = "0.3.0" -dependencies = [ - "env_logger", - "log", - "log-instrument-macros", -] - -[[package]] -name = "log-instrument-macros" -version = "0.1.0" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "lru" version = "0.12.5" @@ -1279,10 +1818,19 @@ dependencies = [ ] [[package]] -name = "managed" -version = "0.8.0" +name = "lru" +version = "0.16.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f66e8d5d03f609abc3a39e6f08e4164ebf1447a732906d39eb9b99b7919ef39" + +[[package]] +name = "malloc_buf" +version = "0.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ca88d725a0a943b096803bd34e73a4437208b6077654cc4ecb2947a5f91618d" +checksum = "62bb907fe88d54d8d9ce32a3cceab4218ed2f6b7d35617cafe9adf84e43919cb" +dependencies = [ + "libc", +] [[package]] name = "memchr" @@ -1300,25 +1848,49 @@ dependencies = [ ] [[package]] -name = "micro_http" -version = "0.1.0" -source = "git+https://github.com/firecracker-microvm/micro-http#5c2254d6cf4f32a668d0d8e57ba20bebad9d4fba" +name = "memmap2" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3" dependencies = [ "libc", - "vmm-sys-util", ] [[package]] -name = "minimal-lexical" -version = "0.2.1" +name = "metal" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +checksum = "c7047791b5bc903b8cd963014b355f71dc9864a9a0b727057676c1dcae5cbc15" +dependencies = [ + "bitflags 2.11.0", + "block", + "core-graphics-types 0.2.0", + "foreign-types", + "log", + "objc", + "paste", +] + +[[package]] +name = "micro_http" +version = "0.1.0" +source = "git+https://github.com/firecracker-microvm/micro-http#5c2254d6cf4f32a668d0d8e57ba20bebad9d4fba" +dependencies = [ + "libc", + "vmm-sys-util", +] + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "mio" -version = "1.1.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" +checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" dependencies = [ "libc", "log", @@ -1326,6 +1898,62 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "naga" +version = "28.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "618f667225063219ddfc61251087db8a9aec3c3f0950c916b614e403486f1135" +dependencies = [ + "arrayvec", + "bit-set", + "bitflags 2.11.0", + "cfg-if", + "cfg_aliases", + "codespan-reporting", + "half", + "hashbrown 0.16.1", + "hexf-parse", + "indexmap", + "libm", + "log", + "num-traits", + "once_cell", + "rustc-hash 1.1.0", + "spirv", + "thiserror 2.0.18", + "unicode-ident", +] + +[[package]] +name = "ndk" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3f42e7bbe13d351b6bead8286a43aac9534b82bd3cc43e47037f012ebfd62d4" +dependencies = [ + "bitflags 2.11.0", + "jni-sys 0.3.1", + "log", + "ndk-sys", + "num_enum", + "raw-window-handle", + "thiserror 1.0.69", +] + +[[package]] +name = "ndk-context" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" + +[[package]] +name = "ndk-sys" +version = "0.6.0+11769913" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee6cda3051665f1fb8d9e08fc35c96d5a244fb1be711a03b71118828afc9a873" +dependencies = [ + "jni-sys 0.3.1", +] + [[package]] name = "nix" version = "0.27.1" @@ -1354,6 +1982,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", + "libm", ] [[package]] @@ -1366,6 +1995,28 @@ dependencies = [ "libc", ] +[[package]] +name = "num_enum" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0bca838442ec211fa11de3a8b0e0e8f3a4522575b5c4c06ed722e005036f26" +dependencies = [ + "num_enum_derive", + "rustversion", +] + +[[package]] +name = "num_enum_derive" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "680998035259dcfcafe653688bf2aa6d3e2dc05e98be6ab46afb089dc84f1df8" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "nyx-lite" version = "0.1.0" @@ -1386,11 +2037,223 @@ dependencies = [ "vmm", ] +[[package]] +name = "objc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "915b1b472bc21c53464d6c8461c9d3af805ba1ef837e1cac254428f4a77177b1" +dependencies = [ + "malloc_buf", +] + +[[package]] +name = "objc-sys" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb91bdd390c7ce1a8607f35f3ca7151b65afc0ff5ff3b34fa350f7d7c7e4310" + +[[package]] +name = "objc2" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46a785d4eeff09c14c487497c162e92766fbb3e4059a71840cecc03d9a50b804" +dependencies = [ + "objc-sys", + "objc2-encode", +] + +[[package]] +name = "objc2-app-kit" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4e89ad9e3d7d297152b17d39ed92cd50ca8063a89a9fa569046d41568891eff" +dependencies = [ + "bitflags 2.11.0", + "block2", + "libc", + "objc2", + "objc2-core-data", + "objc2-core-image", + "objc2-foundation", + "objc2-quartz-core", +] + +[[package]] +name = "objc2-cloud-kit" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74dd3b56391c7a0596a295029734d3c1c5e7e510a4cb30245f8221ccea96b009" +dependencies = [ + "bitflags 2.11.0", + "block2", + "objc2", + "objc2-core-location", + "objc2-foundation", +] + +[[package]] +name = "objc2-contacts" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5ff520e9c33812fd374d8deecef01d4a840e7b41862d849513de77e44aa4889" +dependencies = [ + "block2", + "objc2", + "objc2-foundation", +] + +[[package]] +name = "objc2-core-data" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "617fbf49e071c178c0b24c080767db52958f716d9eabdf0890523aeae54773ef" +dependencies = [ + "bitflags 2.11.0", + "block2", + "objc2", + "objc2-foundation", +] + +[[package]] +name = "objc2-core-image" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55260963a527c99f1819c4f8e3b47fe04f9650694ef348ffd2227e8196d34c80" +dependencies = [ + "block2", + "objc2", + "objc2-foundation", + "objc2-metal", +] + +[[package]] +name = "objc2-core-location" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "000cfee34e683244f284252ee206a27953279d370e309649dc3ee317b37e5781" +dependencies = [ + "block2", + "objc2", + "objc2-contacts", + "objc2-foundation", +] + +[[package]] +name = "objc2-encode" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" + +[[package]] +name = "objc2-foundation" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ee638a5da3799329310ad4cfa62fbf045d5f56e3ef5ba4149e7452dcf89d5a8" +dependencies = [ + "bitflags 2.11.0", + "block2", + "dispatch", + "libc", + "objc2", +] + +[[package]] +name = "objc2-link-presentation" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1a1ae721c5e35be65f01a03b6d2ac13a54cb4fa70d8a5da293d7b0020261398" +dependencies = [ + "block2", + "objc2", + "objc2-app-kit", + "objc2-foundation", +] + +[[package]] +name = "objc2-metal" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd0cba1276f6023976a406a14ffa85e1fdd19df6b0f737b063b95f6c8c7aadd6" +dependencies = [ + "bitflags 2.11.0", + "block2", + "objc2", + "objc2-foundation", +] + +[[package]] +name = "objc2-quartz-core" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e42bee7bff906b14b167da2bac5efe6b6a07e6f7c0a21a7308d40c960242dc7a" +dependencies = [ + "bitflags 2.11.0", + "block2", + "objc2", + "objc2-foundation", + "objc2-metal", +] + +[[package]] +name = "objc2-symbols" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a684efe3dec1b305badae1a28f6555f6ddd3bb2c2267896782858d5a78404dc" +dependencies = [ + "objc2", + "objc2-foundation", +] + +[[package]] +name = "objc2-ui-kit" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8bb46798b20cd6b91cbd113524c490f1686f4c4e8f49502431415f3512e2b6f" +dependencies = [ + "bitflags 2.11.0", + "block2", + "objc2", + "objc2-cloud-kit", + "objc2-core-data", + "objc2-core-image", + "objc2-core-location", + "objc2-foundation", + "objc2-link-presentation", + "objc2-quartz-core", + "objc2-symbols", + "objc2-uniform-type-identifiers", + "objc2-user-notifications", +] + +[[package]] +name = "objc2-uniform-type-identifiers" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44fa5f9748dbfe1ca6c0b79ad20725a11eca7c2218bceb4b005cb1be26273bfe" +dependencies = [ + "block2", + "objc2", + "objc2-foundation", +] + +[[package]] +name = "objc2-user-notifications" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76cfcbf642358e8689af64cee815d139339f3ed8ad05103ed5eaf73db8d84cb3" +dependencies = [ + "bitflags 2.11.0", + "block2", + "objc2", + "objc2-core-location", + "objc2-foundation", +] + [[package]] name = "once_cell" -version = "1.21.3" +version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "once_cell_polyfill" @@ -1404,6 +2267,34 @@ version = "11.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" +[[package]] +name = "orbclient" +version = "0.3.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12c6933ddbbd16539a7672e697bb8d41ac3a4e99ac43eeb40c07236bd7fcb2dd" +dependencies = [ + "libc", + "libredox", +] + +[[package]] +name = "ordered-float" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7d950ca161dc355eaf28f82b11345ed76c6e1f6eb1f4f4479e0323b9e2fbd0e" +dependencies = [ + "num-traits", +] + +[[package]] +name = "owned_ttf_parser" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36820e9051aca1014ddc75770aab4d68bc1e9e632f0f5627c4086bc216fb583b" +dependencies = [ + "ttf-parser", +] + [[package]] name = "parking_lot" version = "0.12.5" @@ -1422,7 +2313,7 @@ checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.5.18", "smallvec", "windows-link", ] @@ -1441,11 +2332,73 @@ dependencies = [ "libc", "log", "serde", - "serde_test", "thiserror 2.0.18", - "vmm-sys-util", ] +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project" +version = "1.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1749c7ed4bcaf4c3d0a3efc28538844fb29bcdd7d2b67b2be7e20ba861ff517" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b20ed30f105399776b9c883e68e536ef602a16ae6f596d2c473591d6ad64c6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + +[[package]] +name = "plain" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" + +[[package]] +name = "polling" +version = "3.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0e4f59085d47d8241c88ead0f274e8a0cb551f3625263c05eb8dd897c34218" +dependencies = [ + "cfg-if", + "concurrent-queue", + "hermit-abi", + "pin-project-lite", + "rustix 1.1.4", + "windows-sys 0.61.2", +] + +[[package]] +name = "pollster" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f3a9f18d041e6d0e102a0a46750538147e5e8992d3b4873aaafee2520b00ce3" + [[package]] name = "portable-atomic" version = "1.13.1" @@ -1454,21 +2407,18 @@ checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" [[package]] name = "portable-atomic-util" -version = "0.2.5" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a9db96d7fa8782dd8c15ce32ffe8680bbd1e978a43bf51a34d39483540495f5" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" dependencies = [ "portable-atomic", ] [[package]] -name = "ppv-lite86" -version = "0.2.21" +name = "presser" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" -dependencies = [ - "zerocopy", -] +checksum = "e8cf8e6a8aa66ce33f63993ffc4ea4271eb5b0530a9002db8455ea6050c77bfa" [[package]] name = "prettyplease" @@ -1480,6 +2430,15 @@ dependencies = [ "syn", ] +[[package]] +name = "proc-macro-crate" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" +dependencies = [ + "toml_edit", +] + [[package]] name = "proc-macro2" version = "1.0.106" @@ -1490,25 +2449,16 @@ dependencies = [ ] [[package]] -name = "proptest" -version = "1.10.0" +name = "profiling" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37566cb3fdacef14c0737f9546df7cfeadbfbc9fef10991038bf5015d0c80532" -dependencies = [ - "bitflags 2.11.0", - "num-traits", - "rand", - "rand_chacha", - "rand_xorshift", - "regex-syntax", - "unarray", -] +checksum = "3eb8486b569e12e2c32ad3e204dbaba5e4b5b216e9367044f25f1dba42341773" [[package]] name = "pyo3" -version = "0.28.2" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf85e27e86080aafd5a22eae58a162e133a589551542b3e5cee4beb27e54f8e1" +checksum = "cd274650b21d4bfc26a0a47587962c1edb425f69287324355cd040c3ea66071c" dependencies = [ "libc", "once_cell", @@ -1520,18 +2470,18 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.28.2" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bf94ee265674bf76c09fa430b0e99c26e319c945d96ca0d5a8215f31bf81cf7" +checksum = "c5e2a7d2f0d013342f295c048ad19237add5154a55b1c5a254c0ec93d4109078" dependencies = [ "target-lexicon", ] [[package]] name = "pyo3-ffi" -version = "0.28.2" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "491aa5fc66d8059dd44a75f4580a2962c1862a1c2945359db36f6c2818b748dc" +checksum = "ca85c467da1bbc8d866eea5deff9cf29ea5f7785054a17da36e65bda9c05845b" dependencies = [ "libc", "pyo3-build-config", @@ -1539,9 +2489,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.28.2" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5d671734e9d7a43449f8480f8b38115df67bef8d21f76837fa75ee7aaa5e52e" +checksum = "9ac53762fd065daa3194dd09337a38bd793a188100fd1a9304c4ab312d901771" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -1551,22 +2501,30 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.28.2" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22faaa1ce6c430a1f71658760497291065e6450d7b5dc2bcf254d49f66ee700a" +checksum = "4ca3a1557399783172dc5bf39cfca835157732532cba56b71d2292161e53b362" dependencies = [ "heck", "proc-macro2", - "pyo3-build-config", "quote", "syn", ] +[[package]] +name = "quick-xml" +version = "0.39.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "958f21e8e7ceb5a1aa7fa87fab28e7c75976e0bfe7e23ff069e0a260f894067d" +dependencies = [ + "memchr", +] + [[package]] name = "quote" -version = "1.0.44" +version = "1.0.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21b2ebcf727b7760c461f091f9f0f539b77b8e87f2fd88131e7f1b433b3cece4" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" dependencies = [ "proc-macro2", ] @@ -1577,6 +2535,12 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + [[package]] name = "radium" version = "0.7.0" @@ -1585,41 +2549,32 @@ checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" [[package]] name = "rand" -version = "0.9.2" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +checksum = "bc266eb313df6c5c09c1c7b1fbe2510961e5bcd3add930c1e31f7ed9da0feff8" dependencies = [ - "rand_chacha", + "chacha20", + "getrandom 0.4.2", "rand_core", ] [[package]] -name = "rand_chacha" -version = "0.9.0" +name = "rand_core" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" -dependencies = [ - "ppv-lite86", - "rand_core", -] +checksum = "0c8d0fd677905edcbeedbf2edb6494d676f0e98d54d5cf9bda0b061cb8fb8aba" [[package]] -name = "rand_core" -version = "0.9.5" +name = "range-alloc" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" -dependencies = [ - "getrandom 0.3.4", -] +checksum = "ca45419789ae5a7899559e9512e58ca889e41f04f1f2445e9f4b290ceccd1d08" [[package]] -name = "rand_xorshift" -version = "0.4.0" +name = "rangemap" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a" -dependencies = [ - "rand_core", -] +checksum = "973443cf09a9c8656b574a866ab68dfa19f0867d0340648c7d2f6a71b8a8ea68" [[package]] name = "ratatui" @@ -1634,7 +2589,7 @@ dependencies = [ "indoc", "instability", "itertools 0.13.0", - "lru", + "lru 0.12.5", "paste", "strum", "unicode-segmentation", @@ -1642,6 +2597,12 @@ dependencies = [ "unicode-width 0.2.0", ] +[[package]] +name = "raw-window-handle" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20675572f6f24e9e76ef639bc5552774ed45f1c30e2951e1e99c59888861c539" + [[package]] name = "rayon" version = "1.11.0" @@ -1662,6 +2623,36 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "read-fonts" +version = "0.35.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6717cf23b488adf64b9d711329542ba34de147df262370221940dfabc2c91358" +dependencies = [ + "bytemuck", + "core_maths", + "font-types 0.10.1", +] + +[[package]] +name = "read-fonts" +version = "0.37.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b634fabf032fab15307ffd272149b622260f55974d9fad689292a5d33df02e5" +dependencies = [ + "bytemuck", + "font-types 0.11.3", +] + +[[package]] +name = "redox_syscall" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -1671,6 +2662,15 @@ dependencies = [ "bitflags 2.11.0", ] +[[package]] +name = "redox_syscall" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f450ad9c3b1da563fb6948a8e0fb0fb9269711c9c73d9ea1de5058c79c8d643a" +dependencies = [ + "bitflags 2.11.0", +] + [[package]] name = "regex" version = "1.12.3" @@ -1700,6 +2700,18 @@ version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" +[[package]] +name = "renderdoc-sys" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b30a45b0cd0bcca8037f3d0dc3421eaf95327a17cad11964fb8179b4fc4832" + +[[package]] +name = "roxmltree" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c20b6793b5c2fa6553b250154b78d6d0db37e72700ae35fad9387a46f487c97" + [[package]] name = "rustc-hash" version = "1.1.0" @@ -1708,9 +2720,9 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] name = "rustc-hash" -version = "2.1.1" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" [[package]] name = "rustc_version" @@ -1777,12 +2789,31 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + [[package]] name = "scopeguard" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "sctk-adwaita" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6277f0217056f77f1d8f49f2950ac6c278c0d607c45f5ee99328d792ede24ec" +dependencies = [ + "ab_glyph", + "log", + "memmap2", + "smithay-client-toolkit", + "tiny-skia", +] + [[package]] name = "seccompiler" version = "1.14.0" @@ -1797,11 +2828,17 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "self_cell" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b12e76d157a900eb52e81bc6e9f3069344290341720e9178cde2407113ac8d89" + [[package]] name = "semver" -version = "1.0.27" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" dependencies = [ "serde", "serde_core", @@ -1850,24 +2887,6 @@ dependencies = [ "zmij", ] -[[package]] -name = "serde_spanned" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8bbf91e5a4d6315eee45e704372590b30e260ee83af6639d64557f51b067776" -dependencies = [ - "serde_core", -] - -[[package]] -name = "serde_test" -version = "1.0.177" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f901ee573cab6b3060453d2d5f0bae4e6d628c23c0a962ff9b5f1d7c8d4f1ed" -dependencies = [ - "serde", -] - [[package]] name = "sha2" version = "0.10.9" @@ -1875,7 +2894,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest", ] @@ -1910,23 +2929,111 @@ dependencies = [ name = "signal-hook-registry" version = "1.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "simd_cesu8" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94f90157bb87cddf702797c5dadfa0be7d266cdf49e22da2fcaa32eff75b2c33" +dependencies = [ + "rustc_version", + "simdutf8", +] + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + +[[package]] +name = "skrifa" +version = "0.37.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c31071dedf532758ecf3fed987cdb4bd9509f900e026ab684b4ecb81ea49841" +dependencies = [ + "bytemuck", + "read-fonts 0.35.0", +] + +[[package]] +name = "skrifa" +version = "0.40.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fbdfe3d2475fbd7ddd1f3e5cf8288a30eb3e5f95832829570cd88115a7434ac" +dependencies = [ + "bytemuck", + "read-fonts 0.37.0", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "slotmap" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdd58c3c93c3d278ca835519292445cb4b0d4dc59ccfdf7ceadaab3f8aeb4038" +dependencies = [ + "version_check", +] + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "smithay-client-toolkit" +version = "0.19.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3457dea1f0eb631b4034d61d4d8c32074caa6cd1ab2d59f2327bd8461e2c0016" dependencies = [ - "errno", + "bitflags 2.11.0", + "calloop", + "calloop-wayland-source", + "cursor-icon", "libc", + "log", + "memmap2", + "rustix 0.38.44", + "thiserror 1.0.69", + "wayland-backend", + "wayland-client", + "wayland-csd-frame", + "wayland-cursor", + "wayland-protocols", + "wayland-protocols-wlr", + "wayland-scanner", + "xkeysym", ] [[package]] -name = "slab" -version = "0.4.12" +name = "smol_str" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" +checksum = "dd538fb6910ac1099850255cf94a94df6551fbdd602454387d0adb2d1ca6dead" +dependencies = [ + "serde", +] [[package]] -name = "smallvec" -version = "1.15.1" +name = "spirv" +version = "0.3.0+sdk-1.3.268.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +checksum = "eda41003dc44290527a59b13432d4a0379379fa074b70174882adfbdfd917844" +dependencies = [ + "bitflags 2.11.0", +] [[package]] name = "static_assertions" @@ -1934,6 +3041,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "strict-num" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6637bab7722d379c8b41ba849228d680cc12d0a45ba1fa2b48f2a30577a06731" + [[package]] name = "strsim" version = "0.11.1" @@ -1962,6 +3075,23 @@ dependencies = [ "syn", ] +[[package]] +name = "svg_fmt" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0193cc4331cfd2f3d2011ef287590868599a2f33c3e69bc22c1a3d3acf9e02fb" + +[[package]] +name = "swash" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "842f3cd369c2ba38966204f983eaa5e54a8e84a7d7159ed36ade2b6c335aae64" +dependencies = [ + "skrifa 0.40.0", + "yazi", + "zeno", +] + [[package]] name = "syn" version = "2.0.117" @@ -1973,6 +3103,15 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sys-locale" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eab9a99a024a169fe8a903cf9d4a3b3601109bcc13bd9e3c6fff259138626c4" +dependencies = [ + "libc", +] + [[package]] name = "tap" version = "1.0.1" @@ -1985,6 +3124,15 @@ version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + [[package]] name = "thiserror" version = "1.0.69" @@ -2034,6 +3182,31 @@ dependencies = [ "rustix 0.38.44", ] +[[package]] +name = "tiny-skia" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83d13394d44dae3207b52a326c0c85a8bf87f1541f23b0d143811088497b09ab" +dependencies = [ + "arrayref", + "arrayvec", + "bytemuck", + "cfg-if", + "log", + "tiny-skia-path", +] + +[[package]] +name = "tiny-skia-path" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c9e7fc0c2e86a30b117d0462aa261b72b7a99b7ebd7deb3a14ceda95c5bdc93" +dependencies = [ + "arrayref", + "bytemuck", + "strict-num", +] + [[package]] name = "tinytemplate" version = "1.2.1" @@ -2045,43 +3218,74 @@ dependencies = [ ] [[package]] -name = "toml" -version = "0.9.12+spec-1.1.0" +name = "tinyvec" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" dependencies = [ - "indexmap", - "serde_core", - "serde_spanned", - "toml_datetime", - "toml_parser", - "toml_writer", - "winnow", + "tinyvec_macros", ] +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + [[package]] name = "toml_datetime" -version = "0.7.5+spec-1.1.0" +version = "1.1.1+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" dependencies = [ "serde_core", ] +[[package]] +name = "toml_edit" +version = "0.25.11+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b59c4d22ed448339746c59b905d24568fcbb3ab65a500494f7b8c3e97739f2b" +dependencies = [ + "indexmap", + "toml_datetime", + "toml_parser", + "winnow", +] + [[package]] name = "toml_parser" -version = "1.0.9+spec-1.1.0" +version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "702d4415e08923e7e1ef96cd5727c0dfed80b4d2fa25db9647fe5eb6f7c5a4c4" +checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" dependencies = [ "winnow", ] [[package]] -name = "toml_writer" -version = "1.0.6+spec-1.1.0" +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" + +[[package]] +name = "ttf-parser" +version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab16f14aed21ee8bfd8ec22513f7287cd4a91aa92e44edfe2c17ddd004e92607" +checksum = "d2df906b07856748fa3f6e0ad0cbaa047052d4a7dd609e231c4f72cee8c36f31" +dependencies = [ + "core_maths", +] [[package]] name = "typenum" @@ -2090,10 +3294,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" [[package]] -name = "unarray" -version = "0.1.4" +name = "unicode-bidi" +version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" [[package]] name = "unicode-ident" @@ -2101,11 +3305,23 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "unicode-linebreak" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f" + +[[package]] +name = "unicode-script" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "383ad40bb927465ec0ce7720e033cb4ca06912855fc35db31b5755d0de75b1ee" + [[package]] name = "unicode-segmentation" -version = "1.12.0" +version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" +checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c" [[package]] name = "unicode-truncate" @@ -2185,17 +3401,16 @@ version = "0.1.0" dependencies = [ "displaydoc", "libc", - "log-instrument", "thiserror 2.0.18", ] [[package]] name = "uuid" -version = "1.21.0" +version = "1.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b672338555252d43fd2240c714dc444b8c6fb0a5c5335e65a07bba7742735ddb" +checksum = "5ac8b6f42ead25368cf5b098aeb3dc8a1a2c05a3eee8a9a1a68c640edbfc79d9" dependencies = [ - "getrandom 0.4.1", + "getrandom 0.4.2", "js-sys", "rand", "wasm-bindgen", @@ -2265,7 +3480,6 @@ name = "vmm" version = "0.1.0" dependencies = [ "acpi_tables", - "arrayvec", "aws-lc-rs", "base64", "bincode", @@ -2273,24 +3487,17 @@ dependencies = [ "bitvec", "byteorder", "crc64", - "criterion 0.7.0", "derive_more", - "device_tree", "displaydoc", "event-manager", - "gdbstub", - "gdbstub_arch", - "itertools 0.14.0", "kvm-bindings", "kvm-ioctls", "libc", "linux-loader", "log", - "log-instrument", "memfd", "micro_http", "pci", - "proptest", "semver", "serde", "serde_json", @@ -2322,123 +3529,412 @@ dependencies = [ ] [[package]] -name = "walkdir" -version = "2.5.0" +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.2+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf938a0bacb0469e83c1e148908bd7d5a6010354cf4fb73279b7447422e3a89" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f371d383f2fb139252e0bfac3b81b265689bf45b6874af544ffa4c975ac1ebf8" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eeff24f84126c0ec2db7a449f0c2ec963c6a49efe0698c4242929da037ca28ed" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d08065faf983b2b80a79fd87d8254c409281cf7de75fc4b773019824196c904" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd04d9e306f1907bd13c6361b5c6bfc7b3b3c095ed3f8a9246390f8dbdee129" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags 2.11.0", + "hashbrown 0.15.5", + "indexmap", + "semver", +] + +[[package]] +name = "wayland-backend" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2857dd20b54e916ec7253b3d6b4d5c4d7d4ca2c33c2e11c6c76a99bd8744755d" +dependencies = [ + "cc", + "downcast-rs", + "rustix 1.1.4", + "scoped-tls", + "smallvec", + "wayland-sys", +] + +[[package]] +name = "wayland-client" +version = "0.31.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "645c7c96bb74690c3189b5c9cb4ca1627062bb23693a4fad9d8c3de958260144" +dependencies = [ + "bitflags 2.11.0", + "rustix 1.1.4", + "wayland-backend", + "wayland-scanner", +] + +[[package]] +name = "wayland-csd-frame" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625c5029dbd43d25e6aa9615e88b829a5cad13b2819c4ae129fdbb7c31ab4c7e" +dependencies = [ + "bitflags 2.11.0", + "cursor-icon", + "wayland-backend", +] + +[[package]] +name = "wayland-cursor" +version = "0.31.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a52d18780be9b1314328a3de5f930b73d2200112e3849ca6cb11822793fb34d" +dependencies = [ + "rustix 1.1.4", + "wayland-client", + "xcursor", +] + +[[package]] +name = "wayland-protocols" +version = "0.32.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "563a85523cade2429938e790815fd7319062103b9f4a2dc806e9b53b95982d8f" +dependencies = [ + "bitflags 2.11.0", + "wayland-backend", + "wayland-client", + "wayland-scanner", +] + +[[package]] +name = "wayland-protocols-plasma" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b6d8cf1eb2c1c31ed1f5643c88a6e53538129d4af80030c8cabd1f9fa884d91" +dependencies = [ + "bitflags 2.11.0", + "wayland-backend", + "wayland-client", + "wayland-protocols", + "wayland-scanner", +] + +[[package]] +name = "wayland-protocols-wlr" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb04e52f7836d7c7976c78ca0250d61e33873c34156a2a1fc9474828ec268234" +dependencies = [ + "bitflags 2.11.0", + "wayland-backend", + "wayland-client", + "wayland-protocols", + "wayland-scanner", +] + +[[package]] +name = "wayland-scanner" +version = "0.31.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +checksum = "9c324a910fd86ebdc364a3e61ec1f11737d3b1d6c273c0239ee8ff4bc0d24b4a" dependencies = [ - "same-file", - "winapi-util", + "proc-macro2", + "quick-xml", + "quote", ] [[package]] -name = "wasi" -version = "0.11.1+wasi-snapshot-preview1" +name = "wayland-sys" +version = "0.31.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" +checksum = "d8eab23fefc9e41f8e841df4a9c707e8a8c4ed26e944ef69297184de2785e3be" +dependencies = [ + "dlib", + "log", + "once_cell", + "pkg-config", +] [[package]] -name = "wasip2" -version = "1.0.2+wasi-0.2.9" +name = "web-sys" +version = "0.3.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" +checksum = "4f2dfbb17949fa2088e5d39408c48368947b86f7834484e87b73de55bc14d97d" dependencies = [ - "wit-bindgen", + "js-sys", + "wasm-bindgen", ] [[package]] -name = "wasip3" -version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +name = "web-time" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" dependencies = [ - "wit-bindgen", + "js-sys", + "wasm-bindgen", ] [[package]] -name = "wasm-bindgen" -version = "0.2.113" +name = "wgpu" +version = "28.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60722a937f594b7fde9adb894d7c092fc1bb6612897c46368d18e7a20208eff2" +checksum = "f9cb534d5ffd109c7d1135f34cdae29e60eab94855a625dcfe1705f8bc7ad79f" dependencies = [ + "arrayvec", + "bitflags 2.11.0", + "bytemuck", "cfg-if", - "once_cell", - "rustversion", - "wasm-bindgen-macro", - "wasm-bindgen-shared", + "cfg_aliases", + "document-features", + "hashbrown 0.16.1", + "js-sys", + "log", + "naga", + "parking_lot", + "portable-atomic", + "profiling", + "raw-window-handle", + "smallvec", + "static_assertions", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "wgpu-core", + "wgpu-hal", + "wgpu-types", ] [[package]] -name = "wasm-bindgen-macro" -version = "0.2.113" +name = "wgpu-core" +version = "28.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fac8c6395094b6b91c4af293f4c79371c163f9a6f56184d2c9a85f5a95f3950" +checksum = "d23f4642f53f666adcfd2d3218ab174d1e6681101aef18696b90cbe64d1c10f9" dependencies = [ - "quote", - "wasm-bindgen-macro-support", + "arrayvec", + "bit-set", + "bit-vec", + "bitflags 2.11.0", + "bytemuck", + "cfg_aliases", + "document-features", + "hashbrown 0.16.1", + "indexmap", + "log", + "naga", + "once_cell", + "parking_lot", + "portable-atomic", + "profiling", + "raw-window-handle", + "rustc-hash 1.1.0", + "smallvec", + "thiserror 2.0.18", + "wgpu-core-deps-apple", + "wgpu-core-deps-emscripten", + "wgpu-core-deps-windows-linux-android", + "wgpu-hal", + "wgpu-types", ] [[package]] -name = "wasm-bindgen-macro-support" -version = "0.2.113" +name = "wgpu-core-deps-apple" +version = "28.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab3fabce6159dc20728033842636887e4877688ae94382766e00b180abac9d60" +checksum = "87b7b696b918f337c486bf93142454080a32a37832ba8a31e4f48221890047da" dependencies = [ - "bumpalo", - "proc-macro2", - "quote", - "syn", - "wasm-bindgen-shared", + "wgpu-hal", ] [[package]] -name = "wasm-bindgen-shared" -version = "0.2.113" +name = "wgpu-core-deps-emscripten" +version = "28.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de0e091bdb824da87dc01d967388880d017a0a9bc4f3bdc0d86ee9f9336e3bb5" +checksum = "34b251c331f84feac147de3c4aa3aa45112622a95dd7ee1b74384fa0458dbd79" dependencies = [ - "unicode-ident", + "wgpu-hal", ] [[package]] -name = "wasm-encoder" -version = "0.244.0" +name = "wgpu-core-deps-windows-linux-android" +version = "28.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +checksum = "68ca976e72b2c9964eb243e281f6ce7f14a514e409920920dcda12ae40febaae" dependencies = [ - "leb128fmt", - "wasmparser", + "wgpu-hal", ] [[package]] -name = "wasm-metadata" -version = "0.244.0" +name = "wgpu-hal" +version = "28.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +checksum = "44d6cb474beb218824dcc9e1ce679d973f719262789bfb27407da560cac20eeb" dependencies = [ - "anyhow", - "indexmap", - "wasm-encoder", - "wasmparser", + "android_system_properties", + "arrayvec", + "ash", + "bit-set", + "bitflags 2.11.0", + "block", + "bytemuck", + "cfg-if", + "cfg_aliases", + "core-graphics-types 0.2.0", + "glow", + "glutin_wgl_sys", + "gpu-allocator", + "gpu-descriptor", + "hashbrown 0.16.1", + "js-sys", + "khronos-egl", + "libc", + "libloading", + "log", + "metal", + "naga", + "ndk-sys", + "objc", + "once_cell", + "ordered-float", + "parking_lot", + "portable-atomic", + "portable-atomic-util", + "profiling", + "range-alloc", + "raw-window-handle", + "renderdoc-sys", + "smallvec", + "thiserror 2.0.18", + "wasm-bindgen", + "web-sys", + "wgpu-types", + "windows", + "windows-core", ] [[package]] -name = "wasmparser" -version = "0.244.0" +name = "wgpu-types" +version = "28.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +checksum = "e18308757e594ed2cd27dddbb16a139c42a683819d32a2e0b1b0167552f5840c" dependencies = [ "bitflags 2.11.0", - "hashbrown 0.15.5", - "indexmap", - "semver", + "bytemuck", + "js-sys", + "log", + "web-sys", ] [[package]] name = "wide" -version = "1.1.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac11b009ebeae802ed758530b6496784ebfee7a87b9abfbcaf3bbe25b814eb25" +checksum = "198f6abc41fab83526d10880fa5c17e2b4ee44e763949b4bb34e2fd1e8ca48e4" dependencies = [ "bytemuck", "safe_arch", @@ -2475,12 +3971,116 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580" +dependencies = [ + "windows-collections", + "windows-core", + "windows-future", + "windows-numerics", +] + +[[package]] +name = "windows-collections" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610" +dependencies = [ + "windows-core", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-future" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb" +dependencies = [ + "windows-core", + "windows-link", + "windows-threading", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-numerics" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26" +dependencies = [ + "windows-core", + "windows-link", +] + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-sys" version = "0.59.0" @@ -2515,6 +4115,15 @@ dependencies = [ "windows_x86_64_msvc", ] +[[package]] +name = "windows-threading" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37" +dependencies = [ + "windows-link", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" @@ -2563,11 +4172,66 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "winit" +version = "0.30.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6755fa58a9f8350bd1e472d4c3fcc25f824ec358933bba33306d0b63df5978d" +dependencies = [ + "ahash", + "android-activity", + "atomic-waker", + "bitflags 2.11.0", + "block2", + "bytemuck", + "calloop", + "cfg_aliases", + "concurrent-queue", + "core-foundation 0.9.4", + "core-graphics", + "cursor-icon", + "dpi", + "js-sys", + "libc", + "memmap2", + "ndk", + "objc2", + "objc2-app-kit", + "objc2-foundation", + "objc2-ui-kit", + "orbclient", + "percent-encoding", + "pin-project", + "raw-window-handle", + "redox_syscall 0.4.1", + "rustix 0.38.44", + "sctk-adwaita", + "smithay-client-toolkit", + "smol_str", + "tracing", + "unicode-segmentation", + "wasm-bindgen", + "wasm-bindgen-futures", + "wayland-backend", + "wayland-client", + "wayland-protocols", + "wayland-protocols-plasma", + "web-sys", + "web-time", + "windows-sys 0.52.0", + "x11-dl", + "x11rb", + "xkbcommon-dl", +] + [[package]] name = "winnow" -version = "0.7.14" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a5364e9d77fcdeeaa6062ced926ee3381faa2ee02d3eb83a5c27a8825540829" +checksum = "2ee1708bef14716a11bae175f579062d4554d95be2c6829f518df847b7b3fdd0" +dependencies = [ + "memchr", +] [[package]] name = "wit-bindgen" @@ -2666,20 +4330,95 @@ dependencies = [ "tap", ] +[[package]] +name = "x11-dl" +version = "2.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38735924fedd5314a6e548792904ed8c6de6636285cb9fec04d5b1db85c1516f" +dependencies = [ + "libc", + "once_cell", + "pkg-config", +] + +[[package]] +name = "x11rb" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9993aa5be5a26815fe2c3eacfc1fde061fc1a1f094bf1ad2a18bf9c495dd7414" +dependencies = [ + "as-raw-xcb-connection", + "gethostname", + "libc", + "libloading", + "once_cell", + "rustix 1.1.4", + "x11rb-protocol", +] + +[[package]] +name = "x11rb-protocol" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea6fc2961e4ef194dcbfe56bb845534d0dc8098940c7e5c012a258bfec6701bd" + +[[package]] +name = "xcursor" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bec9e4a500ca8864c5b47b8b482a73d62e4237670e5b5f1d6b9e3cae50f28f2b" + +[[package]] +name = "xkbcommon-dl" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d039de8032a9a8856a6be89cea3e5d12fdd82306ab7c94d74e6deab2460651c5" +dependencies = [ + "bitflags 2.11.0", + "dlib", + "log", + "once_cell", + "xkeysym", +] + +[[package]] +name = "xkeysym" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9cc00251562a284751c9973bace760d86c0276c471b4be569fe6b068ee97a56" + +[[package]] +name = "xml-rs" +version = "0.8.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ae8337f8a065cfc972643663ea4279e04e7256de865aa66fe25cec5fb912d3f" + +[[package]] +name = "yazi" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e01738255b5a16e78bbb83e7fbba0a1e7dd506905cfc53f4622d89015a03fbb5" + +[[package]] +name = "zeno" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6df3dc4292935e51816d896edcd52aa30bc297907c26167fec31e2b0c6a32524" + [[package]] name = "zerocopy" -version = "0.8.40" +version = "0.8.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a789c6e490b576db9f7e6b6d661bcc9799f7c0ac8352f56ea20193b2681532e5" +checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.40" +version = "0.8.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f65c489a7071a749c849713807783f70672b28094011623e200cb86dcb835953" +checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" dependencies = [ "proc-macro2", "quote", @@ -2700,17 +4439,8 @@ checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" [[package]] name = "zpaq_rs" -version = "1.0.4" +version = "1.0.5" dependencies = [ "cc", "hex", ] - -[[package]] -name = "zpaq_rs" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f6e6e20f814ac75e2f98d6916ca65cece82ffe9f61ed1b222ea81dacd86c27d" -dependencies = [ - "cc", -] diff --git a/Cargo.toml b/Cargo.toml index 9707a1fa..aca832a7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,15 +1,16 @@ -[package] -name = "infotheory" -version = "1.1.1" -edition = "2024" -license = "ISC OR Apache-2.0" -homepage = "https://infotheory.tech" -description = "The algorithmic information theory library." -autobins = false - [workspace] -members = [".", "zpaq_rs", "infotheory_py", "benchman"] -exclude = ["nyx-lite"] +members = [ + "crates/infotheory", + "crates/infotheory_py", + "crates/benchman", + "vendor/zpaq_rs", +] +default-members = [ + "crates/infotheory", + "crates/benchman", + "vendor/zpaq_rs", +] +exclude = ["vendor/nyx-lite"] resolver = "3" [workspace.lints.rust] @@ -18,82 +19,6 @@ warnings = "warn" [workspace.lints.clippy] all = "warn" -[dependencies] -zpaq_rs = { version = "1.0.4", optional = true } -nyx-lite = { path = "./nyx-lite", optional = true } -rayon = "1.11.0" -num_cpus = "1.17.0" -once_cell = "1.21.3" -serde_json = "1.0.149" -anyhow = "1.0.100" -ahash = { version = "0.8.12", default-features = false, features = ["std", "no-rng"] } -crc32fast = "1.5.0" -wide = "1.1.1" - -[features] -default = ["backend-rosa", "backend-mamba", "backend-rwkv", "backend-zpaq"] -backend-rosa = [] -backend-mamba = [] -backend-rwkv = [] -backend-zpaq = ["dep:zpaq_rs"] -cli = [] -vm = ["dep:nyx-lite", "backend-rosa", "backend-mamba", "backend-rwkv", "backend-zpaq"] - -[[bin]] -name = "infotheory" -path = "src/main.rs" -required-features = ["cli"] - - -[[bench]] -name = "par" -harness = false - -[[bench]] -name = "aixi" -harness = false - -[[bench]] -name = "aiqi" -harness = false - -[[bench]] -name = "mixture_backends" -harness = false - -[[bench]] -name = "neural_baseline" -harness = false - -[[bench]] -name = "simd_hotspots" -harness = false - -[[bench]] -name = "rate_backend_coders" -harness = false -required-features = ["backend-rwkv"] - -[[bench]] -name = "mamba_rate" -harness = false -required-features = ["backend-mamba"] - -[[bench]] -name = "mamba_online_train_full" -harness = false -required-features = ["backend-mamba"] - -[[bench]] -name = "rwkv_online_train_full" -harness = false -required-features = ["backend-rwkv"] - -[[bench]] -name = "exact_hotpaths" -harness = false -required-features = ["backend-rwkv", "backend-mamba"] - [profile.dev] opt-level = 3 lto = "fat" @@ -115,8 +40,3 @@ lto = "fat" codegen-units = 1 opt-level = 3 debug = true - -[dev-dependencies] -serde = { version = "1.0", features = ["derive"] } -sha2 = "0.10" -criterion = { version = "0.5", default-features = false, features = ["cargo_bench_support"] } diff --git a/README.md b/README.md index 50998242..c2cc3613 100644 --- a/README.md +++ b/README.md @@ -1,439 +1,179 @@ # InfoTheory -### 1. Unified Information Estimation -Estimate core measures using both **Marginal** (distribution-based) and **Rate** (predictive-based) approaches: -- **NCD (Normalized Compression Distance)**: Approximates information distance using compression. -- **MI (Mutual Information)**: Quantifies shared information between sequences. -- **NED (Normalized Entropy Distance)**: A metric distance based on mutual information. -- **NTE (Normalized Transform Effort)**: Variation of Information (VI). -- **Intrinsic Dependence**: Redundancy Ratio. -- **Resistance**: Information preservation under noise/transform. - -### 2. Multi-Backend Predictive Engine -The core model class in the library is `RateBackend`. A `RateBackend` is the predictive model object used by entropy-rate estimators, rate-coded compression, generation, and the agent world-model interface. - -Switch between different `RateBackend` families seamlessly: -- **ROSA+ (Rapid Online Suffix Automaton + Witten Bell)**: A fast statistical LM. Default backend. -- **CTW (Context Tree Weighting)**: Historically standard for AIXI. Accurate bit-level Bayesian model (KT-estimator). -- **Sequitur**: Exact online grammar induction with Sequitur normalization plus predictive suffix-context readout. -- **Mamba (Neural Network)**: Deterministic CPU-first Mamba-1 backend with online mode + export. -- **RWKV (Neural Network)**: Portable SIMD RWKV7 CPU inference backend (`wide`-based). - -The same `RateBackend` model class also supports ensemble world models. `RateBackend::Mixture` combines `RateBackend` experts into a single predictive model: `Bayes`, `Switching`, and `Convex` follow *On Ensemble Techniques for AIXI Approximation*, while `FadingBayes`, `Mdl`, and `Neural` are extensions implemented in this repository. - -### 3. Integrated MC-AIXI Agent -Includes a full implementation of the **Monte Carlo AIXI (MC-AIXI)** agent described by Hutter et al. It approximates incomputable AIXI with Monte-Carlo Tree Search and can use the library's `RateBackend` model class, including mixture-based ensemble world models, as its world model. - -You can use a trained neural model (Mamba-1 or RWKV7) as a rate backend ("world model") for MC-AIXI. - -- `planner: "mc-aixi"` selects the classic MCTS-based MC-AIXI planner. -- MC-AIXI can also take a full `rate_backend` object instead of relying only on `algorithm`, including nested mixture backends built from other `RateBackend` experts. -- **Mixture families from *On Ensemble Techniques for AIXI Approximation***: `Bayes` and `Convex` are exposed directly, and `Switching` follows the fixed-share update from *On Ensemble Techniques for AIXI Approximation* with a constant switch-rate `alpha`. -- **Extensions**: `FadingBayes`, `Mdl`, and `Neural` remain available. -- **Why recursive `zpaq` is rejected in generic MC-AIXI configs**: `zpaq` cannot roll predictor state backward after hypothetical actions, so it does not satisfy the reversible action-conditioning requirement used by *A Monte-Carlo AIXI Approximation*. The older standalone `algorithm: "zpaq"` mode still exists, but it does not provide that exact rollback behavior. -- **UCB tie-breaking from *A Monte-Carlo AIXI Approximation***: MC-AIXI chooses uniformly at random among unvisited actions and among exactly tied maximal UCB actions. - -### 4. Integrated AIQI Agent -The repository also includes **AIQI**, the model-free return-prediction agent introduced in *A Model-Free Universal AI* by Yegon Kim and Juho Lee, with periodic augmentation (`N >= H`) and discretized H-step return targets. - -- `planner: "aiqi"` enables AIQI in `infotheory aixi `. -- `planner: "mc-aixi"` (default) keeps MC-AIXI as the default planner. -- **Direct AIQI-CTW configuration from *A Model-Free Universal AI***: `algorithm: "ac-ctw"` (or `"ctw"`) selects the AIQI-CTW setup described in *A Model-Free Universal AI*. -- **Extensions**: AIQI also supports `fac-ctw`, `rosa`, `rwkv`, and generic `rate_backend` predictors, including the same mixture JSON format used elsewhere in the repo. -- **Why `zpaq` is excluded from AIQI**: AIQI needs exact frozen predictor states while it scores hypothetical actions and return bins, and `zpaq` does not provide that interface. -- **Validation from *A Model-Free Universal AI***: AIQI enforces `discount_gamma in (0,1)` and `baseline_exploration (tau) in (0,1]`. -- **Tie-breaking from *A Model-Free Universal AI***: greedy action selection uses a fixed tie-break rule (first maximizing action) to match the fixed tie-breaking assumption in *A Model-Free Universal AI*. -- **Optional bounded memory**: set `history_prune_keep_steps` (or `aiqi_history_prune_keep_steps`) to retain only recent history while still keeping the steps needed for exact H-step return construction. -- **Reproducibility**: set `random_seed` in config (or planner-specific `aiqi_random_seed` / `mcaixi_random_seed`) to make agent-side randomness deterministic across runs. -- AIQI uses the same environment interfaces as MC-AIXI, including VM environments. - ---- - -## Compilation & Installation -### Platform Support (tested) -`infotheory` is currently tested on: -- **Linux (GNU libc)** (`x86_64-unknown-linux-gnu`) -- **Linux (musl)** (`x86_64-unknown-linux-musl`) -- **macOS (Intel)** (`x86_64-apple-darwin`) -- **macOS (Apple Silicon)** (`aarch64-apple-darwin`) -- **Windows** (`x86_64-pc-windows-msvc`) -- **FreeBSD** (`x86_64-unknown-freebsd`) -- **OpenBSD** (`x86_64-unknown-openbsd`) -- **NetBSD** (`x86_64-unknown-netbsd`) -- **AArch64 Linux (GNU/musl)** (`aarch64-unknown-linux-gnu`, `aarch64-unknown-linux-musl`) -- **AArch64 Windows** (`aarch64-pc-windows-msvc`) -- **WASM** (`wasm32-unknown-unknown`) - -ZPAQ feature is not supported on WASM targets - -### Build Prerequisites -- Rust toolchain (stable): `rustup` recommended. -- C/C++ toolchain: `clang` + `lld` recommended on Unix-like systems. -- For local repository builds with VM support available: clone recursively (`--recurse-submodules`) so `nyx-lite` is present. - -### Build Configuration -- By default, .cargo/config.toml is set to use march=native as the target-cpu, which will allow LLVM to make full use of your specific CPU. This can improve performance by roughly 2x for the RWKV Model. This may affect binary compatibility depending on your usecase. - -### Build the CLI -Enable the `cli` feature (the binary is feature-gated): +InfoTheory is a Rust library, CLI, and Python extension for algorithmic +information theory and related information-theoretic functions. It's focused on +predictive modelling, compression and its derived complexity estimates, AIXI approximate (and other universal-ish) agents, and tooling. -```bash -cargo build --release --features cli --bin infotheory -``` - -Output binary: -- `./target/release/infotheory` (host target) -- `./target//release/infotheory` (cross target) - -### Build as a library -Add the dependency in your `Cargo.toml`: - -```toml -[dependencies] -infotheory = { path = "." } # Replace with a git or crates.io source as needed. -``` - -### Building nyx-lite -The VM backend is optional (`--features vm`) and depends on `nyx-lite` (and its vendored submodule code). Build it with: -```bash -cargo build --release --features vm -``` -Notes: -- VM is Linux/KVM-oriented (`/dev/kvm` required). -- Some `nyx-lite` tests also require VM image artifacts under `nyx-lite/vm_image`. +**We provide a prediction library, including applications of predictors, and tooling to evaluate and select better predictors for specific tasks.** -### Additional notes -Platform caveats: -- **OpenBSD/NetBSD**: kernel W^X policies can break ZPAQ JIT at runtime. Set `CARGO_FEATURE_NOJIT=true`. -- **NetBSD**: release LTO is problematic in common toolchains; disable release LTO if needed (see `.cargo/config.toml` comments). -- **MacOS**: Supported on both Intel and Apple Silicon natively. +## What Is Included -Optional tooling used by some tests/workflows: -- docker (for tests, or if you want to use it for rootfs generation) -- cpio -- wget (for tests, or to use the provided kernel. you can also use curl instead manually on the download_kernel.sh file ) -- cmake (for VM feature, firecracker needs it) -- Lean4 (Toolchain Version 4.14.0) ---- +- Information metrics: NCD, Normalized Entropy Distance, Normalized Transform Effort, Mutual Information, entropy/cross-entropy/conditional + entropy, intrinsic dependence, resistance to transformation, KL, JS, TVD, and + Hellinger distance. +- Predictive backends: ROSA+, CTW/FAC-CTW, PPMD, Sequitur, match/sparse-match, + ZPAQ-as-rate, Mamba, RWKV7, mixtures, particle filters, and calibrated + wrappers. Availability is feature-gated. +- Compression surfaces: native ZPAQ, generic AC/rANS rate-coded compression over any rate backend. +- Agents: canonical `planner_run` execution for MC-AIXI, AIQI, and VM-backed + environments through Nyx-Lite. +- Tuner: canonical `tune` specs for bounded task-specific backend search. +- Python bindings: `infotheory-rs` on PyPI, imported as `infotheory_rs`. -## CLI Usage +## Repository Layout -The `infotheory` binary provides a powerful interface for file analysis. +- `crates/infotheory`: primary Rust library and optional CLI binary. +- `crates/infotheory_py`: PyO3/maturin Python extension. +- `crates/benchman`: TUI benchmark-summary inspector. +- `configs`: canonical checked-in planner and benchmark configs. +- `examples`: runnable examples, including tuner validation specs. +- `docs`: developer, performance, tuner, warm-start, and canonical-pipeline docs. +- `scripts` and `projman.sh`: local workflow, benchmark, CI-preflight, and + conversion tooling. +- `vendor/zpaq_rs`, `vendor/nyx-lite`, `vendor/gameengine`: optional/path + dependencies used by selected features. -### Primitives -```bash -# Calculate Mutual Information (ROSA backend, order 8) -./infotheory mi file1.txt file2.txt 8 - -# Use CTW backend for NTE (Normalized Transform Effort) -./infotheory nte file1.txt file2.txt --rate-backend ctw - -# Calculate NCD with custom ZPAQ method -./infotheory ncd file1.txt file2.txt 5 -``` - -### Compression Backends +## Build -`CompressionBackend` is the canonical compression enum in the library. +Rust stable is required. Some workflows also need `uv`, `maturin`, Lean, KVM, or +platform tools; see [docs/developer-testing.md](docs/developer-testing.md). -CLI: +Build the CLI: ```bash -# ZPAQ standalone (as before) -./infotheory ncd a.bin b.bin --compression-backend zpaq --method 5 - -# Turn any rate backend into a compressor via AC/rANS -./infotheory ncd a.bin b.bin --compression-backend rate-ac --rate-backend ctw --method 16 -./infotheory ncd a.bin b.bin --compression-backend rate-rans --rate-backend fac-ctw --method 16 +cargo build -p infotheory --release --features cli --bin infotheory --locked ``` -For rate-coded metrics, raw framing is used by default to avoid framing overhead. -Explicit `compress_bytes_backend` / `decompress_bytes_backend` APIs support framed payloads for roundtrip verification. - -### AC Log-Loss Diagnostics - -`ac-log-loss` runs the exact arithmetic-coding predictor path for a top-level mixture spec and streams per-position diagnostics to TSV without keeping the full trace in memory. +Run the default Rust test slice: ```bash -RAYON_NUM_THREADS=4 ./infotheory ac-log-loss corpus.bin \ - --mixture examples/mixture_spec.json \ - --out-prefix /tmp/mixture-diagnostic +cargo test -p infotheory --locked ``` -It writes: - -- `/tmp/mixture-diagnostic.trace.tsv`: per-position mixture probability/bits, oracle fields, root weight statistics, and per-node `prob` / `bits` / `local_weight` / `effective_weight` -- `/tmp/mixture-diagnostic.nodes.tsv`: flattened mixture-tree metadata with stable node ids -- `/tmp/mixture-diagnostic.summary.tsv`: total bits, oracle regret, switch counts, AC payload bits, and per-node aggregates - -### Neural Method Strings - -Mamba and RWKV can be configured with either a model file or compact method string: - -- `file:/abs/or/relative/model.safetensors` -- `file:/abs/or/relative/model.safetensors;policy:...` -- `cfg:key=value,...[;policy:...]` - -Supported `cfg:` keys: -- RWKV7: `hidden,layers,intermediate,decay_rank,a_rank,v_rank,g_rank,seed,train,lr,stride` -- Mamba-1: `hidden,layers,intermediate,state,conv,dt_rank,seed,train,lr,stride` - -`train` supports: `none`, `sgd`, `adam`. -`policy` supports `schedule=...` rules (for example `0..100:infer` or `0..100:train(scope=head+bias,opt=adam,lr=0.001,stride=1,bptt=1,clip=0,momentum=0.9)`). -For RWKV full-parameter training scopes (`scope` touching non-head parameters), `bptt<=1` resolves to the fast default window `8`; specify a larger explicit `bptt` to override it. - -Example: +Run the CLI plus broad backend parity slice: ```bash -./infotheory h file.txt \ - --rate-backend rwkv7 \ - --method "cfg:hidden=64,layers=1,intermediate=64,decay_rank=8,a_rank=8,v_rank=8,g_rank=8,seed=7,train=sgd,lr=0.01,stride=1;policy:schedule=0..100:train(scope=head+bias,opt=sgd,lr=0.01,stride=1,bptt=1,clip=0,momentum=0.9)" +cargo test -p infotheory --no-default-features --features "cli all-backends" --locked ``` -For `examples/two.json` benchmark plotting, `scripts/plot_two_json.sh` also accepts `INFOTHEORY_BASELINE_SUMMARY_TSV=/path/to/baseline-summary.tsv` to emit additional baseline-overlay SVGs. - -The benchmark tooling also supports an `extra` suite for additional rate backends -not in `examples/two.json` (currently `mamba`, `particle` via -`examples/particle_fast.json`, and `sparse-match`): - -```bash -./projman.sh bench extra -./projman.sh plot extra -./projman.sh tui extra -``` +Local workflows respect `INFOTHEORY_BUILD_MODE`: -For interactive benchmark analysis (all `plot_two_json.sh` graph families, subject focus, exact point inspection, overlap-aware readouts), use: +- `native`: local development/benchmark default, using repository-default native CPU + tuning. +- `portable`: generic CPU flags to match portable CI/release behavior. -```bash -./projman.sh tui --summary-tsv /tmp/infotheory-two-json-summary-.tsv -``` +## CLI Quick Start -Manual: +Build the binary first, or use `cargo run -p infotheory --features cli -- ...`. ```bash -./projman.sh tui man +infotheory h README.md +infotheory h_rate README.md --rate-backend ctw --method 32 +infotheory ncd a.bin b.bin --compression-backend zpaq --method 5 +infotheory ncd a.bin b.bin --compression-backend rate-ac --rate-backend ctw --method 16 +infotheory compress in.bin out.itc --compression-backend rate-rans --rate-backend fac-ctw --method 32 +infotheory decompress out.itc restored.bin --compression-backend rate-rans --rate-backend fac-ctw --method 32 +cat prompt.txt | infotheory generate --rate-backend ctw --method 32 --bytes 8 +infotheory aixi configs/aixi/paper_kuhn_poker.json ``` -Optional online export after processing input: +The CLI has topic help: ```bash -./infotheory h file.txt --rate-backend mamba --method "cfg:hidden=128,layers=2,intermediate=256,state=16,conv=4;policy:schedule=0..100:infer" --model-export ./mamba_online.safetensors +infotheory --help +infotheory help backends +infotheory help tune +infotheory ncd --help +infotheory warmstart --help ``` -This writes: -- `rwkv_online.safetensors` -- `rwkv_online.json` (sidecar with resolved config + metadata) +Important CLI surfaces: -### AIXI Agent Mode -```bash -# Run the AIXI agent using config-specified backend -./infotheory aixi conf/kuhn_poker.json -``` +- `batch`: line-oriented JSON request/response mode. +- `aixi`: executes canonical `planner_run` documents. +- `warmstart`: exports, converts, and merges teacher datasets. +- `tune`: executes canonical `tune` documents. +- `ac-log-loss`: emits exact mixture AC/log-loss diagnostics as TSV. -Planner switch in config: - -```json -{ - "planner": "aiqi", - "algorithm": "ac-ctw", - "random_seed": 12345, - "discount_gamma": 0.99, - "return_horizon": 6, - "return_bins": 32, - "augmentation_period": 6, - "history_prune_keep_steps": 2048, - "baseline_exploration": 0.01 -} -``` - -Both planners also accept a `rate_backend` object using the same `RateBackend` schema and mixture language as the rest of the library. This is how the library's model class becomes the planner world model. Recursive `zpaq` is rejected here because these planner integrations need exact reversible or frozen conditioning during planning: - -```json -{ - "planner": "aiqi", - "rate_backend": { - "name": "ppmd", - "order": 10, - "memory_mb": 64 - }, - "rate_backend_max_order": 8 -} -``` - -Example MC-AIXI convex mixture override: - -```json -{ - "planner": "mc-aixi", - "algorithm": "fac-ctw", - "rate_backend": { - "name": "mixture", - "spec": { - "kind": "convex", - "alpha": 1.25, - "experts": [ - {"name": "ctw", "kind": "ctw", "depth": 8}, - {"name": "ppmd", "kind": "ppmd", "order": 8, "memory_mb": 16} - ] - } - }, - "rate_backend_max_order": 8 -} -``` - -### AIXI Agent Mode (VM via Nyx-Lite) -```bash -# VM-backed environment using high-performance Firecracker (Nyx-Lite) -./infotheory aixi aixi_confs/vm_example.json -``` - -Quick benchmark (AIQI vs MC-AIXI): - -```bash -./scripts/bench_aiqi_vs_aixi.sh -``` - -Reproducible competitor benchmark (Infotheory Rust/Python vs PyAIXI + C++ MC-AIXI): - -```bash -./projman.sh bench__aixi_competitors --profile default --trials 1 -``` - -Benchmark correctness notes: -- Stochastic environments are seeded from `random_seed` (or `rng_seed`) in CLI and Python run loops for reproducible trajectories. -- Reward reporting is normalized to native domain scale in competitor reports (for example Kuhn offset removal for C++/PyAIXI), so cross-implementation reward means are apples-to-apples. -- MC-AIXI tree search uses the same UCB scaling convention as common MC-AIXI reference implementations, the uniform-max tie-breaking rule from *A Monte-Carlo AIXI Approximation*, and chance-node cache keys that include reward as well as observation so environments with repeated observations but different rewards are handled correctly. - -VM config highlights: -- **Environment**: Use `"environment": "nyx-vm"` or `"vm"` (requires `vm` feature). -- **Core Config**: - - `vm_config.kernel_image_path`: Path to `vmlinux` kernel. - - `vm_config.rootfs_image_path`: Path to `rootfs.ext4`. - - `vm_config.instance_id`: Unique ID for the VM instance. -- **Performance**: - - `vm_config.shared_memory_policy`: Use `"snapshot"` for fast resets (fork-server style). - - `vm_config.observation_policy`: `"shared_memory"` for zero-copy observations. -- **Rewards & Observations**: - - `vm_reward.mode`: `"guest"` (guest writes to specific address), `"pattern"`, or `"trace-entropy"`. - - `vm_observation.mode`: `"raw"` (bytes) or hash-based. - - `observation_stream_len`: **Critical** for planning consistency. Must match guest output. - -**Prerequisites**: -- Linux with KVM enabled (`/dev/kvm` accessible). -- `vmlinux` kernel and `rootfs.ext4` image valid for Firecracker. -- `nyx-lite` crate (included in workspace). - -**Setup**: -1. Ensure you have the `vmlinux-6.1.58` kernel in the project root (or update config). -2. Ensure `nyx-lite/vm_image/dockerimage/rootfs.ext4` exists or provide your own. -3. Enable the feature: `cargo build --release --features vm`. - ---- - -## Library Usage +## Rust API Quick Start ```rust -use infotheory::*; - -// Entropy rate of a sequence (uses ROSA by default) -let h = entropy_rate_bytes(data, 8); - -// Switch the entire thread to use CTW for all subsequent calls -set_default_ctx(InfotheoryCtx::new( - RateBackend::Ctw { depth: 32 }, - CompressionBackend::default() -)); +use infotheory::api::{CompressionBackend, InfotheoryCtx, RateBackend}; + +fn main() -> Result<(), Box> { + let ctx = InfotheoryCtx::from_specs( + RateBackend::Ctw { depth: 16 }, + CompressionBackend::Rate { + rate_backend: RateBackend::Ctw { depth: 16 }, + coder: infotheory::coders::CoderType::AC, + framing: infotheory::compression::FramingMode::Raw, + }, + )?; + + let bits = ctx.try_entropy_rate_bytes(b"abracadabra")?; + assert!(bits.is_finite()); + Ok(()) +} ``` ---- - -## Supported Primitives - -| Command | Description | Domain | -| :--- | :--- | :--- | -| `ncd` | Normalized Compression Distance | Compression | -| `ned` | Normalized Entropy Distance | Shannon | -| `nte` | Variation of Information | Shannon | -| `mi` | Mutual Information | Shannon | -| `id` | Internal Redundancy | Algorithmic | -| `rt` | Resistance to Transform | Algorithmic | -and more! ---- - -## Python Bindings (`infotheory-rs`) +`InfotheoryCtx::from_specs()` validates, canonicalizes, and compiles wrapper specs +before execution. Advanced callers can explicitly use `validate()` and +`compile()` on `RateBackend` and `CompressionBackend`, then construct +`InfotheoryCtx::new(compiled_rate, compiled_compression)`. -This repository now includes PyO3/maturin bindings with package name: -- PyPI distribution: `infotheory-rs` -- Python import: `infotheory_rs` +## Python Quick Start -Quickstart (local, via `uv`): +Local editable build: ```bash -uv run maturin develop --release -uv run python -c "import infotheory_rs as ait; print(ait.ncd_paths('README.md','README.md', backend='zpaq', method='5', variant='vitanyi'))" +uv run maturin develop +uv run pytest -q python/tests ``` -Python exposes both string-based backend parsing and direct backend objects. The -Python API includes `RateBackend.match(...)`, `RateBackend.sparse_match(...)`, -`RateBackend.ppmd(...)`, `RateBackend.mixture(...)`, `RateBackend.particle(...)`, -and `RateBackend.calibrated(...)`, plus `CalibrationContextKind` for calibrated -backends. - Example: ```python import infotheory_rs as ait -match_backend = ait.RateBackend.match() -particle_backend = ait.RateBackend.particle( - ait.ParticleSpec(num_particles=4, num_cells=4, cell_dim=8) -) -cal_backend = ait.RateBackend.calibrated( - ait.RateBackend.ctw(8), - ait.CalibrationContextKind.Text, -) - -assert ait.entropy_rate_backend(b"abracadabra", 4, backend=match_backend) >= 0.0 -framed = ait.CompressionBackend.rate_rans(particle_backend, "framed") +backend = ait.RateBackend.ctw(16) +assert ait.entropy_rate_backend(b"abracadabra", backend=backend) >= 0.0 + +framed = ait.CompressionBackend.rate_ac(backend, "framed") blob = ait.compress_bytes_backend(b"payload", compression_backend=framed) assert ait.decompress_bytes_backend(blob, compression_backend=framed) == b"payload" -assert ait.compress_size_backend( - b"payload", - compression_backend="rwkv7", - method="cfg:hidden=64,layers=1,intermediate=64,decay_rank=8,a_rank=8,v_rank=8,g_rank=8,seed=11,train=none,lr=0.0,stride=1;policy:schedule=0..100:infer", -) > 0 ``` -Run Python tests: +The Python package is built as `abi3-py310`. Published wheels are portable and exclude VM support by default; Linux source builds can opt into VM +bindings with the Rust `vm` feature, through git. The published versions on crates.io have a shimmed(useless) VM feature. -```bash -uv run pytest -q python/tests -``` +## Documentation Map -Run Python wrapper coverage (enforced in CI): +- [Canonical pipeline invariants](docs/developer-canonical-pipeline-invariants.md) + defines the normative parse/validate/compile/runtime boundary. +- [Developer testing](docs/developer-testing.md) records local test, coverage, + rustdoc, Python, and CI-preflight commands. +- [CLI reference](docs/cli.md) summarizes commands, topic help, and backend + selection. +- [Performance and benchmarking](docs/perf.md) covers build modes and benchmark + workflow details. + [examples/tuner](examples/tuner/README.md) cover tuner specs and manual + validation commands. -```bash -uv run pytest \ - --cov=infotheory_rs \ - --cov-report=term-missing \ - --cov-report=xml:target/python-coverage.xml \ - --cov-fail-under=100 \ - python/tests -``` +## Platform Notes -For full developer test and coverage workflows (Rust + Python + VM), see: -`docs/developer-testing.md`. +CI covers Linux GNU/musl, macOS, Windows, *BSD, AArch64 variants, and +WASM slices where applicable. ZPAQ is not supported on WASM. VM support is +Linux/KVM-specific and depends on `vendor/nyx-lite`. -Notes: -- Built as `abi3-py310` (compatible with Python 3.10+). -- Published wheels are intended to be portable and exclude `vm` support by default. -- Linux source builds can opt into VM bindings by enabling the Rust `vm` feature when building the extension. - Example: `uv run maturin develop --release --features vm` -- Python trait-callback adapters (`PredictorABC`, `EnvironmentABC`, `AgentSimulatorABC`) are fail-fast: - unhandled callback exceptions terminate the process after printing traceback context. This prevents - silently continuing planning/search with invalid fallback values. +For local VM setup: + +```bash +./projman.sh init-vm +cargo build -p infotheory --release --features vm --locked +``` ## License -- This is free software, which you may use under either the Apache-2.0 License, or the ISC License, at your choice. Those are available at LICENSE-APACHE and LICENSE respectively. -- Contributing to this repository means you agree to submit all contributions under the above Licensing arrangement. In other words, such that it is available to others under either license(ISC and Apache-2.0), at the others choice. -- Don't forget to add your Copyright notice to the LICENSE file. + +InfoTheory is available under either the Apache-2.0 License or the ISC License, +at your choice. Contributing to this repository means you agree to submit +contributions under that dual-license arrangement. diff --git a/aixi_confs/aixi_bash.json b/aixi_confs/aixi_bash.json deleted file mode 100644 index 4a918f78..00000000 --- a/aixi_confs/aixi_bash.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "algorithm": "ctw", - "environment": "external", - "ct_depth": 32, - "agent_horizon": 4, - "observation_bits": 64, - "reward_bits": 8, - "agent_actions": 4, - "num_simulations": 100, - "exploration_exploitation_ratio": 1.4, - "terminate-lifetime": 20, - "external_config": { - "command": "/bin/bash", - "args": [ - "--norc", - "--noprofile" - ], - "actions": [ - "ls -1", - "cat LICENSE", - "cat Cargo.toml", - "echo 'Target found! Apache'" - ], - "reward_pattern": "Apache" - } -} \ No newline at end of file diff --git a/aixi_confs/aixi_search.json b/aixi_confs/aixi_search.json deleted file mode 100644 index c73ff843..00000000 --- a/aixi_confs/aixi_search.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "algorithm": "ac-ctw", - "environment": "external", - "ct_depth": 32, - "agent_horizon": 1, - "observation_bits": 64, - "reward_bits": 8, - "agent_actions": 5, - "num_simulations": 640, - "exploration_exploitation_ratio": 1.141, - "terminate-lifetime": 300, - "external_config": { - "command": "/bin/bash", - "args": [ - "--norc", - "--noprofile" - ], - "actions": [ - "ls -F", - "ls src/", - "cat src/main.rs", - "grep -i 'ai' src/main.rs", - "echo 'Scanning complete.'" - ], - "reward_pattern": "run_aixi_mode", - "step_cost": 2, - "verbose": false - } -} \ No newline at end of file diff --git a/aixi_confs/aixi_sequence.json b/aixi_confs/aixi_sequence.json deleted file mode 100644 index 3254ae5f..00000000 --- a/aixi_confs/aixi_sequence.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "algorithm": "ac-ctw", - "environment": "external", - "ct_depth": 32, - "agent_horizon": 4, - "observation_bits": 64, - "reward_bits": 8, - "agent_actions": 4, - "num_simulations": 600, - "exploration_exploitation_ratio": 1, - "terminate-lifetime": 420, - "external_config": { - "command": "/bin/bash", - "args": [ - "--norc", - "--noprofile" - ], - "actions": [ - "touch .lvl1 && echo 'L1_ACTIVE'", - "[ -f .lvl1 ] && touch .lvl2 && echo 'L2_ACTIVE' || echo 'FAIL_L1'", - "[ -f .lvl2 ] && echo 'ULTRA_SECRET_REACHED' || echo 'FAIL_L2'", - "rm -f .lvl1 .lvl2 && echo 'RESET'" - ], - "reward_pattern": "ULTRA_SECRET_REACHED", - "step_cost": 2, - "verbose": true - } -} \ No newline at end of file diff --git a/aixi_confs/builtin_tictactoe.json b/aixi_confs/builtin_tictactoe.json deleted file mode 100644 index fe991a24..00000000 --- a/aixi_confs/builtin_tictactoe.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "environment": "tictactoe", - "algorithm": "fac-ctw", - "ct_depth": 32, - "agent_horizon": 4, - "observation_bits": 18, - "reward_bits": 3, - "agent_actions": 9, - "reward_offset": 3, - "num_simulations": 500, - "exploration_exploitation_ratio": 1.4, - "explore_epsilon": 0.9999, - "explore_gamma": 0.999999, - "learn_cycles": 500000, - "eval_cycles": 5000 -} \ No newline at end of file diff --git a/aixi_confs/game.json b/aixi_confs/game.json deleted file mode 100644 index b71f0127..00000000 --- a/aixi_confs/game.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "algorithm": "fac-ctw", - "environment": "external", - "agent_horizon": 4, - "ct_depth": 128, - "observation_bits": 64, - "reward_bits": 8, - "agent_actions": 6, - "num_simulations": 700, - "exploration_exploitation_ratio": 1.14, - "terminate-lifetime": 500, - "external_config": { - "command": "/bin/bash", - "args": [ - "--norc", - "--noprofile" - ], - "actions": [ - "bash sim/step.sh scan", - "bash sim/step.sh reroute_power", - "bash sim/step.sh repair_A", - "bash sim/step.sh repair_B", - "bash sim/step.sh repair_C", - "bash sim/step.sh commit" - ], - "reward_pattern": "SUCCESS", - "step_cost": 2, - "verbose": true - } -} \ No newline at end of file diff --git a/aixi_confs/paper_biased_rps.json b/aixi_confs/paper_biased_rps.json deleted file mode 100644 index d7610968..00000000 --- a/aixi_confs/paper_biased_rps.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "environment": "biased-rock-paper-scissor", - "algorithm": "fac-ctw", - "ct_depth": 32, - "agent_horizon": 4, - "observation_bits": 2, - "reward_bits": 2, - "agent_actions": 3, - "reward_offset": 1, - "num_simulations": 100, - "exploration_exploitation_ratio": 0.5, - "explore_epsilon": 0.999, - "explore_gamma": 0.99999, - "learn_cycles": 15000, - "eval_cycles": 200 -} diff --git a/aixi_confs/paper_extended_tiger.json b/aixi_confs/paper_extended_tiger.json deleted file mode 100644 index 8cf9c222..00000000 --- a/aixi_confs/paper_extended_tiger.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "environment": "extended-tiger", - "algorithm": "fac-ctw", - "ct_depth": 96, - "agent_horizon": 4, - "observation_bits": 3, - "reward_bits": 8, - "agent_actions": 4, - "reward_offset": 100, - "num_simulations": 500, - "exploration_exploitation_ratio": 1.4, - "explore_epsilon": 0.99, - "explore_gamma": 0.99999, - "learn_cycles": 50000, - "eval_cycles": 5000 -} diff --git a/aixi_confs/paper_kuhn_poker.json b/aixi_confs/paper_kuhn_poker.json deleted file mode 100644 index 9afad384..00000000 --- a/aixi_confs/paper_kuhn_poker.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "environment": "kuhn-poker", - "algorithm": "fac-ctw", - "ct_depth": 16, - "trace_bit01_path": "runs/kuhnpoker.bits01.bin", - "trace_jsonl_path": "runs/kuhnpoker.trace.jsonl", - "agent_horizon": 2, - "observation_bits": 4, - "reward_bits": 3, - "agent_actions": 2, - "reward_offset": 2, - "num_simulations": 200, - "exploration_exploitation_ratio": 1.4, - "explore_epsilon": 0.99, - "explore_gamma": 0.9999, - "learn_cycles": 25000, - "eval_cycles": 200 -} diff --git a/aixi_confs/paper_tictactoe.json b/aixi_confs/paper_tictactoe.json deleted file mode 100644 index 2c81c814..00000000 --- a/aixi_confs/paper_tictactoe.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "environment": "tictactoe", - "algorithm": "fac-ctw", - "ct_depth": 64, - "agent_horizon": 9, - "observation_bits": 18, - "reward_bits": 3, - "agent_actions": 9, - "reward_offset": 3, - "num_simulations": 500, - "exploration_exploitation_ratio": 1.4, - "explore_epsilon": 0.9999, - "explore_gamma": 0.999999, - "learn_cycles": 500000, - "eval_cycles": 5000 -} diff --git a/aixi_confs/ui_biased_rps.json b/aixi_confs/ui_biased_rps.json deleted file mode 100644 index 72587728..00000000 --- a/aixi_confs/ui_biased_rps.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "environment": "biased-rock-paper-scissor", - "algorithm": "fac-ctw", - "ct_depth": 32, - "agent_horizon": 4, - "observation_bits": 2, - "reward_bits": 2, - "agent_actions": 3, - "reward_offset": 1, - "num_simulations": 120, - "exploration_exploitation_ratio": 0.6, - "explore_epsilon": 0.995, - "explore_gamma": 0.999, - "learn_cycles": 3000, - "eval_cycles": 200, - "log_every": 50 -} diff --git a/aixi_confs/ui_coin_flip.json b/aixi_confs/ui_coin_flip.json deleted file mode 100644 index 9b6f0ddb..00000000 --- a/aixi_confs/ui_coin_flip.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "environment": "coin-flip", - "algorithm": "fac-ctw", - "ct_depth": 16, - "agent_horizon": 2, - "observation_bits": 1, - "reward_bits": 1, - "agent_actions": 2, - "num_simulations": 120, - "exploration_exploitation_ratio": 1.0, - "explore_epsilon": 0.995, - "explore_gamma": 0.999, - "learn_cycles": 2000, - "eval_cycles": 200, - "log_every": 50 -} diff --git a/aixi_confs/ui_kuhn_poker.json b/aixi_confs/ui_kuhn_poker.json deleted file mode 100644 index 2e0e1574..00000000 --- a/aixi_confs/ui_kuhn_poker.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "environment": "kuhn-poker", - "algorithm": "fac-ctw", - "ct_depth": 36, - "agent_horizon": 2, - "observation_bits": 4, - "reward_bits": 3, - "agent_actions": 2, - "reward_offset": 2, - "num_simulations": 200, - "exploration_exploitation_ratio": 1.4, - "explore_epsilon": 0.99, - "explore_gamma": 0.9999, - "learn_cycles": 5000, - "eval_cycles": 500, - "log_every": 100 -} diff --git a/aixi_confs/vm_example.json b/aixi_confs/vm_example.json deleted file mode 100644 index c8873988..00000000 --- a/aixi_confs/vm_example.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "algorithm": "ctw", - "environment": "vm", - "ct_depth": 32, - "agent_horizon": 8, - "observation_bits": 8, - "observation_stream_len": 1, - "observation_key_mode": "first", - "reward_bits": 8, - "num_simulations": 0, - "exploration_exploitation_ratio": 1.4, - "discount_gamma": 1.0, - "terminate-lifetime": 50, - "explore_epsilon": 1.0, - "explore_gamma": 1.0, - "log_every": 0, - "perf": true, - "vm_config": { - "firecracker_config": "nyx-lite/vm_image/vmconfig.json", - "instance_id": "aixi-nyx", - "shared_region_name": "shared", - "shared_region_size": 4096, - "shared_memory_policy": "snapshot", - "stats_backend": { - "name": "ctw", - "ct_depth": 32 - }, - "episode_steps": 8, - "step_cost": 1, - "boot_timeout_ms": 30000, - "step_timeout_ms": 5000, - "verbose": false - }, - "vm_actions": { - "mode": "literal", - "actions": [ - { - "name": "ping", - "payload": "PING", - "encoding": "utf8" - }, - { - "name": "status", - "payload": "STATUS", - "encoding": "utf8" - } - ] - }, - "vm_observation": { - "mode": "from-guest", - "stream_len": 1, - "stream_mode": "pad-truncate" - }, - "vm_reward": { - "mode": "from-guest" - } -} diff --git a/aixi_confs/vm_perf_debug.json b/aixi_confs/vm_perf_debug.json deleted file mode 100644 index 723cbf62..00000000 --- a/aixi_confs/vm_perf_debug.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "algorithm": "ctw", - "environment": "vm", - "ct_depth": 8, - "agent_horizon": 1, - "observation_bits": 8, - "observation_stream_len": 1, - "observation_key_mode": "first", - "reward_bits": 8, - "num_simulations": 0, - "exploration_exploitation_ratio": 1.0, - "discount_gamma": 1.0, - "terminate-lifetime": 5, - "perf_cycles": 5, - "log_every": 1, - "perf": true, - "vm_perf_only": true, - "vm_config": { - "firecracker_config": "nyx-lite/vm_image/vmconfig.json", - "instance_id": "aixi-nyx", - "shared_region_name": "shared", - "shared_region_size": 4096, - "shared_memory_policy": "snapshot", - "episode_steps": 1000, - "step_cost": 0, - "boot_timeout_ms": 30000, - "step_timeout_ms": 1000, - "verbose": true - }, - "vm_actions": { - "mode": "literal", - "actions": [ - { - "name": "ping", - "payload": "PING", - "encoding": "utf8" - } - ] - }, - "vm_observation": { - "mode": "from-guest", - "stream_len": 1, - "stream_mode": "pad-truncate" - }, - "vm_reward": { - "mode": "from-guest" - } -} diff --git a/aixi_confs/vm_perf_fast.json b/aixi_confs/vm_perf_fast.json deleted file mode 100644 index 604768d6..00000000 --- a/aixi_confs/vm_perf_fast.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "algorithm": "ctw", - "environment": "vm", - "ct_depth": 8, - "agent_horizon": 1, - "observation_bits": 8, - "observation_stream_len": 1, - "observation_key_mode": "first", - "reward_bits": 8, - "num_simulations": 0, - "exploration_exploitation_ratio": 1.0, - "discount_gamma": 1.0, - "terminate-lifetime": 20000, - "perf_cycles": 2000, - "log_every": 0, - "perf": true, - "vm_perf_only": true, - "vm_config": { - "firecracker_config": "nyx-lite/vm_image/vmconfig.json", - "instance_id": "aixi-nyx", - "shared_region_name": "shared", - "shared_region_size": 4096, - "shared_memory_policy": "snapshot", - "episode_steps": 1000, - "step_cost": 0, - "boot_timeout_ms": 30000, - "step_timeout_ms": 50, - "verbose": false - }, - "vm_actions": { - "mode": "literal", - "actions": [ - { - "name": "ping", - "payload": "PING", - "encoding": "utf8" - } - ] - }, - "vm_observation": { - "mode": "from-guest", - "stream_len": 1, - "stream_mode": "pad-truncate" - }, - "vm_reward": { - "mode": "from-guest" - } -} diff --git a/aixi_confs/vm_sudo_fuzz.json b/aixi_confs/vm_sudo_fuzz.json deleted file mode 100644 index fcc34434..00000000 --- a/aixi_confs/vm_sudo_fuzz.json +++ /dev/null @@ -1,93 +0,0 @@ -{ - "algorithm": "ctw", - "environment": "vm", - "ct_depth": 32, - "agent_horizon": 4, - "observation_bits": 8, - "observation_stream_len": 64, - "observation_key_mode": "stream-hash", - "reward_bits": 16, - "num_simulations": 0, - "exploration_exploitation_ratio": 2.0, - "discount_gamma": 0.95, - "terminate-lifetime": 10000, - "log_every": 500, - "perf": true, - "vm_config": { - "firecracker_config": "nyx-lite/vm_image/vmconfig.json", - "instance_id": "aixi-nyx", - "shared_region_name": "shared", - "shared_region_size": 4096, - "shared_memory_policy": "snapshot", - "stats_backend": { - "name": "rosa", - "max_order": 4 - }, - "episode_steps": 1, - "step_cost": 0, - "boot_timeout_ms": 30000, - "step_timeout_ms": 3000, - "verbose": false, - "crash_log": "sudo_crashes.jsonl" - }, - "vm_actions": { - "mode": "fuzz", - "fuzz": { - "seed_inputs": [ - "sudo -l\n", - "sudo -u root id\n", - "sudo -u#0 id\n", - "sudo -u#-1 id\n", - "sudo whoami\n", - "sudo -s\n", - "sudo -i\n", - "sudo -k\n", - "sudo -v\n", - "sudo env\n", - "sudo -E whoami\n", - "sudo -H whoami\n", - "sudo -u root -g root id\n", - "sudo -- id\n", - "sudo -n whoami\n", - "sudo -p 'Password:' whoami\n", - "sudo -S whoami\n", - "sudo -A whoami\n", - "sudo -b whoami\n", - "sudo -C 5 whoami\n", - "sudo --\n", - "sudo -u \n", - "sudo -u '' id\n", - "sudo -u root id\n", - "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\n", - "sudo ", - "\n\n\n\n\n", - "\u0000\u0000\u0000\u0000", - "sudo -u $(id -u) id\n", - "sudo -u `whoami` id\n" - ], - "mutators": [ - "flip_bit", - "insert_byte", - "delete_byte", - "havoc", - "splice" - ], - "min_len": 1, - "max_len": 512, - "rng_seed": 12345 - } - }, - "vm_observation": { - "mode": "shared-memory", - "stream_len": 64, - "stream_mode": "pad-truncate" - }, - "vm_reward": { - "mode": "entropy-reduction", - "baseline_path": "aixi_confs/sudo_baseline.txt", - "max_order": 3, - "scale": 10.0, - "crash_bonus": 100, - "timeout_bonus": 50 - } -} \ No newline at end of file diff --git a/aixi_confs/vm_sudo_fuzz_test.json b/aixi_confs/vm_sudo_fuzz_test.json deleted file mode 100644 index 35bcfc75..00000000 --- a/aixi_confs/vm_sudo_fuzz_test.json +++ /dev/null @@ -1,93 +0,0 @@ -{ - "algorithm": "ctw", - "environment": "vm", - "ct_depth": 32, - "agent_horizon": 4, - "observation_bits": 8, - "observation_stream_len": 64, - "observation_key_mode": "stream-hash", - "reward_bits": 16, - "num_simulations": 0, - "exploration_exploitation_ratio": 2.0, - "discount_gamma": 0.95, - "terminate-lifetime": 500, - "log_every": 50, - "perf": true, - "vm_config": { - "firecracker_config": "nyx-lite/vm_image/vmconfig.json", - "instance_id": "aixi-nyx", - "shared_region_name": "shared", - "shared_region_size": 4096, - "shared_memory_policy": "snapshot", - "stats_backend": { - "name": "rosa", - "max_order": 4 - }, - "episode_steps": 1, - "step_cost": 0, - "boot_timeout_ms": 30000, - "step_timeout_ms": 3000, - "verbose": false, - "crash_log": "sudo_crashes.jsonl" - }, - "vm_actions": { - "mode": "fuzz", - "fuzz": { - "seed_inputs": [ - "sudo -l\n", - "sudo -u root id\n", - "sudo -u#0 id\n", - "sudo -u#-1 id\n", - "sudo whoami\n", - "sudo -s\n", - "sudo -i\n", - "sudo -k\n", - "sudo -v\n", - "sudo env\n", - "sudo -E whoami\n", - "sudo -H whoami\n", - "sudo -u root -g root id\n", - "sudo -- id\n", - "sudo -n whoami\n", - "sudo -p 'Password:' whoami\n", - "sudo -S whoami\n", - "sudo -A whoami\n", - "sudo -b whoami\n", - "sudo -C 5 whoami\n", - "sudo --\n", - "sudo -u \n", - "sudo -u '' id\n", - "sudo -u root id\n", - "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\n", - "sudo ", - "\n\n\n\n\n", - "\u0000\u0000\u0000\u0000", - "sudo -u $(id -u) id\n", - "sudo -u `whoami` id\n" - ], - "mutators": [ - "flip_bit", - "insert_byte", - "delete_byte", - "havoc", - "splice" - ], - "min_len": 1, - "max_len": 512, - "rng_seed": 12345 - } - }, - "vm_observation": { - "mode": "shared-memory", - "stream_len": 64, - "stream_mode": "pad-truncate" - }, - "vm_reward": { - "mode": "entropy-reduction", - "baseline_path": "aixi_confs/sudo_baseline.txt", - "max_order": 3, - "scale": 10.0, - "crash_bonus": 100, - "timeout_bonus": 50 - } -} \ No newline at end of file diff --git a/aixi_confs/vm_trace_example.json b/aixi_confs/vm_trace_example.json deleted file mode 100644 index 8bc3d439..00000000 --- a/aixi_confs/vm_trace_example.json +++ /dev/null @@ -1,63 +0,0 @@ -{ - "algorithm": "ctw", - "environment": "vm", - "ct_depth": 32, - "agent_horizon": 8, - "observation_bits": 8, - "observation_stream_len": 64, - "observation_key_mode": "stream-hash", - "reward_bits": 16, - "num_simulations": 100, - "exploration_exploitation_ratio": 1.4, - "discount_gamma": 1.0, - "terminate-lifetime": 50, - "log_every": 0, - "vm_config": { - "firecracker_config": "nyx-lite/vm_image/vmconfig.json", - "instance_id": "aixi-nyx", - "shared_region_name": "shared", - "shared_region_size": 4096, - "shared_memory_policy": "snapshot", - "stats_backend": { - "name": "ctw", - "ct_depth": 32 - }, - "episode_steps": 8, - "step_cost": 1, - "boot_timeout_ms": 30000, - "step_timeout_ms": 5000, - "verbose": false - }, - "vm_trace": { - "mode": "shared-memory", - "shared_region_name": "trace", - "max_bytes": 1048576, - "reset_on_episode": false - }, - "vm_actions": { - "mode": "literal", - "actions": [ - { - "name": "ping", - "payload": "PING", - "encoding": "utf8" - }, - { - "name": "status", - "payload": "STATUS", - "encoding": "utf8" - } - ] - }, - "vm_observation": { - "mode": "shared-memory", - "stream_len": 64, - "stream_mode": "pad-truncate" - }, - "vm_reward": { - "mode": "trace-entropy", - "max_order": 8, - "scale": 1.0, - "normalize": false - } -} diff --git a/benches/aixi.rs b/benches/aixi.rs deleted file mode 100644 index 14c55491..00000000 --- a/benches/aixi.rs +++ /dev/null @@ -1,183 +0,0 @@ -use infotheory::aixi::agent::{Agent, AgentConfig}; -use infotheory::aixi::environment::{BiasedRockPaperScissor, Environment}; -use infotheory::{MixtureExpertSpec, MixtureKind, MixtureScheduleMode, MixtureSpec, RateBackend}; -use std::sync::Arc; -use std::time::{Duration, Instant}; - -fn env_usize(name: &str, default: usize) -> usize { - std::env::var(name) - .ok() - .and_then(|value| value.parse::().ok()) - .filter(|&value| value > 0) - .unwrap_or(default) -} - -fn bench_agent( - mut agent: Agent, - mut env: Box, - cycles: usize, - warmup: usize, -) -> Duration { - let mut prev_action = 0u64; - let mut obs_stream = env.drain_observations(); - let mut rew = env.get_reward(); - - // Warmup - for _ in 0..warmup { - agent.model_update_percept_stream(&obs_stream, rew); - let action = agent.get_planned_action(&obs_stream, rew, prev_action); - agent.model_update_action_external(action); - env.perform_action(action); - obs_stream = env.drain_observations(); - rew = env.get_reward(); - prev_action = action; - } - - let now = Instant::now(); - for _ in 0..cycles { - agent.model_update_percept_stream(&obs_stream, rew); - let action = agent.get_planned_action(&obs_stream, rew, prev_action); - agent.model_update_action_external(action); - env.perform_action(action); - obs_stream = env.drain_observations(); - rew = env.get_reward(); - prev_action = action; - } - now.elapsed() -} - -fn main() { - // Keep these fixed so backends are comparable. - let cycles = env_usize("AIXI_BENCH_CYCLES", 2_000); - let warmup = env_usize("AIXI_BENCH_WARMUP", 200); - - // Environment chosen because it has non-trivial observation+reward bits (2+2) and 3 actions. - let env_name = "biased-rock-paper-scissor"; - - // Benchmark config: moderate horizon + simulations so MCTS is the dominant cost. - // reward_offset is required for correct unsigned reward encoding. - let base_cfg = |algorithm: &str| AgentConfig { - algorithm: algorithm.to_string(), - ct_depth: 32, - agent_horizon: 5, - observation_bits: 2, - observation_stream_len: 1, - observation_key_mode: infotheory::aixi::common::ObservationKeyMode::FullStream, - reward_bits: 2, - agent_actions: 3, - num_simulations: 400, - exploration_exploitation_ratio: 1.4, - discount_gamma: 1.0, - min_reward: -1, - max_reward: 1, - reward_offset: 1, - random_seed: Some(1), - rate_backend: None, - rate_backend_max_order: 20, - rwkv_model_path: None, - rwkv_method: None, - mamba_model_path: None, - mamba_method: None, - rosa_max_order: Some(20), - zpaq_method: None, - }; - - let make_mixture = - |kind: MixtureKind, alpha: f64, schedule: MixtureScheduleMode| RateBackend::Mixture { - spec: Arc::new( - MixtureSpec::new( - kind, - vec![ - MixtureExpertSpec { - name: Some("ctw".to_string()), - log_prior: 0.0, - max_order: -1, - backend: RateBackend::Ctw { depth: 32 }, - }, - MixtureExpertSpec { - name: Some("rosa".to_string()), - log_prior: 0.0, - max_order: 20, - backend: RateBackend::RosaPlus, - }, - ], - ) - .with_schedule(schedule) - .with_alpha(alpha), - ), - }; - let rate_backend_cfg = |backend: RateBackend| { - let mut cfg = base_cfg("mixture"); - cfg.rate_backend = Some(backend); - cfg - }; - - let benches = [ - ("fac-ctw", base_cfg("fac-ctw")), - ("rosa", base_cfg("rosa")), - ("rate-ctw", rate_backend_cfg(RateBackend::Ctw { depth: 32 })), - ("rate-rosa", rate_backend_cfg(RateBackend::RosaPlus)), - ( - "mix-bayes", - rate_backend_cfg(make_mixture( - MixtureKind::Bayes, - 0.01, - MixtureScheduleMode::Default, - )), - ), - ( - "mix-switch", - rate_backend_cfg(make_mixture( - MixtureKind::Switching, - 0.17, - MixtureScheduleMode::Default, - )), - ), - ( - "mix-switch-thm", - rate_backend_cfg(make_mixture( - MixtureKind::Switching, - 0.99, - MixtureScheduleMode::Theorem, - )), - ), - ( - "mix-convex", - rate_backend_cfg(make_mixture( - MixtureKind::Convex, - 1.25, - MixtureScheduleMode::Default, - )), - ), - ( - "mix-convex-thm", - rate_backend_cfg(make_mixture( - MixtureKind::Convex, - 7.5, - MixtureScheduleMode::Theorem, - )), - ), - ]; - - println!( - "MC-AIXI benchmark (env={}, warmup={}, cycles={})", - env_name, warmup, cycles - ); - for (name, cfg) in benches { - // Fresh env per backend to keep interaction distribution identical. - // (Environment is stochastic; we intentionally measure typical runtime not identical traces.) - let env: Box = Box::new(BiasedRockPaperScissor::new()); - let agent = Agent::new(cfg.clone()); - let elapsed = bench_agent(agent, env, cycles, warmup); - - let ns_per_cycle = (elapsed.as_nanos() as f64) / (cycles as f64); - let cycles_per_s = (cycles as f64) / elapsed.as_secs_f64().max(1e-12); - println!( - "{:>7}: {:>8.3} ms total | {:>10.1} ns/cycle | {:>10.1} cycles/s", - name, - elapsed.as_secs_f64() * 1e3, - ns_per_cycle, - cycles_per_s - ); - } -} diff --git a/benches/par.rs b/benches/par.rs deleted file mode 100644 index 39165e4a..00000000 --- a/benches/par.rs +++ /dev/null @@ -1,72 +0,0 @@ -use infotheory::*; -use std::time::Instant; - -fn main() { - let paths: Vec<&str> = vec!["compressme", "scompressme", "largebench"]; - for _ in 0..32 { - println!( - "{:?}", - get_sequential_compressed_sizes_from_sequential_paths(&paths, "x4.3ci1") - ); - } - let now = Instant::now(); - for _ in 0..32 { - println!( - "{:?}", - get_sequential_compressed_sizes_from_sequential_paths(&paths, "x4.3ci1") - ); - } - let elapsed = now.elapsed(); - println!("seq_seq: Elapsed time for 32 runs: {:?}", elapsed); - for _ in 0..32 { - println!( - "{:?}", - get_sequential_compressed_sizes_from_parallel_paths(&paths, "x4.3ci1") - ); - } - let now = Instant::now(); - for _ in 0..32 { - println!( - "{:?}", - get_sequential_compressed_sizes_from_parallel_paths(&paths, "x4.3ci1") - ); - } - let elapsed = now.elapsed(); - println!("seq_par: Elapsed time for 32 runs: {:?}", elapsed); - for _ in 0..32 { - println!( - "{:?}", - get_parallel_compressed_sizes_from_sequential_paths(&paths, "x4.3ci1", 4) - ); - } - let now = Instant::now(); - for _ in 0..32 { - println!( - "{:?}", - get_parallel_compressed_sizes_from_sequential_paths(&paths, "x4.3ci1", 4) - ); - } - let elapsed = now.elapsed(); - println!("par_seq: Elapsed time for 32 runs: {:?}", elapsed); - for _ in 0..32 { - println!( - "{:?}", - get_parallel_compressed_sizes_from_parallel_paths(&paths, "x4.3ci1", 4) - ); - } - let now = Instant::now(); - for _ in 0..32 { - println!( - "{:?}", - get_parallel_compressed_sizes_from_parallel_paths(&paths, "x4.3ci1", 4) - ); - } - let elapsed = now.elapsed(); - println!("par_par: Elapsed time for 32 runs: {:?}", elapsed); - let now = Instant::now(); - for _ in 0..32 { - println!("{:?}", get_compressed_sizes_from_paths(&paths, "x4.3ci1")); - } - let elapsed = now.elapsed(); - println!("par_par: Elapsed time for 32 runs: {:?}", elapsed); -} diff --git a/benchmarks/36ca6395/infotheory-two-json-summary-current-full.tsv b/benchmarks/36ca6395/infotheory-two-json-summary-current-full.tsv index 1541f5c0..02ec3fa4 100644 --- a/benchmarks/36ca6395/infotheory-two-json-summary-current-full.tsv +++ b/benchmarks/36ca6395/infotheory-two-json-summary-current-full.tsv @@ -1,145 +1,145 @@ -operation subject subject_kind expert_kind series size_bytes repeats cpu compression_backend input_sha256 real_seconds_mean real_seconds_stdev real_seconds_median real_seconds_min real_seconds_max user_seconds_mean sys_seconds_mean throughput_mib_s_mean throughput_mib_s_median rss_kib_mean rss_kib_stdev rss_kib_median rss_kib_min rss_kib_max archive_bytes_mean archive_bytes_median archive_ratio_mean archive_ratio_median entropy_bpb_mean entropy_bpb_median verified_all -h ctw expert ctw h:ctw 4096 1 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.04 0 0.04 0.04 0.04 0.04 0 0.09765625 0.09765625 11824 0 11824 11824 11824 2.56549624542 2.56549624542 1 -h ctw expert ctw h:ctw 16384 1 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.18 0 0.18 0.18 0.18 0.17 0 0.0868055555556 0.0868055555556 34480 0 34480 34480 34480 3.20340951209 3.20340951209 1 -h ctw expert ctw h:ctw 65536 1 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.77 0 0.77 0.77 0.77 0.75 0.02 0.0811688311688 0.0811688311688 76356 0 76356 76356 76356 2.77213895634 2.77213895634 1 -h ctw expert ctw h:ctw 262144 1 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 3.28 0 3.28 3.28 3.28 3.22 0.06 0.0762195121951 0.0762195121951 185300 0 185300 185300 185300 2.44096714152 2.44096714152 1 -h ctw expert ctw h:ctw 1048576 1 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 14.46 0 14.46 14.46 14.46 14.24 0.21 0.0691562932227 0.0691562932227 470788 0 470788 470788 470788 2.30174274755 2.30174274755 1 -h ctw expert ctw h:ctw 2097152 1 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 30.58 0 30.58 30.58 30.58 30.24 0.3 0.0654022236756 0.0654022236756 774868 0 774868 774868 774868 2.27731545661 2.27731545661 1 -h ctw expert ctw h:ctw 4194304 1 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 64.74 0 64.74 64.74 64.74 64.19 0.48 0.0617856039543 0.0617856039543 1291640 0 1291640 1291640 1291640 2.24674930722 2.24674930722 1 -h ctw expert ctw h:ctw 10000000 1 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 166.38 0 166.38 166.38 166.38 165.17 1.03 0.0573190477465 0.0573190477465 2487416 0 2487416 2487416 2487416 2.19747398481 2.19747398481 1 -h match expert match h:match 4096 1 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0 0 0 0 0 0 0 5688 0 5688 5688 5688 5.57734993252 5.57734993252 1 -h match expert match h:match 16384 1 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0 0 0 0 0 0 0 5612 0 5612 5612 5612 6.58586286407 6.58586286407 1 -h match expert match h:match 65536 1 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.01 0 0.01 0.01 0.01 0 0 6.25 6.25 6632 0 6632 6632 6632 6.5588255305 6.5588255305 1 -h match expert match h:match 262144 1 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 0.03 0 0.03 0.03 0.03 0.03 0 8.33333333333 8.33333333333 8012 0 8012 8012 8012 6.29225944669 6.29225944669 1 -h match expert match h:match 1048576 1 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 0.13 0 0.13 0.13 0.13 0.13 0 7.69230769231 7.69230769231 11452 0 11452 11452 11452 6.23692635511 6.23692635511 1 -h match expert match h:match 2097152 1 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 0.28 0 0.28 0.28 0.28 0.28 0 7.14285714286 7.14285714286 19132 0 19132 19132 19132 6.2741772649 6.2741772649 1 -h match expert match h:match 4194304 1 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 0.56 0 0.56 0.56 0.56 0.55 0 7.14285714286 7.14285714286 21268 0 21268 21268 21268 6.28038800899 6.28038800899 1 -h match expert match h:match 10000000 1 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 1.35 0 1.35 1.35 1.35 1.34 0.01 7.0642541956 7.0642541956 40264 0 40264 40264 40264 6.28744436593 6.28744436593 1 -h neural_mixture mixture neural-mixture h:neural_mixture 4096 1 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.19 0 0.19 0.19 0.19 0.19 0 0.0205592105263 0.0205592105263 17912 0 17912 17912 17912 1.91857366907 1.91857366907 1 -h neural_mixture mixture neural-mixture h:neural_mixture 16384 1 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.82 0 0.82 0.82 0.82 0.8 0.01 0.0190548780488 0.0190548780488 52724 0 52724 52724 52724 2.69245779825 2.69245779825 1 -h neural_mixture mixture neural-mixture h:neural_mixture 65536 1 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 3.52 0 3.52 3.52 3.52 3.47 0.03 0.0177556818182 0.0177556818182 139048 0 139048 139048 139048 2.35902117116 2.35902117116 1 -h neural_mixture mixture neural-mixture h:neural_mixture 262144 1 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 15.02 0 15.02 15.02 15.02 14.82 0.17 0.0166444740346 0.0166444740346 430348 0 430348 430348 430348 2.05795611601 2.05795611601 1 -h neural_mixture mixture neural-mixture h:neural_mixture 1048576 1 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 65.26 0 65.26 65.26 65.26 64.66 0.52 0.0153233220962 0.0153233220962 1108724 0 1108724 1108724 1108724 1.93963669332 1.93963669332 1 -h neural_mixture mixture neural-mixture h:neural_mixture 2097152 1 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 135.64 0 135.64 135.64 135.64 134.76 0.74 0.014744913005 0.014744913005 1719704 0 1719704 1719704 1719704 1.92069854644 1.92069854644 1 -h neural_mixture mixture neural-mixture h:neural_mixture 4194304 1 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 283.39 0 283.39 283.39 283.39 282.01 1.07 0.014114824094 0.014114824094 2623920 0 2623920 2623920 2623920 1.88509670528 1.88509670528 1 -h neural_mixture mixture neural-mixture h:neural_mixture 10000000 1 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 715.33 0 715.33 715.33 715.33 712.24 2.27 0.0133319491201 0.0133319491201 4760388 0 4760388 4760388 4760388 1.81617547371 1.81617547371 1 -h ppmd expert ppmd h:ppmd 4096 1 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.01 0 0.01 0.01 0.01 0.01 0 0.390625 0.390625 7352 0 7352 7352 7352 2.02916276221 2.02916276221 1 -h ppmd expert ppmd h:ppmd 16384 1 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.03 0 0.03 0.03 0.03 0.03 0 0.520833333333 0.520833333333 16100 0 16100 16100 16100 3.05111741489 3.05111741489 1 -h ppmd expert ppmd h:ppmd 65536 1 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.19 0 0.19 0.19 0.19 0.18 0 0.328947368421 0.328947368421 45260 0 45260 45260 45260 2.82800309907 2.82800309907 1 -h ppmd expert ppmd h:ppmd 262144 1 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 0.93 0 0.93 0.93 0.93 0.87 0.06 0.268817204301 0.268817204301 144212 0 144212 144212 144212 2.54059561824 2.54059561824 1 -h ppmd expert ppmd h:ppmd 1048576 1 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 3.93 0 3.93 3.93 3.93 3.72 0.19 0.254452926209 0.254452926209 398696 0 398696 398696 398696 2.48956475398 2.48956475398 1 -h ppmd expert ppmd h:ppmd 2097152 1 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 7.61 0 7.61 7.61 7.61 7.39 0.21 0.262812089356 0.262812089356 457100 0 457100 457100 457100 2.52508057302 2.52508057302 1 -h ppmd expert ppmd h:ppmd 4194304 1 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 15.19 0 15.19 15.19 15.19 14.94 0.24 0.263331138907 0.263331138907 459116 0 459116 459116 459116 2.5353209835 2.5353209835 1 -h ppmd expert ppmd h:ppmd 10000000 1 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 36.63 0 36.63 36.63 36.63 36.27 0.32 0.260353348732 0.260353348732 557108 0 557108 557108 557108 2.52355814767 2.52355814767 1 -h rosa expert rosaplus h:rosa 4096 1 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.02 0 0.02 0.02 0.02 0.02 0 0.1953125 0.1953125 5924 0 5924 5924 5924 2.11172124286 2.11172124286 1 -h rosa expert rosaplus h:rosa 16384 1 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.12 0 0.12 0.12 0.12 0.11 0.01 0.130208333333 0.130208333333 8224 0 8224 8224 8224 3.48630622986 3.48630622986 1 -h rosa expert rosaplus h:rosa 65536 1 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.62 0 0.62 0.62 0.62 0.6 0.02 0.100806451613 0.100806451613 16676 0 16676 16676 16676 3.19056696201 3.19056696201 1 -h rosa expert rosaplus h:rosa 262144 1 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 3.6 0 3.6 3.6 3.6 3.49 0.1 0.0694444444444 0.0694444444444 47052 0 47052 47052 47052 2.90567254487 2.90567254487 1 -h rosa expert rosaplus h:rosa 1048576 1 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 21.23 0 21.23 21.23 21.23 20.34 0.86 0.0471031559114 0.0471031559114 203892 0 203892 203892 203892 2.80123027825 2.80123027825 1 -h rosa expert rosaplus h:rosa 2097152 1 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 50.81 0 50.81 50.81 50.81 48.72 2.02 0.03936233025 0.03936233025 337584 0 337584 337584 337584 2.70437138702 2.70437138702 1 -h rosa expert rosaplus h:rosa 4194304 1 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 121.27 0 121.27 121.27 121.27 115.95 5.18 0.0329842500206 0.0329842500206 643432 0 643432 643432 643432 2.62099230176 2.62099230176 1 -h rosa expert rosaplus h:rosa 10000000 1 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 356.22 0 356.22 356.22 356.22 342.7 13.11 0.0267720598621 0.0267720598621 1551456 0 1551456 1551456 1551456 2.48481897791 2.48481897791 1 -h rwkv expert rwkv h:rwkv 4096 1 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.06 0 0.06 0.06 0.06 0.06 0 0.0651041666667 0.0651041666667 8000 0 8000 8000 8000 7.21701437947 7.21701437947 1 -h rwkv expert rwkv h:rwkv 16384 1 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.24 0 0.24 0.24 0.24 0.24 0 0.0651041666667 0.0651041666667 8172 0 8172 8172 8172 5.85021072042 5.85021072042 1 -h rwkv expert rwkv h:rwkv 65536 1 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.94 0 0.94 0.94 0.94 0.94 0 0.0664893617021 0.0664893617021 8336 0 8336 8336 8336 4.31173851819 4.31173851819 1 -h rwkv expert rwkv h:rwkv 262144 1 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 3.69 0 3.69 3.69 3.69 3.69 0 0.0677506775068 0.0677506775068 8320 0 8320 8320 8320 4.18065126521 4.18065126521 1 -h rwkv expert rwkv h:rwkv 1048576 1 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 15.57 0 15.57 15.57 15.57 15.55 0 0.0642260757868 0.0642260757868 8796 0 8796 8796 8796 3.61829048151 3.61829048151 1 -h rwkv expert rwkv h:rwkv 2097152 1 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 31.61 0 31.61 31.61 31.61 31.58 0 0.0632711167352 0.0632711167352 9912 0 9912 9912 9912 3.49833649966 3.49833649966 1 -h rwkv expert rwkv h:rwkv 4194304 1 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 63.22 0 63.22 63.22 63.22 63.18 0 0.0632711167352 0.0632711167352 11888 0 11888 11888 11888 3.33570098711 3.33570098711 1 -h rwkv expert rwkv h:rwkv 10000000 1 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 156.53 0 156.53 156.53 156.53 156.4 0 0.0609259768994 0.0609259768994 17764 0 17764 17764 17764 3.17490278117 3.17490278117 1 -compress ctw expert ctw compress:ctw 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.05 0 0.05 0.05 0.05 0.04 0 0.078125 0.078125 11616 0 11616 11616 11616 1332 1332 0.3251953125 0.3251953125 1 -compress ctw expert ctw compress:ctw 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.23 0 0.23 0.23 0.23 0.22 0 0.0679347826087 0.0679347826087 34400 0 34400 34400 34400 6579 6579 0.401550292969 0.401550292969 1 -compress ctw expert ctw compress:ctw 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 1 0 1 1 1 0.97 0.03 0.0625 0.0625 76372 0 76372 76372 76372 22728 22728 0.346801757812 0.346801757812 1 -compress ctw expert ctw compress:ctw 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 4.21 0 4.21 4.21 4.21 4.13 0.07 0.0593824228029 0.0593824228029 185720 0 185720 185720 185720 80004 80004 0.305191040039 0.305191040039 1 -compress ctw expert ctw compress:ctw 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 18.06 0 18.06 18.06 18.06 17.84 0.19 0.0553709856035 0.0553709856035 471292 0 471292 471292 471292 301713 301713 0.287735939026 0.287735939026 1 -compress ctw expert ctw compress:ctw 2097152 1 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 37.54 0 37.54 37.54 37.54 37.23 0.27 0.0532765050613 0.0532765050613 776116 0 776116 776116 776116 597003 597003 0.284673213959 0.284673213959 1 -compress ctw expert ctw compress:ctw 4194304 1 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 78.4 0 78.4 78.4 78.4 77.77 0.55 0.0510204081633 0.0510204081633 1294124 0 1294124 1294124 1294124 1177962 1177962 0.280848026276 0.280848026276 1 -compress ctw expert ctw compress:ctw 10000000 1 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 198.36 0 198.36 198.36 198.36 196.98 1.16 0.0480779550517 0.0480779550517 2493308 0 2493308 2493308 2493308 2746861 2746861 0.2746861 0.2746861 1 -compress match expert match compress:match 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0 0 0 0 0 0 0 5528 0 5528 5528 5528 2874 2874 0.70166015625 0.70166015625 1 -compress match expert match compress:match 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.01 0 0.01 0.01 0.01 0.01 0 1.5625 1.5625 5604 0 5604 5604 5604 13506 13506 0.824340820312 0.824340820312 1 -compress match expert match compress:match 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.03 0 0.03 0.03 0.03 0.03 0 2.08333333333 2.08333333333 6332 0 6332 6332 6332 53748 53748 0.820129394531 0.820129394531 1 -compress match expert match compress:match 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 0.14 0 0.14 0.14 0.14 0.13 0 1.78571428571 1.78571428571 7984 0 7984 7984 7984 206203 206203 0.786602020264 0.786602020264 1 -compress match expert match compress:match 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 0.53 0 0.53 0.53 0.53 0.53 0 1.88679245283 1.88679245283 11972 0 11972 11972 11972 817505 817505 0.779633522034 0.779633522034 1 -compress match expert match compress:match 2097152 1 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 1.08 0 1.08 1.08 1.08 1.07 0 1.85185185185 1.85185185185 22024 0 22024 22024 22024 1644756 1644756 0.784280776978 0.784280776978 1 -compress match expert match compress:match 4194304 1 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 2.15 0 2.15 2.15 2.15 2.13 0 1.86046511628 1.86046511628 28812 0 28812 28812 28812 3292751 3292751 0.785053014755 0.785053014755 1 -compress match expert match compress:match 10000000 1 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 5.18 0 5.18 5.18 5.18 5.15 0.02 1.84107010889 1.84107010889 55408 0 55408 55408 55408 7859324 7859324 0.7859324 0.7859324 1 -compress neural_mixture mixture neural-mixture compress:neural_mixture 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.16 0 0.16 0.16 0.16 0.16 0 0.0244140625 0.0244140625 17464 0 17464 17464 17464 1001 1001 0.244384765625 0.244384765625 1 -compress neural_mixture mixture neural-mixture compress:neural_mixture 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.67 0 0.67 0.67 0.67 0.65 0.01 0.0233208955224 0.0233208955224 52280 0 52280 52280 52280 5533 5533 0.337707519531 0.337707519531 1 -compress neural_mixture mixture neural-mixture compress:neural_mixture 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 2.87 0 2.87 2.87 2.87 2.8 0.06 0.0217770034843 0.0217770034843 139660 0 139660 139660 139660 19344 19344 0.295166015625 0.295166015625 1 -compress neural_mixture mixture neural-mixture compress:neural_mixture 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 12.6 0 12.6 12.6 12.6 12.41 0.17 0.0198412698413 0.0198412698413 428292 0 428292 428292 428292 67454 67454 0.257316589355 0.257316589355 1 -compress neural_mixture mixture neural-mixture compress:neural_mixture 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 56.06 0 56.06 56.06 56.06 55.47 0.53 0.0178380306814 0.0178380306814 1103428 0 1103428 1103428 1103428 254251 254251 0.242472648621 0.242472648621 1 -compress neural_mixture mixture neural-mixture compress:neural_mixture 2097152 1 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 117.74 0 117.74 117.74 117.74 116.91 0.69 0.0169865806013 0.0169865806013 1734756 0 1734756 1734756 1734756 503519 503519 0.240096569061 0.240096569061 1 -compress neural_mixture mixture neural-mixture compress:neural_mixture 4194304 1 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 248.57 0 248.57 248.57 248.57 247.17 1.11 0.016092046506 0.016092046506 2664248 0 2664248 2664248 2664248 988355 988355 0.235642194748 0.235642194748 1 -compress neural_mixture mixture neural-mixture compress:neural_mixture 10000000 1 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 637.69 0 637.69 637.69 637.69 634.67 2.31 0.0149551399019 0.0149551399019 4777992 0 4777992 4777992 4777992 2270248 2270248 0.2270248 0.2270248 1 -compress ppmd expert ppmd compress:ppmd 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.01 0 0.01 0.01 0.01 0 0 0.390625 0.390625 7036 0 7036 7036 7036 1058 1058 0.25830078125 0.25830078125 1 -compress ppmd expert ppmd compress:ppmd 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.04 0 0.04 0.04 0.04 0.03 0 0.390625 0.390625 16076 0 16076 16076 16076 6267 6267 0.382507324219 0.382507324219 1 -compress ppmd expert ppmd compress:ppmd 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.22 0 0.22 0.22 0.22 0.2 0.01 0.284090909091 0.284090909091 44976 0 44976 44976 44976 23186 23186 0.353790283203 0.353790283203 1 -compress ppmd expert ppmd compress:ppmd 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 1.07 0 1.07 1.07 1.07 1 0.07 0.233644859813 0.233644859813 143988 0 143988 143988 143988 83269 83269 0.317646026611 0.317646026611 1 -compress ppmd expert ppmd compress:ppmd 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 4.29 0 4.29 4.29 4.29 4.11 0.17 0.2331002331 0.2331002331 399344 0 399344 399344 399344 326331 326331 0.311213493347 0.311213493347 1 -compress ppmd expert ppmd compress:ppmd 2097152 1 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 8.35 0 8.35 8.35 8.35 8.15 0.18 0.239520958084 0.239520958084 457680 0 457680 457680 457680 661953 661953 0.315643787384 0.315643787384 1 -compress ppmd expert ppmd compress:ppmd 4194304 1 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 16.55 0 16.55 16.55 16.55 16.34 0.19 0.2416918429 0.2416918429 459764 0 459764 459764 459764 1329257 1329257 0.316919565201 0.316919565201 1 -compress ppmd expert ppmd compress:ppmd 10000000 1 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 40.16 0 40.16 40.16 40.16 39.86 0.25 0.237468704284 0.237468704284 568928 0 568928 568928 568928 3154465 3154465 0.3154465 0.3154465 1 -compress rosa expert rosaplus compress:rosa 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0 0 0 0 0 0 0 5768 0 5768 5768 5768 1127 1127 0.275146484375 0.275146484375 1 -compress rosa expert rosaplus compress:rosa 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.03 0 0.03 0.03 0.03 0.03 0 0.520833333333 0.520833333333 8544 0 8544 8544 8544 6359 6359 0.388122558594 0.388122558594 1 -compress rosa expert rosaplus compress:rosa 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.15 0 0.15 0.15 0.15 0.14 0 0.416666666667 0.416666666667 19472 0 19472 19472 19472 22843 22843 0.348556518555 0.348556518555 1 -compress rosa expert rosaplus compress:rosa 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 0.88 0 0.88 0.88 0.88 0.85 0.01 0.284090909091 0.284090909091 61972 0 61972 61972 61972 80590 80590 0.307426452637 0.307426452637 1 -compress rosa expert rosaplus compress:rosa 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 4.9 0 4.9 4.9 4.9 4.79 0.1 0.204081632653 0.204081632653 172400 0 172400 172400 172400 306522 306522 0.292322158813 0.292322158813 1 -compress rosa expert rosaplus compress:rosa 2097152 1 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 11.69 0 11.69 11.69 11.69 11.52 0.15 0.171086398631 0.171086398631 337400 0 337400 337400 337400 608681 608681 0.290241718292 0.290241718292 1 -compress rosa expert rosaplus compress:rosa 4194304 1 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 28.05 0 28.05 28.05 28.05 27.74 0.28 0.142602495544 0.142602495544 669264 0 669264 669264 669264 1199345 1199345 0.285946130753 0.285946130753 1 -compress rosa expert rosaplus compress:rosa 10000000 1 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 82.26 0 82.26 82.26 82.26 81.48 0.69 0.115934149818 0.115934149818 1556712 0 1556712 1556712 1556712 2752778 2752778 0.2752778 0.2752778 1 -compress rwkv expert rwkv compress:rwkv 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.06 0 0.06 0.06 0.06 0.06 0 0.0651041666667 0.0651041666667 7692 0 7692 7692 7692 3714 3714 0.90673828125 0.90673828125 1 -compress rwkv expert rwkv compress:rwkv 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.24 0 0.24 0.24 0.24 0.24 0 0.0651041666667 0.0651041666667 7612 0 7612 7612 7612 12000 12000 0.732421875 0.732421875 1 -compress rwkv expert rwkv compress:rwkv 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.96 0 0.96 0.96 0.96 0.96 0 0.0651041666667 0.0651041666667 7836 0 7836 7836 7836 35340 35340 0.539245605469 0.539245605469 1 -compress rwkv expert rwkv compress:rwkv 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 3.94 0 3.94 3.94 3.94 3.93 0 0.0634517766497 0.0634517766497 7736 0 7736 7736 7736 137010 137010 0.522651672363 0.522651672363 1 -compress rwkv expert rwkv compress:rwkv 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 16.15 0 16.15 16.15 16.15 16.05 0.08 0.061919504644 0.061919504644 9268 0 9268 9268 9268 474275 474275 0.452303886414 0.452303886414 1 -compress rwkv expert rwkv compress:rwkv 2097152 1 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 33.03 0 33.03 33.03 33.03 32.43 0.56 0.0605510142295 0.0605510142295 11044 0 11044 11044 11044 917086 917086 0.437300682068 0.437300682068 1 -compress rwkv expert rwkv compress:rwkv 4194304 1 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 67.81 0 67.81 67.81 67.81 66.05 1.66 0.0589883498009 0.0589883498009 14756 0 14756 14756 14756 1748887 1748887 0.416967153549 0.416967153549 1 -compress rwkv expert rwkv compress:rwkv 10000000 1 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 159.41 0 159.41 159.41 159.41 157.3 1.95 0.0598252503862 0.0598252503862 24808 0 24808 24808 24808 3968645 3968645 0.3968645 0.3968645 1 -decompress ctw expert ctw decompress:ctw 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.05 0 0.05 0.05 0.05 0.05 0 0.078125 0.078125 11788 0 11788 11788 11788 1332 1332 0.3251953125 0.3251953125 1 -decompress ctw expert ctw decompress:ctw 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.23 0 0.23 0.23 0.23 0.22 0 0.0679347826087 0.0679347826087 34332 0 34332 34332 34332 6579 6579 0.401550292969 0.401550292969 1 -decompress ctw expert ctw decompress:ctw 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 1.01 0 1.01 1.01 1.01 0.98 0.02 0.0618811881188 0.0618811881188 76292 0 76292 76292 76292 22728 22728 0.346801757812 0.346801757812 1 -decompress ctw expert ctw decompress:ctw 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 4.24 0 4.24 4.24 4.24 4.17 0.06 0.0589622641509 0.0589622641509 185824 0 185824 185824 185824 80004 80004 0.305191040039 0.305191040039 1 -decompress ctw expert ctw decompress:ctw 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 18.15 0 18.15 18.15 18.15 17.91 0.22 0.0550964187328 0.0550964187328 471276 0 471276 471276 471276 301713 301713 0.287735939026 0.287735939026 1 -decompress ctw expert ctw decompress:ctw 2097152 1 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 37.7 0 37.7 37.7 37.7 37.37 0.29 0.053050397878 0.053050397878 775708 0 775708 775708 775708 597003 597003 0.284673213959 0.284673213959 1 -decompress ctw expert ctw decompress:ctw 4194304 1 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 78.64 0 78.64 78.64 78.64 78.01 0.54 0.0508646998983 0.0508646998983 1292724 0 1292724 1292724 1292724 1177962 1177962 0.280848026276 0.280848026276 1 -decompress ctw expert ctw decompress:ctw 10000000 1 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 198.85 0 198.85 198.85 198.85 197.55 1.1 0.0479594828467 0.0479594828467 2490528 0 2490528 2490528 2490528 2746861 2746861 0.2746861 0.2746861 1 -decompress match expert match decompress:match 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0 0 0 0 0 0 0 5516 0 5516 5516 5516 2874 2874 0.70166015625 0.70166015625 1 -decompress match expert match decompress:match 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.01 0 0.01 0.01 0.01 0.01 0 1.5625 1.5625 5572 0 5572 5572 5572 13506 13506 0.824340820312 0.824340820312 1 -decompress match expert match decompress:match 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.03 0 0.03 0.03 0.03 0.03 0 2.08333333333 2.08333333333 6248 0 6248 6248 6248 53748 53748 0.820129394531 0.820129394531 1 -decompress match expert match decompress:match 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 0.14 0 0.14 0.14 0.14 0.14 0 1.78571428571 1.78571428571 7860 0 7860 7860 7860 206203 206203 0.786602020264 0.786602020264 1 -decompress match expert match decompress:match 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 0.57 0 0.57 0.57 0.57 0.56 0 1.75438596491 1.75438596491 11528 0 11528 11528 11528 817505 817505 0.779633522034 0.779633522034 1 -decompress match expert match decompress:match 2097152 1 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 1.13 0 1.13 1.13 1.13 1.11 0 1.76991150442 1.76991150442 19996 0 19996 19996 19996 1644756 1644756 0.784280776978 0.784280776978 1 -decompress match expert match decompress:match 4194304 1 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 2.27 0 2.27 2.27 2.27 2.26 0 1.76211453744 1.76211453744 23828 0 23828 23828 23828 3292751 3292751 0.785053014755 0.785053014755 1 -decompress match expert match decompress:match 10000000 1 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 5.52 0 5.52 5.52 5.52 5.49 0.02 1.72767086305 1.72767086305 46012 0 46012 46012 46012 7859324 7859324 0.7859324 0.7859324 1 -decompress neural_mixture mixture neural-mixture decompress:neural_mixture 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.16 0 0.16 0.16 0.16 0.14 0 0.0244140625 0.0244140625 17692 0 17692 17692 17692 1001 1001 0.244384765625 0.244384765625 1 -decompress neural_mixture mixture neural-mixture decompress:neural_mixture 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.67 0 0.67 0.67 0.67 0.65 0.02 0.0233208955224 0.0233208955224 53200 0 53200 53200 53200 5533 5533 0.337707519531 0.337707519531 1 -decompress neural_mixture mixture neural-mixture decompress:neural_mixture 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 2.9 0 2.9 2.9 2.9 2.85 0.04 0.0215517241379 0.0215517241379 140252 0 140252 140252 140252 19344 19344 0.295166015625 0.295166015625 1 -decompress neural_mixture mixture neural-mixture decompress:neural_mixture 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 12.63 0 12.63 12.63 12.63 12.42 0.18 0.0197941409343 0.0197941409343 427948 0 427948 427948 427948 67454 67454 0.257316589355 0.257316589355 1 -decompress neural_mixture mixture neural-mixture decompress:neural_mixture 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 56.04 0 56.04 56.04 56.04 55.48 0.49 0.0178443968594 0.0178443968594 1108996 0 1108996 1108996 1108996 254251 254251 0.242472648621 0.242472648621 1 -decompress neural_mixture mixture neural-mixture decompress:neural_mixture 2097152 1 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 117.94 0 117.94 117.94 117.94 117.01 0.81 0.0169577751399 0.0169577751399 1717596 0 1717596 1717596 1717596 503519 503519 0.240096569061 0.240096569061 1 -decompress neural_mixture mixture neural-mixture decompress:neural_mixture 4194304 1 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 249 0 249 249 249 247.56 1.16 0.0160642570281 0.0160642570281 2628016 0 2628016 2628016 2628016 988355 988355 0.235642194748 0.235642194748 1 -decompress neural_mixture mixture neural-mixture decompress:neural_mixture 10000000 1 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 637.98 0 637.98 637.98 637.98 635.02 2.25 0.014948341898 0.014948341898 4748240 0 4748240 4748240 4748240 2270248 2270248 0.2270248 0.2270248 1 -decompress ppmd expert ppmd decompress:ppmd 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.01 0 0.01 0.01 0.01 0.01 0 0.390625 0.390625 7224 0 7224 7224 7224 1058 1058 0.25830078125 0.25830078125 1 -decompress ppmd expert ppmd decompress:ppmd 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.04 0 0.04 0.04 0.04 0.04 0 0.390625 0.390625 16076 0 16076 16076 16076 6267 6267 0.382507324219 0.382507324219 1 -decompress ppmd expert ppmd decompress:ppmd 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.22 0 0.22 0.22 0.22 0.2 0.01 0.284090909091 0.284090909091 44956 0 44956 44956 44956 23186 23186 0.353790283203 0.353790283203 1 -decompress ppmd expert ppmd decompress:ppmd 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 1.04 0 1.04 1.04 1.04 0.96 0.07 0.240384615385 0.240384615385 143808 0 143808 143808 143808 83269 83269 0.317646026611 0.317646026611 1 -decompress ppmd expert ppmd decompress:ppmd 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 4.33 0 4.33 4.33 4.33 4.11 0.21 0.230946882217 0.230946882217 399312 0 399312 399312 399312 326331 326331 0.311213493347 0.311213493347 1 -decompress ppmd expert ppmd decompress:ppmd 2097152 1 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 8.55 0 8.55 8.55 8.55 8.31 0.22 0.233918128655 0.233918128655 457456 0 457456 457456 457456 661953 661953 0.315643787384 0.315643787384 1 -decompress ppmd expert ppmd decompress:ppmd 4194304 1 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 17.01 0 17.01 17.01 17.01 16.81 0.18 0.235155790711 0.235155790711 458160 0 458160 458160 458160 1329257 1329257 0.316919565201 0.316919565201 1 -decompress ppmd expert ppmd decompress:ppmd 10000000 1 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 40.57 0 40.57 40.57 40.57 40.28 0.25 0.235068848017 0.235068848017 559224 0 559224 559224 559224 3154465 3154465 0.3154465 0.3154465 1 -decompress rosa expert rosaplus decompress:rosa 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0 0 0 0 0 0 0 5864 0 5864 5864 5864 1127 1127 0.275146484375 0.275146484375 1 -decompress rosa expert rosaplus decompress:rosa 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.03 0 0.03 0.03 0.03 0.03 0 0.520833333333 0.520833333333 8524 0 8524 8524 8524 6359 6359 0.388122558594 0.388122558594 1 -decompress rosa expert rosaplus decompress:rosa 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.16 0 0.16 0.16 0.16 0.16 0 0.390625 0.390625 19224 0 19224 19224 19224 22843 22843 0.348556518555 0.348556518555 1 -decompress rosa expert rosaplus decompress:rosa 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 0.88 0 0.88 0.88 0.88 0.86 0.01 0.284090909091 0.284090909091 61956 0 61956 61956 61956 80590 80590 0.307426452637 0.307426452637 1 -decompress rosa expert rosaplus decompress:rosa 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 4.98 0 4.98 4.98 4.98 4.93 0.04 0.200803212851 0.200803212851 172624 0 172624 172624 172624 306522 306522 0.292322158813 0.292322158813 1 -decompress rosa expert rosaplus decompress:rosa 2097152 1 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 11.75 0 11.75 11.75 11.75 11.62 0.11 0.170212765957 0.170212765957 338084 0 338084 338084 338084 608681 608681 0.290241718292 0.290241718292 1 -decompress rosa expert rosaplus decompress:rosa 4194304 1 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 28.13 0 28.13 28.13 28.13 27.8 0.3 0.142196942766 0.142196942766 658228 0 658228 658228 658228 1199345 1199345 0.285946130753 0.285946130753 1 -decompress rosa expert rosaplus decompress:rosa 10000000 1 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 82.75 0 82.75 82.75 82.75 81.89 0.77 0.115247651529 0.115247651529 1555068 0 1555068 1555068 1555068 2752778 2752778 0.2752778 0.2752778 1 -decompress rwkv expert rwkv decompress:rwkv 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.06 0 0.06 0.06 0.06 0.06 0 0.0651041666667 0.0651041666667 7364 0 7364 7364 7364 3714 3714 0.90673828125 0.90673828125 1 -decompress rwkv expert rwkv decompress:rwkv 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.24 0 0.24 0.24 0.24 0.24 0 0.0651041666667 0.0651041666667 7652 0 7652 7652 7652 12000 12000 0.732421875 0.732421875 1 -decompress rwkv expert rwkv decompress:rwkv 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.95 0 0.95 0.95 0.95 0.95 0 0.0657894736842 0.0657894736842 7820 0 7820 7820 7820 35340 35340 0.539245605469 0.539245605469 1 -decompress rwkv expert rwkv decompress:rwkv 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 3.96 0 3.96 3.96 3.96 3.95 0 0.0631313131313 0.0631313131313 7836 0 7836 7836 7836 137010 137010 0.522651672363 0.522651672363 1 -decompress rwkv expert rwkv decompress:rwkv 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 15.86 0 15.86 15.86 15.86 15.84 0 0.063051702396 0.063051702396 8628 0 8628 8628 8628 474275 474275 0.452303886414 0.452303886414 1 -decompress rwkv expert rwkv decompress:rwkv 2097152 1 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 32.41 0 32.41 32.41 32.41 32.38 0 0.0617093489664 0.0617093489664 10224 0 10224 10224 10224 917086 917086 0.437300682068 0.437300682068 1 -decompress rwkv expert rwkv decompress:rwkv 4194304 1 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 65.59 0 65.59 65.59 65.59 65.53 0 0.0609849062357 0.0609849062357 13284 0 13284 13284 13284 1748887 1748887 0.416967153549 0.416967153549 1 -decompress rwkv expert rwkv decompress:rwkv 10000000 1 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 159.41 0 159.41 159.41 159.41 159.27 0 0.0598252503862 0.0598252503862 20828 0 20828 20828 20828 3968645 3968645 0.3968645 0.3968645 1 +operation subject subject_kind expert_kind series size_bytes repeats cpu compression_backend input_sha256 real_seconds_mean real_seconds_stdev real_seconds_median real_seconds_min real_seconds_max user_seconds_mean sys_seconds_mean throughput_mib_s_mean throughput_mib_s_median rss_kib_mean rss_kib_stdev rss_kib_median rss_kib_min rss_kib_max archive_bytes_mean archive_bytes_median archive_ratio_mean archive_ratio_median entropy_bpb_mean entropy_bpb_median verified_all +h ctw expert ctw h:ctw 4096 1 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.04 0 0.04 0.04 0.04 0.04 0 0.09765625 0.09765625 11824 0 11824 11824 11824 2.56549624542 2.56549624542 1 +h ctw expert ctw h:ctw 16384 1 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.18 0 0.18 0.18 0.18 0.17 0 0.0868055555556 0.0868055555556 34480 0 34480 34480 34480 3.20340951209 3.20340951209 1 +h ctw expert ctw h:ctw 65536 1 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.77 0 0.77 0.77 0.77 0.75 0.02 0.0811688311688 0.0811688311688 76356 0 76356 76356 76356 2.77213895634 2.77213895634 1 +h ctw expert ctw h:ctw 262144 1 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 3.28 0 3.28 3.28 3.28 3.22 0.06 0.0762195121951 0.0762195121951 185300 0 185300 185300 185300 2.44096714152 2.44096714152 1 +h ctw expert ctw h:ctw 1048576 1 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 14.46 0 14.46 14.46 14.46 14.24 0.21 0.0691562932227 0.0691562932227 470788 0 470788 470788 470788 2.30174274755 2.30174274755 1 +h ctw expert ctw h:ctw 2097152 1 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 30.58 0 30.58 30.58 30.58 30.24 0.3 0.0654022236756 0.0654022236756 774868 0 774868 774868 774868 2.27731545661 2.27731545661 1 +h ctw expert ctw h:ctw 4194304 1 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 64.74 0 64.74 64.74 64.74 64.19 0.48 0.0617856039543 0.0617856039543 1291640 0 1291640 1291640 1291640 2.24674930722 2.24674930722 1 +h ctw expert ctw h:ctw 10000000 1 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 166.38 0 166.38 166.38 166.38 165.17 1.03 0.0573190477465 0.0573190477465 2487416 0 2487416 2487416 2487416 2.19747398481 2.19747398481 1 +h match expert match h:match 4096 1 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0 0 0 0 0 0 0 5688 0 5688 5688 5688 5.57734993252 5.57734993252 1 +h match expert match h:match 16384 1 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0 0 0 0 0 0 0 5612 0 5612 5612 5612 6.58586286407 6.58586286407 1 +h match expert match h:match 65536 1 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.01 0 0.01 0.01 0.01 0 0 6.25 6.25 6632 0 6632 6632 6632 6.5588255305 6.5588255305 1 +h match expert match h:match 262144 1 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 0.03 0 0.03 0.03 0.03 0.03 0 8.33333333333 8.33333333333 8012 0 8012 8012 8012 6.29225944669 6.29225944669 1 +h match expert match h:match 1048576 1 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 0.13 0 0.13 0.13 0.13 0.13 0 7.69230769231 7.69230769231 11452 0 11452 11452 11452 6.23692635511 6.23692635511 1 +h match expert match h:match 2097152 1 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 0.28 0 0.28 0.28 0.28 0.28 0 7.14285714286 7.14285714286 19132 0 19132 19132 19132 6.2741772649 6.2741772649 1 +h match expert match h:match 4194304 1 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 0.56 0 0.56 0.56 0.56 0.55 0 7.14285714286 7.14285714286 21268 0 21268 21268 21268 6.28038800899 6.28038800899 1 +h match expert match h:match 10000000 1 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 1.35 0 1.35 1.35 1.35 1.34 0.01 7.0642541956 7.0642541956 40264 0 40264 40264 40264 6.28744436593 6.28744436593 1 +h neural_mixture mixture neural-mixture h:neural_mixture 4096 1 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.19 0 0.19 0.19 0.19 0.19 0 0.0205592105263 0.0205592105263 17912 0 17912 17912 17912 1.91857366907 1.91857366907 1 +h neural_mixture mixture neural-mixture h:neural_mixture 16384 1 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.82 0 0.82 0.82 0.82 0.8 0.01 0.0190548780488 0.0190548780488 52724 0 52724 52724 52724 2.69245779825 2.69245779825 1 +h neural_mixture mixture neural-mixture h:neural_mixture 65536 1 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 3.52 0 3.52 3.52 3.52 3.47 0.03 0.0177556818182 0.0177556818182 139048 0 139048 139048 139048 2.35902117116 2.35902117116 1 +h neural_mixture mixture neural-mixture h:neural_mixture 262144 1 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 15.02 0 15.02 15.02 15.02 14.82 0.17 0.0166444740346 0.0166444740346 430348 0 430348 430348 430348 2.05795611601 2.05795611601 1 +h neural_mixture mixture neural-mixture h:neural_mixture 1048576 1 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 65.26 0 65.26 65.26 65.26 64.66 0.52 0.0153233220962 0.0153233220962 1108724 0 1108724 1108724 1108724 1.93963669332 1.93963669332 1 +h neural_mixture mixture neural-mixture h:neural_mixture 2097152 1 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 135.64 0 135.64 135.64 135.64 134.76 0.74 0.014744913005 0.014744913005 1719704 0 1719704 1719704 1719704 1.92069854644 1.92069854644 1 +h neural_mixture mixture neural-mixture h:neural_mixture 4194304 1 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 283.39 0 283.39 283.39 283.39 282.01 1.07 0.014114824094 0.014114824094 2623920 0 2623920 2623920 2623920 1.88509670528 1.88509670528 1 +h neural_mixture mixture neural-mixture h:neural_mixture 10000000 1 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 715.33 0 715.33 715.33 715.33 712.24 2.27 0.0133319491201 0.0133319491201 4760388 0 4760388 4760388 4760388 1.81617547371 1.81617547371 1 +h ppmd expert ppmd h:ppmd 4096 1 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.01 0 0.01 0.01 0.01 0.01 0 0.390625 0.390625 7352 0 7352 7352 7352 2.02916276221 2.02916276221 1 +h ppmd expert ppmd h:ppmd 16384 1 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.03 0 0.03 0.03 0.03 0.03 0 0.520833333333 0.520833333333 16100 0 16100 16100 16100 3.05111741489 3.05111741489 1 +h ppmd expert ppmd h:ppmd 65536 1 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.19 0 0.19 0.19 0.19 0.18 0 0.328947368421 0.328947368421 45260 0 45260 45260 45260 2.82800309907 2.82800309907 1 +h ppmd expert ppmd h:ppmd 262144 1 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 0.93 0 0.93 0.93 0.93 0.87 0.06 0.268817204301 0.268817204301 144212 0 144212 144212 144212 2.54059561824 2.54059561824 1 +h ppmd expert ppmd h:ppmd 1048576 1 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 3.93 0 3.93 3.93 3.93 3.72 0.19 0.254452926209 0.254452926209 398696 0 398696 398696 398696 2.48956475398 2.48956475398 1 +h ppmd expert ppmd h:ppmd 2097152 1 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 7.61 0 7.61 7.61 7.61 7.39 0.21 0.262812089356 0.262812089356 457100 0 457100 457100 457100 2.52508057302 2.52508057302 1 +h ppmd expert ppmd h:ppmd 4194304 1 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 15.19 0 15.19 15.19 15.19 14.94 0.24 0.263331138907 0.263331138907 459116 0 459116 459116 459116 2.5353209835 2.5353209835 1 +h ppmd expert ppmd h:ppmd 10000000 1 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 36.63 0 36.63 36.63 36.63 36.27 0.32 0.260353348732 0.260353348732 557108 0 557108 557108 557108 2.52355814767 2.52355814767 1 +h rosa expert rosaplus h:rosa 4096 1 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.02 0 0.02 0.02 0.02 0.02 0 0.1953125 0.1953125 5924 0 5924 5924 5924 2.11172124286 2.11172124286 1 +h rosa expert rosaplus h:rosa 16384 1 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.12 0 0.12 0.12 0.12 0.11 0.01 0.130208333333 0.130208333333 8224 0 8224 8224 8224 3.48630622986 3.48630622986 1 +h rosa expert rosaplus h:rosa 65536 1 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.62 0 0.62 0.62 0.62 0.6 0.02 0.100806451613 0.100806451613 16676 0 16676 16676 16676 3.19056696201 3.19056696201 1 +h rosa expert rosaplus h:rosa 262144 1 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 3.6 0 3.6 3.6 3.6 3.49 0.1 0.0694444444444 0.0694444444444 47052 0 47052 47052 47052 2.90567254487 2.90567254487 1 +h rosa expert rosaplus h:rosa 1048576 1 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 21.23 0 21.23 21.23 21.23 20.34 0.86 0.0471031559114 0.0471031559114 203892 0 203892 203892 203892 2.80123027825 2.80123027825 1 +h rosa expert rosaplus h:rosa 2097152 1 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 50.81 0 50.81 50.81 50.81 48.72 2.02 0.03936233025 0.03936233025 337584 0 337584 337584 337584 2.70437138702 2.70437138702 1 +h rosa expert rosaplus h:rosa 4194304 1 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 121.27 0 121.27 121.27 121.27 115.95 5.18 0.0329842500206 0.0329842500206 643432 0 643432 643432 643432 2.62099230176 2.62099230176 1 +h rosa expert rosaplus h:rosa 10000000 1 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 356.22 0 356.22 356.22 356.22 342.7 13.11 0.0267720598621 0.0267720598621 1551456 0 1551456 1551456 1551456 2.48481897791 2.48481897791 1 +h rwkv expert rwkv h:rwkv 4096 1 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.06 0 0.06 0.06 0.06 0.06 0 0.0651041666667 0.0651041666667 8000 0 8000 8000 8000 7.21701437947 7.21701437947 1 +h rwkv expert rwkv h:rwkv 16384 1 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.24 0 0.24 0.24 0.24 0.24 0 0.0651041666667 0.0651041666667 8172 0 8172 8172 8172 5.85021072042 5.85021072042 1 +h rwkv expert rwkv h:rwkv 65536 1 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.94 0 0.94 0.94 0.94 0.94 0 0.0664893617021 0.0664893617021 8336 0 8336 8336 8336 4.31173851819 4.31173851819 1 +h rwkv expert rwkv h:rwkv 262144 1 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 3.69 0 3.69 3.69 3.69 3.69 0 0.0677506775068 0.0677506775068 8320 0 8320 8320 8320 4.18065126521 4.18065126521 1 +h rwkv expert rwkv h:rwkv 1048576 1 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 15.57 0 15.57 15.57 15.57 15.55 0 0.0642260757868 0.0642260757868 8796 0 8796 8796 8796 3.61829048151 3.61829048151 1 +h rwkv expert rwkv h:rwkv 2097152 1 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 31.61 0 31.61 31.61 31.61 31.58 0 0.0632711167352 0.0632711167352 9912 0 9912 9912 9912 3.49833649966 3.49833649966 1 +h rwkv expert rwkv h:rwkv 4194304 1 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 63.22 0 63.22 63.22 63.22 63.18 0 0.0632711167352 0.0632711167352 11888 0 11888 11888 11888 3.33570098711 3.33570098711 1 +h rwkv expert rwkv h:rwkv 10000000 1 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 156.53 0 156.53 156.53 156.53 156.4 0 0.0609259768994 0.0609259768994 17764 0 17764 17764 17764 3.17490278117 3.17490278117 1 +compress ctw expert ctw compress:ctw 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.05 0 0.05 0.05 0.05 0.04 0 0.078125 0.078125 11616 0 11616 11616 11616 1332 1332 0.3251953125 0.3251953125 1 +compress ctw expert ctw compress:ctw 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.23 0 0.23 0.23 0.23 0.22 0 0.0679347826087 0.0679347826087 34400 0 34400 34400 34400 6579 6579 0.401550292969 0.401550292969 1 +compress ctw expert ctw compress:ctw 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 1 0 1 1 1 0.97 0.03 0.0625 0.0625 76372 0 76372 76372 76372 22728 22728 0.346801757812 0.346801757812 1 +compress ctw expert ctw compress:ctw 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 4.21 0 4.21 4.21 4.21 4.13 0.07 0.0593824228029 0.0593824228029 185720 0 185720 185720 185720 80004 80004 0.305191040039 0.305191040039 1 +compress ctw expert ctw compress:ctw 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 18.06 0 18.06 18.06 18.06 17.84 0.19 0.0553709856035 0.0553709856035 471292 0 471292 471292 471292 301713 301713 0.287735939026 0.287735939026 1 +compress ctw expert ctw compress:ctw 2097152 1 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 37.54 0 37.54 37.54 37.54 37.23 0.27 0.0532765050613 0.0532765050613 776116 0 776116 776116 776116 597003 597003 0.284673213959 0.284673213959 1 +compress ctw expert ctw compress:ctw 4194304 1 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 78.4 0 78.4 78.4 78.4 77.77 0.55 0.0510204081633 0.0510204081633 1294124 0 1294124 1294124 1294124 1177962 1177962 0.280848026276 0.280848026276 1 +compress ctw expert ctw compress:ctw 10000000 1 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 198.36 0 198.36 198.36 198.36 196.98 1.16 0.0480779550517 0.0480779550517 2493308 0 2493308 2493308 2493308 2746861 2746861 0.2746861 0.2746861 1 +compress match expert match compress:match 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0 0 0 0 0 0 0 5528 0 5528 5528 5528 2874 2874 0.70166015625 0.70166015625 1 +compress match expert match compress:match 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.01 0 0.01 0.01 0.01 0.01 0 1.5625 1.5625 5604 0 5604 5604 5604 13506 13506 0.824340820312 0.824340820312 1 +compress match expert match compress:match 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.03 0 0.03 0.03 0.03 0.03 0 2.08333333333 2.08333333333 6332 0 6332 6332 6332 53748 53748 0.820129394531 0.820129394531 1 +compress match expert match compress:match 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 0.14 0 0.14 0.14 0.14 0.13 0 1.78571428571 1.78571428571 7984 0 7984 7984 7984 206203 206203 0.786602020264 0.786602020264 1 +compress match expert match compress:match 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 0.53 0 0.53 0.53 0.53 0.53 0 1.88679245283 1.88679245283 11972 0 11972 11972 11972 817505 817505 0.779633522034 0.779633522034 1 +compress match expert match compress:match 2097152 1 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 1.08 0 1.08 1.08 1.08 1.07 0 1.85185185185 1.85185185185 22024 0 22024 22024 22024 1644756 1644756 0.784280776978 0.784280776978 1 +compress match expert match compress:match 4194304 1 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 2.15 0 2.15 2.15 2.15 2.13 0 1.86046511628 1.86046511628 28812 0 28812 28812 28812 3292751 3292751 0.785053014755 0.785053014755 1 +compress match expert match compress:match 10000000 1 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 5.18 0 5.18 5.18 5.18 5.15 0.02 1.84107010889 1.84107010889 55408 0 55408 55408 55408 7859324 7859324 0.7859324 0.7859324 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.16 0 0.16 0.16 0.16 0.16 0 0.0244140625 0.0244140625 17464 0 17464 17464 17464 1001 1001 0.244384765625 0.244384765625 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.67 0 0.67 0.67 0.67 0.65 0.01 0.0233208955224 0.0233208955224 52280 0 52280 52280 52280 5533 5533 0.337707519531 0.337707519531 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 2.87 0 2.87 2.87 2.87 2.8 0.06 0.0217770034843 0.0217770034843 139660 0 139660 139660 139660 19344 19344 0.295166015625 0.295166015625 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 12.6 0 12.6 12.6 12.6 12.41 0.17 0.0198412698413 0.0198412698413 428292 0 428292 428292 428292 67454 67454 0.257316589355 0.257316589355 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 56.06 0 56.06 56.06 56.06 55.47 0.53 0.0178380306814 0.0178380306814 1103428 0 1103428 1103428 1103428 254251 254251 0.242472648621 0.242472648621 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 2097152 1 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 117.74 0 117.74 117.74 117.74 116.91 0.69 0.0169865806013 0.0169865806013 1734756 0 1734756 1734756 1734756 503519 503519 0.240096569061 0.240096569061 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 4194304 1 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 248.57 0 248.57 248.57 248.57 247.17 1.11 0.016092046506 0.016092046506 2664248 0 2664248 2664248 2664248 988355 988355 0.235642194748 0.235642194748 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 10000000 1 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 637.69 0 637.69 637.69 637.69 634.67 2.31 0.0149551399019 0.0149551399019 4777992 0 4777992 4777992 4777992 2270248 2270248 0.2270248 0.2270248 1 +compress ppmd expert ppmd compress:ppmd 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.01 0 0.01 0.01 0.01 0 0 0.390625 0.390625 7036 0 7036 7036 7036 1058 1058 0.25830078125 0.25830078125 1 +compress ppmd expert ppmd compress:ppmd 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.04 0 0.04 0.04 0.04 0.03 0 0.390625 0.390625 16076 0 16076 16076 16076 6267 6267 0.382507324219 0.382507324219 1 +compress ppmd expert ppmd compress:ppmd 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.22 0 0.22 0.22 0.22 0.2 0.01 0.284090909091 0.284090909091 44976 0 44976 44976 44976 23186 23186 0.353790283203 0.353790283203 1 +compress ppmd expert ppmd compress:ppmd 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 1.07 0 1.07 1.07 1.07 1 0.07 0.233644859813 0.233644859813 143988 0 143988 143988 143988 83269 83269 0.317646026611 0.317646026611 1 +compress ppmd expert ppmd compress:ppmd 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 4.29 0 4.29 4.29 4.29 4.11 0.17 0.2331002331 0.2331002331 399344 0 399344 399344 399344 326331 326331 0.311213493347 0.311213493347 1 +compress ppmd expert ppmd compress:ppmd 2097152 1 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 8.35 0 8.35 8.35 8.35 8.15 0.18 0.239520958084 0.239520958084 457680 0 457680 457680 457680 661953 661953 0.315643787384 0.315643787384 1 +compress ppmd expert ppmd compress:ppmd 4194304 1 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 16.55 0 16.55 16.55 16.55 16.34 0.19 0.2416918429 0.2416918429 459764 0 459764 459764 459764 1329257 1329257 0.316919565201 0.316919565201 1 +compress ppmd expert ppmd compress:ppmd 10000000 1 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 40.16 0 40.16 40.16 40.16 39.86 0.25 0.237468704284 0.237468704284 568928 0 568928 568928 568928 3154465 3154465 0.3154465 0.3154465 1 +compress rosa expert rosaplus compress:rosa 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0 0 0 0 0 0 0 5768 0 5768 5768 5768 1127 1127 0.275146484375 0.275146484375 1 +compress rosa expert rosaplus compress:rosa 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.03 0 0.03 0.03 0.03 0.03 0 0.520833333333 0.520833333333 8544 0 8544 8544 8544 6359 6359 0.388122558594 0.388122558594 1 +compress rosa expert rosaplus compress:rosa 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.15 0 0.15 0.15 0.15 0.14 0 0.416666666667 0.416666666667 19472 0 19472 19472 19472 22843 22843 0.348556518555 0.348556518555 1 +compress rosa expert rosaplus compress:rosa 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 0.88 0 0.88 0.88 0.88 0.85 0.01 0.284090909091 0.284090909091 61972 0 61972 61972 61972 80590 80590 0.307426452637 0.307426452637 1 +compress rosa expert rosaplus compress:rosa 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 4.9 0 4.9 4.9 4.9 4.79 0.1 0.204081632653 0.204081632653 172400 0 172400 172400 172400 306522 306522 0.292322158813 0.292322158813 1 +compress rosa expert rosaplus compress:rosa 2097152 1 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 11.69 0 11.69 11.69 11.69 11.52 0.15 0.171086398631 0.171086398631 337400 0 337400 337400 337400 608681 608681 0.290241718292 0.290241718292 1 +compress rosa expert rosaplus compress:rosa 4194304 1 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 28.05 0 28.05 28.05 28.05 27.74 0.28 0.142602495544 0.142602495544 669264 0 669264 669264 669264 1199345 1199345 0.285946130753 0.285946130753 1 +compress rosa expert rosaplus compress:rosa 10000000 1 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 82.26 0 82.26 82.26 82.26 81.48 0.69 0.115934149818 0.115934149818 1556712 0 1556712 1556712 1556712 2752778 2752778 0.2752778 0.2752778 1 +compress rwkv expert rwkv compress:rwkv 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.06 0 0.06 0.06 0.06 0.06 0 0.0651041666667 0.0651041666667 7692 0 7692 7692 7692 3714 3714 0.90673828125 0.90673828125 1 +compress rwkv expert rwkv compress:rwkv 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.24 0 0.24 0.24 0.24 0.24 0 0.0651041666667 0.0651041666667 7612 0 7612 7612 7612 12000 12000 0.732421875 0.732421875 1 +compress rwkv expert rwkv compress:rwkv 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.96 0 0.96 0.96 0.96 0.96 0 0.0651041666667 0.0651041666667 7836 0 7836 7836 7836 35340 35340 0.539245605469 0.539245605469 1 +compress rwkv expert rwkv compress:rwkv 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 3.94 0 3.94 3.94 3.94 3.93 0 0.0634517766497 0.0634517766497 7736 0 7736 7736 7736 137010 137010 0.522651672363 0.522651672363 1 +compress rwkv expert rwkv compress:rwkv 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 16.15 0 16.15 16.15 16.15 16.05 0.08 0.061919504644 0.061919504644 9268 0 9268 9268 9268 474275 474275 0.452303886414 0.452303886414 1 +compress rwkv expert rwkv compress:rwkv 2097152 1 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 33.03 0 33.03 33.03 33.03 32.43 0.56 0.0605510142295 0.0605510142295 11044 0 11044 11044 11044 917086 917086 0.437300682068 0.437300682068 1 +compress rwkv expert rwkv compress:rwkv 4194304 1 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 67.81 0 67.81 67.81 67.81 66.05 1.66 0.0589883498009 0.0589883498009 14756 0 14756 14756 14756 1748887 1748887 0.416967153549 0.416967153549 1 +compress rwkv expert rwkv compress:rwkv 10000000 1 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 159.41 0 159.41 159.41 159.41 157.3 1.95 0.0598252503862 0.0598252503862 24808 0 24808 24808 24808 3968645 3968645 0.3968645 0.3968645 1 +decompress ctw expert ctw decompress:ctw 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.05 0 0.05 0.05 0.05 0.05 0 0.078125 0.078125 11788 0 11788 11788 11788 1332 1332 0.3251953125 0.3251953125 1 +decompress ctw expert ctw decompress:ctw 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.23 0 0.23 0.23 0.23 0.22 0 0.0679347826087 0.0679347826087 34332 0 34332 34332 34332 6579 6579 0.401550292969 0.401550292969 1 +decompress ctw expert ctw decompress:ctw 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 1.01 0 1.01 1.01 1.01 0.98 0.02 0.0618811881188 0.0618811881188 76292 0 76292 76292 76292 22728 22728 0.346801757812 0.346801757812 1 +decompress ctw expert ctw decompress:ctw 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 4.24 0 4.24 4.24 4.24 4.17 0.06 0.0589622641509 0.0589622641509 185824 0 185824 185824 185824 80004 80004 0.305191040039 0.305191040039 1 +decompress ctw expert ctw decompress:ctw 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 18.15 0 18.15 18.15 18.15 17.91 0.22 0.0550964187328 0.0550964187328 471276 0 471276 471276 471276 301713 301713 0.287735939026 0.287735939026 1 +decompress ctw expert ctw decompress:ctw 2097152 1 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 37.7 0 37.7 37.7 37.7 37.37 0.29 0.053050397878 0.053050397878 775708 0 775708 775708 775708 597003 597003 0.284673213959 0.284673213959 1 +decompress ctw expert ctw decompress:ctw 4194304 1 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 78.64 0 78.64 78.64 78.64 78.01 0.54 0.0508646998983 0.0508646998983 1292724 0 1292724 1292724 1292724 1177962 1177962 0.280848026276 0.280848026276 1 +decompress ctw expert ctw decompress:ctw 10000000 1 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 198.85 0 198.85 198.85 198.85 197.55 1.1 0.0479594828467 0.0479594828467 2490528 0 2490528 2490528 2490528 2746861 2746861 0.2746861 0.2746861 1 +decompress match expert match decompress:match 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0 0 0 0 0 0 0 5516 0 5516 5516 5516 2874 2874 0.70166015625 0.70166015625 1 +decompress match expert match decompress:match 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.01 0 0.01 0.01 0.01 0.01 0 1.5625 1.5625 5572 0 5572 5572 5572 13506 13506 0.824340820312 0.824340820312 1 +decompress match expert match decompress:match 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.03 0 0.03 0.03 0.03 0.03 0 2.08333333333 2.08333333333 6248 0 6248 6248 6248 53748 53748 0.820129394531 0.820129394531 1 +decompress match expert match decompress:match 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 0.14 0 0.14 0.14 0.14 0.14 0 1.78571428571 1.78571428571 7860 0 7860 7860 7860 206203 206203 0.786602020264 0.786602020264 1 +decompress match expert match decompress:match 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 0.57 0 0.57 0.57 0.57 0.56 0 1.75438596491 1.75438596491 11528 0 11528 11528 11528 817505 817505 0.779633522034 0.779633522034 1 +decompress match expert match decompress:match 2097152 1 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 1.13 0 1.13 1.13 1.13 1.11 0 1.76991150442 1.76991150442 19996 0 19996 19996 19996 1644756 1644756 0.784280776978 0.784280776978 1 +decompress match expert match decompress:match 4194304 1 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 2.27 0 2.27 2.27 2.27 2.26 0 1.76211453744 1.76211453744 23828 0 23828 23828 23828 3292751 3292751 0.785053014755 0.785053014755 1 +decompress match expert match decompress:match 10000000 1 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 5.52 0 5.52 5.52 5.52 5.49 0.02 1.72767086305 1.72767086305 46012 0 46012 46012 46012 7859324 7859324 0.7859324 0.7859324 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.16 0 0.16 0.16 0.16 0.14 0 0.0244140625 0.0244140625 17692 0 17692 17692 17692 1001 1001 0.244384765625 0.244384765625 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.67 0 0.67 0.67 0.67 0.65 0.02 0.0233208955224 0.0233208955224 53200 0 53200 53200 53200 5533 5533 0.337707519531 0.337707519531 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 2.9 0 2.9 2.9 2.9 2.85 0.04 0.0215517241379 0.0215517241379 140252 0 140252 140252 140252 19344 19344 0.295166015625 0.295166015625 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 12.63 0 12.63 12.63 12.63 12.42 0.18 0.0197941409343 0.0197941409343 427948 0 427948 427948 427948 67454 67454 0.257316589355 0.257316589355 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 56.04 0 56.04 56.04 56.04 55.48 0.49 0.0178443968594 0.0178443968594 1108996 0 1108996 1108996 1108996 254251 254251 0.242472648621 0.242472648621 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 2097152 1 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 117.94 0 117.94 117.94 117.94 117.01 0.81 0.0169577751399 0.0169577751399 1717596 0 1717596 1717596 1717596 503519 503519 0.240096569061 0.240096569061 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 4194304 1 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 249 0 249 249 249 247.56 1.16 0.0160642570281 0.0160642570281 2628016 0 2628016 2628016 2628016 988355 988355 0.235642194748 0.235642194748 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 10000000 1 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 637.98 0 637.98 637.98 637.98 635.02 2.25 0.014948341898 0.014948341898 4748240 0 4748240 4748240 4748240 2270248 2270248 0.2270248 0.2270248 1 +decompress ppmd expert ppmd decompress:ppmd 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.01 0 0.01 0.01 0.01 0.01 0 0.390625 0.390625 7224 0 7224 7224 7224 1058 1058 0.25830078125 0.25830078125 1 +decompress ppmd expert ppmd decompress:ppmd 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.04 0 0.04 0.04 0.04 0.04 0 0.390625 0.390625 16076 0 16076 16076 16076 6267 6267 0.382507324219 0.382507324219 1 +decompress ppmd expert ppmd decompress:ppmd 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.22 0 0.22 0.22 0.22 0.2 0.01 0.284090909091 0.284090909091 44956 0 44956 44956 44956 23186 23186 0.353790283203 0.353790283203 1 +decompress ppmd expert ppmd decompress:ppmd 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 1.04 0 1.04 1.04 1.04 0.96 0.07 0.240384615385 0.240384615385 143808 0 143808 143808 143808 83269 83269 0.317646026611 0.317646026611 1 +decompress ppmd expert ppmd decompress:ppmd 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 4.33 0 4.33 4.33 4.33 4.11 0.21 0.230946882217 0.230946882217 399312 0 399312 399312 399312 326331 326331 0.311213493347 0.311213493347 1 +decompress ppmd expert ppmd decompress:ppmd 2097152 1 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 8.55 0 8.55 8.55 8.55 8.31 0.22 0.233918128655 0.233918128655 457456 0 457456 457456 457456 661953 661953 0.315643787384 0.315643787384 1 +decompress ppmd expert ppmd decompress:ppmd 4194304 1 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 17.01 0 17.01 17.01 17.01 16.81 0.18 0.235155790711 0.235155790711 458160 0 458160 458160 458160 1329257 1329257 0.316919565201 0.316919565201 1 +decompress ppmd expert ppmd decompress:ppmd 10000000 1 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 40.57 0 40.57 40.57 40.57 40.28 0.25 0.235068848017 0.235068848017 559224 0 559224 559224 559224 3154465 3154465 0.3154465 0.3154465 1 +decompress rosa expert rosaplus decompress:rosa 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0 0 0 0 0 0 0 5864 0 5864 5864 5864 1127 1127 0.275146484375 0.275146484375 1 +decompress rosa expert rosaplus decompress:rosa 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.03 0 0.03 0.03 0.03 0.03 0 0.520833333333 0.520833333333 8524 0 8524 8524 8524 6359 6359 0.388122558594 0.388122558594 1 +decompress rosa expert rosaplus decompress:rosa 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.16 0 0.16 0.16 0.16 0.16 0 0.390625 0.390625 19224 0 19224 19224 19224 22843 22843 0.348556518555 0.348556518555 1 +decompress rosa expert rosaplus decompress:rosa 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 0.88 0 0.88 0.88 0.88 0.86 0.01 0.284090909091 0.284090909091 61956 0 61956 61956 61956 80590 80590 0.307426452637 0.307426452637 1 +decompress rosa expert rosaplus decompress:rosa 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 4.98 0 4.98 4.98 4.98 4.93 0.04 0.200803212851 0.200803212851 172624 0 172624 172624 172624 306522 306522 0.292322158813 0.292322158813 1 +decompress rosa expert rosaplus decompress:rosa 2097152 1 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 11.75 0 11.75 11.75 11.75 11.62 0.11 0.170212765957 0.170212765957 338084 0 338084 338084 338084 608681 608681 0.290241718292 0.290241718292 1 +decompress rosa expert rosaplus decompress:rosa 4194304 1 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 28.13 0 28.13 28.13 28.13 27.8 0.3 0.142196942766 0.142196942766 658228 0 658228 658228 658228 1199345 1199345 0.285946130753 0.285946130753 1 +decompress rosa expert rosaplus decompress:rosa 10000000 1 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 82.75 0 82.75 82.75 82.75 81.89 0.77 0.115247651529 0.115247651529 1555068 0 1555068 1555068 1555068 2752778 2752778 0.2752778 0.2752778 1 +decompress rwkv expert rwkv decompress:rwkv 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.06 0 0.06 0.06 0.06 0.06 0 0.0651041666667 0.0651041666667 7364 0 7364 7364 7364 3714 3714 0.90673828125 0.90673828125 1 +decompress rwkv expert rwkv decompress:rwkv 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.24 0 0.24 0.24 0.24 0.24 0 0.0651041666667 0.0651041666667 7652 0 7652 7652 7652 12000 12000 0.732421875 0.732421875 1 +decompress rwkv expert rwkv decompress:rwkv 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.95 0 0.95 0.95 0.95 0.95 0 0.0657894736842 0.0657894736842 7820 0 7820 7820 7820 35340 35340 0.539245605469 0.539245605469 1 +decompress rwkv expert rwkv decompress:rwkv 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 3.96 0 3.96 3.96 3.96 3.95 0 0.0631313131313 0.0631313131313 7836 0 7836 7836 7836 137010 137010 0.522651672363 0.522651672363 1 +decompress rwkv expert rwkv decompress:rwkv 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 15.86 0 15.86 15.86 15.86 15.84 0 0.063051702396 0.063051702396 8628 0 8628 8628 8628 474275 474275 0.452303886414 0.452303886414 1 +decompress rwkv expert rwkv decompress:rwkv 2097152 1 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 32.41 0 32.41 32.41 32.41 32.38 0 0.0617093489664 0.0617093489664 10224 0 10224 10224 10224 917086 917086 0.437300682068 0.437300682068 1 +decompress rwkv expert rwkv decompress:rwkv 4194304 1 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 65.59 0 65.59 65.59 65.59 65.53 0 0.0609849062357 0.0609849062357 13284 0 13284 13284 13284 1748887 1748887 0.416967153549 0.416967153549 1 +decompress rwkv expert rwkv decompress:rwkv 10000000 1 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 159.41 0 159.41 159.41 159.41 159.27 0 0.0598252503862 0.0598252503862 20828 0 20828 20828 20828 3968645 3968645 0.3968645 0.3968645 1 diff --git a/benchmarks/current/infotheory-two-json-raw-20260322-120428.tsv b/benchmarks/53f48242/infotheory-two-json-raw-20260322-120428.tsv similarity index 100% rename from benchmarks/current/infotheory-two-json-raw-20260322-120428.tsv rename to benchmarks/53f48242/infotheory-two-json-raw-20260322-120428.tsv diff --git a/benchmarks/current/infotheory-two-json-summary-20260322-120428.tsv b/benchmarks/53f48242/infotheory-two-json-summary-20260322-120428.tsv similarity index 99% rename from benchmarks/current/infotheory-two-json-summary-20260322-120428.tsv rename to benchmarks/53f48242/infotheory-two-json-summary-20260322-120428.tsv index 261c6dac..44a84453 100644 --- a/benchmarks/current/infotheory-two-json-summary-20260322-120428.tsv +++ b/benchmarks/53f48242/infotheory-two-json-summary-20260322-120428.tsv @@ -1,145 +1,145 @@ -operation subject subject_kind expert_kind series size_bytes repeats cpu compression_backend input_sha256 real_seconds_mean real_seconds_stdev real_seconds_median real_seconds_min real_seconds_max user_seconds_mean sys_seconds_mean throughput_mib_s_mean throughput_mib_s_median rss_kib_mean rss_kib_stdev rss_kib_median rss_kib_min rss_kib_max archive_bytes_mean archive_bytes_median archive_ratio_mean archive_ratio_median entropy_bpb_mean entropy_bpb_median verified_all -h ctw expert ctw h:ctw 4096 2 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.02 0 0.02 0.02 0.02 0.015 0 0.1953125 0.1953125 6768 22.627416998 6768 6752 6784 2.56549624542 2.56549624542 1 -h ctw expert ctw h:ctw 16384 2 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.095 0.00707106781187 0.095 0.09 0.1 0.09 0 0.164930555556 0.164930555556 12464 5.65685424949 12464 12460 12468 3.20340951209 3.20340951209 1 -h ctw expert ctw h:ctw 65536 2 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.48 0 0.48 0.48 0.48 0.475 0 0.130208333333 0.130208333333 22972 141.421356237 22972 22872 23072 2.77213895634 2.77213895634 1 -h ctw expert ctw h:ctw 262144 2 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 2.37 0.0141421356237 2.37 2.36 2.38 2.35 0.01 0.105487110098 0.105487110098 49240 96.1665222414 49240 49172 49308 2.44096714152 2.44096714152 1 -h ctw expert ctw h:ctw 1048576 2 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 12.105 0.0212132034356 12.105 12.09 12.12 12.04 0.05 0.0826106183819 0.0826106183819 121828 22.627416998 121828 121812 121844 2.30174274755 2.30174274755 1 -h ctw expert ctw h:ctw 2097152 2 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 26.985 0.0494974746831 26.985 26.95 27.02 26.85 0.105 0.0741153738933 0.0741153738933 198884 5.65685424949 198884 198880 198888 2.27731545661 2.27731545661 1 -h ctw expert ctw h:ctw 4194304 2 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 59.545 0.176776695297 59.545 59.42 59.67 59.335 0.15 0.0671763813507 0.0671763813507 326188 84.8528137424 326188 326128 326248 2.24674930722 2.24674930722 1 -h ctw expert ctw h:ctw 10000000 2 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 157.65 0.197989898732 157.65 157.51 157.79 157.205 0.305 0.0604931854421 0.0604931854421 608744 22.627416998 608744 608728 608760 2.19747398481 2.19747398481 1 -h match expert match h:match 4096 2 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0 0 0 0 0 0 0 5698 115.965512115 5698 5616 5780 5.57734993252 5.57734993252 1 -h match expert match h:match 16384 2 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0 0 0 0 0 0 0 5796 107.48023074 5796 5720 5872 6.58586286407 6.58586286407 1 -h match expert match h:match 65536 2 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.005 0.00707106781187 0.005 0 0.01 0.005 0 6618 14.1421356237 6618 6608 6628 6.5588255305 6.5588255305 1 -h match expert match h:match 262144 2 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 0.03 0 0.03 0.03 0.03 0.03 0 8.33333333333 8.33333333333 8152 16.9705627485 8152 8140 8164 6.29225944669 6.29225944669 1 -h match expert match h:match 1048576 2 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 0.13 0 0.13 0.13 0.13 0.13 0 7.69230769231 7.69230769231 11494 98.9949493661 11494 11424 11564 6.23692635511 6.23692635511 1 -h match expert match h:match 2097152 2 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 0.265 0.00707106781187 0.265 0.26 0.27 0.255 0 7.54985754986 7.54985754986 19376 96.1665222414 19376 19308 19444 6.2741772649 6.2741772649 1 -h match expert match h:match 4194304 2 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 0.53 0 0.53 0.53 0.53 0.525 0 7.54716981132 7.54716981132 21430 82.0243866176 21430 21372 21488 6.28038800899 6.28038800899 1 -h match expert match h:match 10000000 2 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 1.345 0.0353553390593 1.345 1.32 1.37 1.33 0.01 7.09296591222 7.09296591222 40576 169.705627485 40576 40456 40696 6.28744436593 6.28744436593 1 -h neural_mixture mixture neural-mixture h:neural_mixture 4096 2 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.12 0 0.12 0.12 0.12 0.12 0 0.0325520833333 0.0325520833333 12004 5.65685424949 12004 12000 12008 1.91857366907 1.91857366907 1 -h neural_mixture mixture neural-mixture h:neural_mixture 16384 2 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.52 0 0.52 0.52 0.52 0.505 0 0.0300480769231 0.0300480769231 28948 96.1665222414 28948 28880 29016 2.69245779825 2.69245779825 1 -h neural_mixture mixture neural-mixture h:neural_mixture 65536 2 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 2.305 0.0212132034356 2.305 2.29 2.32 2.265 0.03 0.0271161157958 0.0271161157958 80390 2.82842712475 80390 80388 80392 2.35902117116 2.35902117116 1 -h neural_mixture mixture neural-mixture h:neural_mixture 262144 2 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 9.97 0 9.97 9.97 9.97 9.865 0.085 0.025075225677 0.025075225677 254872 50.9116882454 254872 254836 254908 2.05795611601 2.05795611601 1 -h neural_mixture mixture neural-mixture h:neural_mixture 1048576 2 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 44.785 0.0777817459305 44.785 44.73 44.84 44.405 0.33 0.0223289384438 0.0223289384438 701906 31.1126983722 701906 701884 701928 1.93963669332 1.93963669332 1 -h neural_mixture mixture neural-mixture h:neural_mixture 2097152 2 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 94.55 0.141421356237 94.55 94.45 94.65 94.02 0.44 0.0211528528526 0.0211528528526 1014206 14.1421356237 1014206 1014196 1014216 1.92069854644 1.92069854644 1 -h neural_mixture mixture neural-mixture h:neural_mixture 4194304 2 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 200.01 0.0282842712475 200.01 199.99 200.03 199.115 0.7 0.01999900025 0.01999900025 1471332 11.313708499 1471332 1471324 1471340 1.88509670528 1.88509670528 1 -h neural_mixture mixture neural-mixture h:neural_mixture 10000000 2 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 513.375 1.13844191771 513.375 512.57 514.18 511.63 1.23 0.018576608937 0.018576608937 2658496 192.333044483 2658496 2658360 2658632 1.81617547371 1.81617547371 1 -h ppmd expert ppmd h:ppmd 4096 2 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.01 0 0.01 0.01 0.01 0.01 0 0.390625 0.390625 7284 16.9705627485 7284 7272 7296 2.02916276221 2.02916276221 1 -h ppmd expert ppmd h:ppmd 16384 2 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.03 0 0.03 0.03 0.03 0.03 0 0.520833333333 0.520833333333 16126 183.847763109 16126 15996 16256 3.05111741489 3.05111741489 1 -h ppmd expert ppmd h:ppmd 65536 2 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.19 0 0.19 0.19 0.19 0.175 0.01 0.328947368421 0.328947368421 45210 246.073159853 45210 45036 45384 2.82800309907 2.82800309907 1 -h ppmd expert ppmd h:ppmd 262144 2 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 0.94 0.0282842712475 0.94 0.92 0.96 0.87 0.065 0.266077898551 0.266077898551 144134 115.965512115 144134 144052 144216 2.54059561824 2.54059561824 1 -h ppmd expert ppmd h:ppmd 1048576 2 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 3.815 0.00707106781187 3.815 3.81 3.82 3.65 0.155 0.262123648157 0.262123648157 399028 67.8822509939 399028 398980 399076 2.48956475398 2.48956475398 1 -h ppmd expert ppmd h:ppmd 2097152 2 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 7.4 0.0424264068712 7.4 7.37 7.43 7.155 0.235 0.270274712331 0.270274712331 457128 141.421356237 457128 457028 457228 2.52508057302 2.52508057302 1 -h ppmd expert ppmd h:ppmd 4194304 2 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 14.67 0 14.67 14.67 14.67 14.475 0.175 0.27266530334 0.27266530334 459302 93.3380951166 459302 459236 459368 2.5353209835 2.5353209835 1 -h ppmd expert ppmd h:ppmd 10000000 2 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 35.64 0.0141421356237 35.64 35.63 35.65 35.33 0.27 0.267585407263 0.267585407263 557404 73.5391052434 557404 557352 557456 2.52355814767 2.52355814767 1 -h rosa expert rosaplus h:rosa 4096 2 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.02 0 0.02 0.02 0.02 0.02 0 0.1953125 0.1953125 6170 48.0832611207 6170 6136 6204 2.11172124286 2.11172124286 1 -h rosa expert rosaplus h:rosa 16384 2 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.13 0 0.13 0.13 0.13 0.125 0.005 0.120192307692 0.120192307692 7874 308.298556597 7874 7656 8092 3.48630622986 3.48630622986 1 -h rosa expert rosaplus h:rosa 65536 2 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.635 0.00707106781187 0.635 0.63 0.64 0.605 0.03 0.0984312996032 0.0984312996032 15582 53.7401153702 15582 15544 15620 3.19056696201 3.19056696201 1 -h rosa expert rosaplus h:rosa 262144 2 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 3.535 0.00707106781187 3.535 3.53 3.54 3.435 0.09 0.0707214993358 0.0707214993358 45186 110.308657865 45186 45108 45264 2.90567254487 2.90567254487 1 -h rosa expert rosaplus h:rosa 1048576 2 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 19.315 0.00707106781187 19.315 19.31 19.32 18.81 0.475 0.0517732367078 0.0517732367078 200130 19.7989898732 200130 200116 200144 2.80123027825 2.80123027825 1 -h rosa expert rosaplus h:rosa 2097152 2 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 46.355 0.0777817459305 46.355 46.3 46.41 44.615 1.68 0.0431453525088 0.0431453525088 324604 22.627416998 324604 324588 324620 2.70437138702 2.70437138702 1 -h rosa expert rosaplus h:rosa 4194304 2 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 112.08 0.0565685424949 112.08 112.04 112.12 107.095 4.84 0.0356887982644 0.0356887982644 641732 90.5096679919 641732 641668 641796 2.62099230176 2.62099230176 1 -h rosa expert rosaplus h:rosa 10000000 2 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 333.705 0.53033008589 333.705 333.33 334.08 320.715 12.59 0.0285784007047 0.0285784007047 1477922 8.48528137424 1477922 1477916 1477928 2.48481897791 2.48481897791 1 -h rwkv expert rwkv h:rwkv 4096 2 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.065 0.00707106781187 0.065 0.06 0.07 0.065 0 0.0604538690476 0.0604538690476 7982 82.0243866176 7982 7924 8040 7.21701437947 7.21701437947 1 -h rwkv expert rwkv h:rwkv 16384 2 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.23 0 0.23 0.23 0.23 0.225 0 0.0679347826087 0.0679347826087 8102 115.965512115 8102 8020 8184 5.85021072042 5.85021072042 1 -h rwkv expert rwkv h:rwkv 65536 2 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.91 0 0.91 0.91 0.91 0.9 0 0.0686813186813 0.0686813186813 8258 65.0538238692 8258 8212 8304 4.31173851819 4.31173851819 1 -h rwkv expert rwkv h:rwkv 262144 2 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 3.645 0.0353553390593 3.645 3.62 3.67 3.645 0 0.0685903322444 0.0685903322444 8200 39.5979797464 8200 8172 8228 4.18065126521 4.18065126521 1 -h rwkv expert rwkv h:rwkv 1048576 2 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 15.02 0.0848528137424 15.02 14.96 15.08 15.01 0 0.0665789585668 0.0665789585668 8700 28.2842712475 8700 8680 8720 3.61829048151 3.61829048151 1 -h rwkv expert rwkv h:rwkv 2097152 2 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 30.515 0.162634559673 30.515 30.4 30.63 30.485 0 0.0655424678248 0.0655424678248 9858 82.0243866176 9858 9800 9916 3.49833649966 3.49833649966 1 -h rwkv expert rwkv h:rwkv 4194304 2 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 62.84 1.00409162928 62.84 62.13 63.55 62.79 0 0.0636618506183 0.0636618506183 11954 87.6812408671 11954 11892 12016 3.33570098711 3.33570098711 1 -h rwkv expert rwkv h:rwkv 10000000 2 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 148.11 0.11313708499 148.11 148.03 148.19 148.005 0 0.0643896154643 0.0643896154643 17642 8.48528137424 17642 17636 17648 3.17490278117 3.17490278117 1 -compress ctw expert ctw compress:ctw 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.03 0 0.03 0.03 0.03 0.03 0 0.130208333333 0.130208333333 6822 110.308657865 6822 6744 6900 1332 1332 0.3251953125 0.3251953125 1 -compress ctw expert ctw compress:ctw 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.18 0 0.18 0.18 0.18 0.17 0 0.0868055555556 0.0868055555556 12502 25.4558441227 12502 12484 12520 6579 6579 0.401550292969 0.401550292969 1 -compress ctw expert ctw compress:ctw 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.815 0.00707106781187 0.815 0.81 0.82 0.805 0 0.0766900030111 0.0766900030111 23010 200.818325857 23010 22868 23152 22728 22728 0.346801757812 0.346801757812 1 -compress ctw expert ctw compress:ctw 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 3.51 0 3.51 3.51 3.51 3.5 0.005 0.0712250712251 0.0712250712251 49558 8.48528137424 49558 49552 49564 80004 80004 0.305191040039 0.305191040039 1 -compress ctw expert ctw compress:ctw 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 15.84 0.0565685424949 15.84 15.8 15.88 15.765 0.055 0.063131715716 0.063131715716 122616 5.65685424949 122616 122612 122620 301713 301713 0.287735939026 0.287735939026 1 -compress ctw expert ctw compress:ctw 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 33.56 0 33.56 33.56 33.56 33.435 0.085 0.0595947556615 0.0595947556615 200070 65.0538238692 200070 200024 200116 597003 597003 0.284673213959 0.284673213959 1 -compress ctw expert ctw compress:ctw 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 70.575 0.120208152802 70.575 70.49 70.66 70.36 0.15 0.0566773758732 0.0566773758732 328660 135.764501988 328660 328564 328756 1177962 1177962 0.280848026276 0.280848026276 1 -compress ctw expert ctw compress:ctw 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 179.265 0.671751442127 179.265 178.79 179.74 178.77 0.33 0.0531995097848 0.0531995097848 614158 53.7401153702 614158 614120 614196 2746861 2746861 0.2746861 0.2746861 1 -compress match expert match compress:match 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0 0 0 0 0 0 0 5296 73.5391052434 5296 5244 5348 2874 2874 0.70166015625 0.70166015625 1 -compress match expert match compress:match 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.01 0 0.01 0.01 0.01 0.005 0 1.5625 1.5625 5580 39.5979797464 5580 5552 5608 13506 13506 0.824340820312 0.824340820312 1 -compress match expert match compress:match 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.03 0 0.03 0.03 0.03 0.03 0 2.08333333333 2.08333333333 6428 265.872149726 6428 6240 6616 53748 53748 0.820129394531 0.820129394531 1 -compress match expert match compress:match 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 0.13 0 0.13 0.13 0.13 0.125 0 1.92307692308 1.92307692308 8156 50.9116882454 8156 8120 8192 206203 206203 0.786602020264 0.786602020264 1 -compress match expert match compress:match 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 0.52 0 0.52 0.52 0.52 0.515 0.005 1.92307692308 1.92307692308 12104 220.61731573 12104 11948 12260 817505 817505 0.779633522034 0.779633522034 1 -compress match expert match compress:match 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 1.04 0 1.04 1.04 1.04 1.04 0 1.92307692308 1.92307692308 22090 25.4558441227 22090 22072 22108 1644756 1644756 0.784280776978 0.784280776978 1 -compress match expert match compress:match 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 2.14 0.0141421356237 2.14 2.13 2.15 2.13 0 1.86919969429 1.86919969429 29032 147.078210487 29032 28928 29136 3292751 3292751 0.785053014755 0.785053014755 1 -compress match expert match compress:match 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 5.04 0.0141421356237 5.04 5.03 5.05 5.015 0.015 1.89221839446 1.89221839446 55600 130.107647738 55600 55508 55692 7859324 7859324 0.7859324 0.7859324 1 -compress neural_mixture mixture neural-mixture compress:neural_mixture 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.13 0 0.13 0.13 0.13 0.12 0 0.0300480769231 0.0300480769231 12186 70.7106781187 12186 12136 12236 1001 1001 0.244384765625 0.244384765625 1 -compress neural_mixture mixture neural-mixture compress:neural_mixture 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.57 0 0.57 0.57 0.57 0.555 0.005 0.0274122807018 0.0274122807018 29508 33.941125497 29508 29484 29532 5533 5533 0.337707519531 0.337707519531 1 -compress neural_mixture mixture neural-mixture compress:neural_mixture 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 2.485 0.00707106781187 2.485 2.48 2.49 2.445 0.035 0.0251510072548 0.0251510072548 80902 115.965512115 80902 80820 80984 19344 19344 0.295166015625 0.295166015625 1 -compress neural_mixture mixture neural-mixture compress:neural_mixture 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 10.88 0.0424264068712 10.88 10.85 10.91 10.76 0.1 0.022978115879 0.022978115879 257328 84.8528137424 257328 257268 257388 67454 67454 0.257316589355 0.257316589355 1 -compress neural_mixture mixture neural-mixture compress:neural_mixture 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 49.005 0.332340187158 49.005 48.77 49.24 48.63 0.325 0.020406550284 0.020406550284 694028 39.5979797464 694028 694000 694056 254251 254251 0.242472648621 0.242472648621 1 -compress neural_mixture mixture neural-mixture compress:neural_mixture 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 103.785 0.106066017178 103.785 103.71 103.86 103.205 0.47 0.0192706175694 0.0192706175694 1026486 3385.62726832 1026486 1024092 1028880 503520 503520 0.240097045898 0.240097045898 1 -compress neural_mixture mixture neural-mixture compress:neural_mixture 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 220.565 0.615182899632 220.565 220.13 221 219.64 0.695 0.0181353141182 0.0181353141182 1453754 82.0243866176 1453754 1453696 1453812 988355 988355 0.235642194748 0.235642194748 1 -compress neural_mixture mixture neural-mixture compress:neural_mixture 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 575.54 2.07889393669 575.54 574.07 577.01 573.66 1.265 0.0165701869164 0.0165701869164 2646600 5221.27647228 2646600 2642908 2650292 2270248 2270248 0.2270248 0.2270248 1 -compress ppmd expert ppmd compress:ppmd 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.01 0 0.01 0.01 0.01 0.005 0 0.390625 0.390625 7262 42.4264068712 7262 7232 7292 1058 1058 0.25830078125 0.25830078125 1 -compress ppmd expert ppmd compress:ppmd 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.04 0 0.04 0.04 0.04 0.04 0 0.390625 0.390625 15978 93.3380951166 15978 15912 16044 6267 6267 0.382507324219 0.382507324219 1 -compress ppmd expert ppmd compress:ppmd 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.21 0 0.21 0.21 0.21 0.19 0.01 0.297619047619 0.297619047619 45062 98.9949493661 45062 44992 45132 23186 23186 0.353790283203 0.353790283203 1 -compress ppmd expert ppmd compress:ppmd 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 1.01 0.0141421356237 1.01 1 1.02 0.965 0.04 0.247549019608 0.247549019608 143994 189.504617358 143994 143860 144128 83269 83269 0.317646026611 0.317646026611 1 -compress ppmd expert ppmd compress:ppmd 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 4.195 0.00707106781187 4.195 4.19 4.2 4.015 0.17 0.238379361291 0.238379361291 399382 104.651803616 399382 399308 399456 326331 326331 0.311213493347 0.311213493347 1 -compress ppmd expert ppmd compress:ppmd 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 8.16 0.0141421356237 8.16 8.15 8.17 7.94 0.205 0.245098407311 0.245098407311 457658 14.1421356237 457658 457648 457668 661953 661953 0.315643787384 0.315643787384 1 -compress ppmd expert ppmd compress:ppmd 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 16.215 0.0494974746831 16.215 16.18 16.25 15.98 0.215 0.246686317391 0.246686317391 459716 339.41125497 459716 459476 459956 1329257 1329257 0.316919565201 0.316919565201 1 -compress ppmd expert ppmd compress:ppmd 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 39.19 0.0565685424949 39.19 39.15 39.23 38.89 0.265 0.243346596048 0.243346596048 568906 31.1126983722 568906 568884 568928 3154465 3154465 0.3154465 0.3154465 1 -compress rosa expert rosaplus compress:rosa 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0 0 0 0 0 0 0 5736 56.5685424949 5736 5696 5776 1127 1127 0.275146484375 0.275146484375 1 -compress rosa expert rosaplus compress:rosa 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.02 0 0.02 0.02 0.02 0.02 0 0.78125 0.78125 8320 113.13708499 8320 8240 8400 6359 6359 0.388122558594 0.388122558594 1 -compress rosa expert rosaplus compress:rosa 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.12 0 0.12 0.12 0.12 0.115 0.005 0.520833333333 0.520833333333 17588 186.676190233 17588 17456 17720 22843 22843 0.348556518555 0.348556518555 1 -compress rosa expert rosaplus compress:rosa 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 0.695 0.00707106781187 0.695 0.69 0.7 0.68 0.01 0.359730848861 0.359730848861 55486 104.651803616 55486 55412 55560 80590 80590 0.307426452637 0.307426452637 1 -compress rosa expert rosaplus compress:rosa 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 3.84 0.0141421356237 3.84 3.83 3.85 3.78 0.05 0.260418432742 0.260418432742 146162 59.3969696197 146162 146120 146204 306522 306522 0.292322158813 0.292322158813 1 -compress rosa expert rosaplus compress:rosa 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 9.32 0.0141421356237 9.32 9.31 9.33 9.175 0.135 0.214592521727 0.214592521727 284848 67.8822509939 284848 284800 284896 608681 608681 0.290241718292 0.290241718292 1 -compress rosa expert rosaplus compress:rosa 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 23.395 0.106066017178 23.395 23.32 23.47 23.145 0.225 0.17097846161 0.17097846161 564220 22.627416998 564220 564204 564236 1199345 1199345 0.285946130753 0.285946130753 1 -compress rosa expert rosaplus compress:rosa 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 73.8 0.0848528137424 73.8 73.74 73.86 73.085 0.64 0.12922424753 0.12922424753 1306208 50.9116882454 1306208 1306172 1306244 2752778 2752778 0.2752778 0.2752778 1 -compress rwkv expert rwkv compress:rwkv 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.06 0 0.06 0.06 0.06 0.06 0 0.0651041666667 0.0651041666667 7710 65.0538238692 7710 7664 7756 3714 3714 0.90673828125 0.90673828125 1 -compress rwkv expert rwkv compress:rwkv 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.245 0.00707106781187 0.245 0.24 0.25 0.24 0 0.0638020833333 0.0638020833333 7666 36.7695526217 7666 7640 7692 12000 12000 0.732421875 0.732421875 1 -compress rwkv expert rwkv compress:rwkv 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.96 0.0282842712475 0.96 0.94 0.98 0.955 0 0.0651324359531 0.0651324359531 7722 104.651803616 7722 7648 7796 35340 35340 0.539245605469 0.539245605469 1 -compress rwkv expert rwkv compress:rwkv 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 3.81 0.0424264068712 3.81 3.78 3.84 3.805 0 0.0656208664021 0.0656208664021 7872 107.48023074 7872 7796 7948 137010 137010 0.522651672363 0.522651672363 1 -compress rwkv expert rwkv compress:rwkv 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 15.805 0.346482322781 15.805 15.56 16.05 15.71 0.085 0.0632863240676 0.0632863240676 9256 96.1665222414 9256 9188 9324 474275 474275 0.452303886414 0.452303886414 1 -compress rwkv expert rwkv compress:rwkv 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 32.195 0.0212132034356 32.195 32.18 32.21 31.66 0.5 0.0621214609146 0.0621214609146 11250 189.504617358 11250 11116 11384 917086 917086 0.437300682068 0.437300682068 1 -compress rwkv expert rwkv compress:rwkv 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 66.09 0.141421356237 66.09 65.99 66.19 64.365 1.66 0.0605236670868 0.0605236670868 14946 48.0832611207 14946 14912 14980 1748887 1748887 0.416967153549 0.416967153549 1 -compress rwkv expert rwkv compress:rwkv 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 155.31 0.537401153702 155.31 154.93 155.69 153.32 1.86 0.0614049337165 0.0614049337165 24942 149.906637612 24942 24836 25048 3968645 3968645 0.3968645 0.3968645 1 -decompress ctw expert ctw decompress:ctw 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.03 0 0.03 0.03 0.03 0.03 0 0.130208333333 0.130208333333 6874 115.965512115 6874 6792 6956 1332 1332 0.3251953125 0.3251953125 1 -decompress ctw expert ctw decompress:ctw 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.18 0 0.18 0.18 0.18 0.18 0 0.0868055555556 0.0868055555556 12706 70.7106781187 12706 12656 12756 6579 6579 0.401550292969 0.401550292969 1 -decompress ctw expert ctw decompress:ctw 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.815 0.00707106781187 0.815 0.81 0.82 0.805 0.005 0.0766900030111 0.0766900030111 22796 67.8822509939 22796 22748 22844 22728 22728 0.346801757812 0.346801757812 1 -decompress ctw expert ctw decompress:ctw 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 3.525 0.00707106781187 3.525 3.52 3.53 3.5 0.02 0.0709221285089 0.0709221285089 49508 62.2253967444 49508 49464 49552 80004 80004 0.305191040039 0.305191040039 1 -decompress ctw expert ctw decompress:ctw 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 15.89 0.0141421356237 15.89 15.88 15.9 15.82 0.05 0.0629326869762 0.0629326869762 122218 31.1126983722 122218 122196 122240 301713 301713 0.287735939026 0.287735939026 1 -decompress ctw expert ctw decompress:ctw 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 33.545 0.00707106781186 33.545 33.54 33.55 33.405 0.11 0.0596214054087 0.0596214054087 199622 48.0832611207 199622 199588 199656 597003 597003 0.284673213959 0.284673213959 1 -decompress ctw expert ctw decompress:ctw 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 70.815 0.00707106781186 70.815 70.81 70.82 70.605 0.15 0.0564852082178 0.0564852082178 327672 45.2548339959 327672 327640 327704 1177962 1177962 0.280848026276 0.280848026276 1 -decompress ctw expert ctw decompress:ctw 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 179.705 0.0919238815543 179.705 179.64 179.77 179.205 0.33 0.0530688874085 0.0530688874085 611636 90.5096679919 611636 611572 611700 2746861 2746861 0.2746861 0.2746861 1 -decompress match expert match decompress:match 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0 0 0 0 0 0 0 5498 42.4264068712 5498 5468 5528 2874 2874 0.70166015625 0.70166015625 1 -decompress match expert match decompress:match 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.01 0 0.01 0.01 0.01 0.01 0 1.5625 1.5625 5642 144.249783362 5642 5540 5744 13506 13506 0.824340820312 0.824340820312 1 -decompress match expert match decompress:match 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.03 0 0.03 0.03 0.03 0.03 0 2.08333333333 2.08333333333 6314 59.3969696197 6314 6272 6356 53748 53748 0.820129394531 0.820129394531 1 -decompress match expert match decompress:match 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 0.14 0 0.14 0.14 0.14 0.135 0 1.78571428571 1.78571428571 8110 36.7695526217 8110 8084 8136 206203 206203 0.786602020264 0.786602020264 1 -decompress match expert match decompress:match 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 0.56 0 0.56 0.56 0.56 0.56 0 1.78571428571 1.78571428571 11610 2.82842712475 11610 11608 11612 817505 817505 0.779633522034 0.779633522034 1 -decompress match expert match decompress:match 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 1.125 0.00707106781187 1.125 1.12 1.13 1.12 0 1.77781289507 1.77781289507 20102 115.965512115 20102 20020 20184 1644756 1644756 0.784280776978 0.784280776978 1 -decompress match expert match decompress:match 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 2.3 0.0141421356237 2.3 2.29 2.31 2.29 0 1.73916331122 1.73916331122 23962 161.220346111 23962 23848 24076 3292751 3292751 0.785053014755 0.785053014755 1 -decompress match expert match decompress:match 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 5.45 0.0141421356237 5.45 5.44 5.46 5.425 0.015 1.74986702233 1.74986702233 46162 76.3675323681 46162 46108 46216 7859324 7859324 0.7859324 0.7859324 1 -decompress neural_mixture mixture neural-mixture decompress:neural_mixture 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.13 0 0.13 0.13 0.13 0.125 0 0.0300480769231 0.0300480769231 11858 48.0832611207 11858 11824 11892 1001 1001 0.244384765625 0.244384765625 1 -decompress neural_mixture mixture neural-mixture decompress:neural_mixture 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.565 0.00707106781187 0.565 0.56 0.57 0.555 0.005 0.027657033208 0.027657033208 29926 8.48528137424 29926 29920 29932 5533 5533 0.337707519531 0.337707519531 1 -decompress neural_mixture mixture neural-mixture decompress:neural_mixture 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 2.5 0.0141421356237 2.5 2.49 2.51 2.465 0.025 0.0250004000064 0.0250004000064 80690 82.0243866176 80690 80632 80748 19344 19344 0.295166015625 0.295166015625 1 -decompress neural_mixture mixture neural-mixture decompress:neural_mixture 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 10.935 0.0494974746831 10.935 10.9 10.97 10.825 0.09 0.0228626027615 0.0228626027615 259300 22.627416998 259300 259284 259316 67454 67454 0.257316589355 0.257316589355 1 -decompress neural_mixture mixture neural-mixture decompress:neural_mixture 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 48.96 0.0141421356237 48.96 48.95 48.97 48.595 0.315 0.0204248374534 0.0204248374534 701542 104.651803616 701542 701468 701616 254251 254251 0.242472648621 0.242472648621 1 -decompress neural_mixture mixture neural-mixture decompress:neural_mixture 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 104.045 0.0777817459305 104.045 103.99 104.1 103.49 0.44 0.0192224571952 0.0192224571952 1013456 16.9705627485 1013456 1013444 1013468 503520 503520 0.240097045898 0.240097045898 1 -decompress neural_mixture mixture neural-mixture decompress:neural_mixture 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 221.8 0.381837661841 221.8 221.53 222.07 220.84 0.72 0.0180342918278 0.0180342918278 1471378 161.220346111 1471378 1471264 1471492 988355 988355 0.235642194748 0.235642194748 1 -decompress neural_mixture mixture neural-mixture decompress:neural_mixture 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 574.545 0.997020561473 574.545 573.84 575.25 572.645 1.265 0.0165987999605 0.0165987999605 2655396 33.941125497 2655396 2655372 2655420 2270248 2270248 0.2270248 0.2270248 1 -decompress ppmd expert ppmd decompress:ppmd 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.01 0 0.01 0.01 0.01 0.005 0 0.390625 0.390625 7364 11.313708499 7364 7356 7372 1058 1058 0.25830078125 0.25830078125 1 -decompress ppmd expert ppmd decompress:ppmd 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.04 0 0.04 0.04 0.04 0.03 0 0.390625 0.390625 16078 36.7695526217 16078 16052 16104 6267 6267 0.382507324219 0.382507324219 1 -decompress ppmd expert ppmd decompress:ppmd 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.21 0 0.21 0.21 0.21 0.185 0.02 0.297619047619 0.297619047619 45074 8.48528137424 45074 45068 45080 23186 23186 0.353790283203 0.353790283203 1 -decompress ppmd expert ppmd decompress:ppmd 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 1.02 0.0141421356237 1.02 1.01 1.03 0.945 0.07 0.245121599539 0.245121599539 144204 90.5096679919 144204 144140 144268 83269 83269 0.317646026611 0.317646026611 1 -decompress ppmd expert ppmd decompress:ppmd 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 4.245 0.0212132034356 4.245 4.23 4.26 4.06 0.175 0.235574201711 0.235574201711 399112 16.9705627485 399112 399100 399124 326331 326331 0.311213493347 0.311213493347 1 -decompress ppmd expert ppmd decompress:ppmd 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 8.25 0 8.25 8.25 8.25 8.035 0.205 0.242424242424 0.242424242424 457468 158.391918986 457468 457356 457580 661953 661953 0.315643787384 0.315643787384 1 -decompress ppmd expert ppmd decompress:ppmd 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 16.285 0.0353553390593 16.285 16.26 16.31 16.05 0.21 0.245625386971 0.245625386971 458390 121.622366364 458390 458304 458476 1329257 1329257 0.316919565201 0.316919565201 1 -decompress ppmd expert ppmd decompress:ppmd 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 39.55 0.0707106781187 39.55 39.5 39.6 39.225 0.285 0.241131691688 0.241131691688 558980 124.450793489 558980 558892 559068 3154465 3154465 0.3154465 0.3154465 1 -decompress rosa expert rosaplus decompress:rosa 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0 0 0 0 0 0 0 6016 33.941125497 6016 5992 6040 1127 1127 0.275146484375 0.275146484375 1 -decompress rosa expert rosaplus decompress:rosa 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.02 0 0.02 0.02 0.02 0.02 0 0.78125 0.78125 8344 62.2253967444 8344 8300 8388 6359 6359 0.388122558594 0.388122558594 1 -decompress rosa expert rosaplus decompress:rosa 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.13 0 0.13 0.13 0.13 0.115 0.005 0.480769230769 0.480769230769 17808 79.1959594929 17808 17752 17864 22843 22843 0.348556518555 0.348556518555 1 -decompress rosa expert rosaplus decompress:rosa 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 0.7 0 0.7 0.7 0.7 0.69 0.005 0.357142857143 0.357142857143 55634 42.4264068712 55634 55604 55664 80590 80590 0.307426452637 0.307426452637 1 -decompress rosa expert rosaplus decompress:rosa 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 3.855 0.00707106781187 3.855 3.85 3.86 3.8 0.045 0.259403808627 0.259403808627 146626 31.1126983722 146626 146604 146648 306522 306522 0.292322158813 0.292322158813 1 -decompress rosa expert rosaplus decompress:rosa 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 9.395 0.0636396103068 9.395 9.35 9.44 9.285 0.095 0.212884075048 0.212884075048 285428 118.793939239 285428 285344 285512 608681 608681 0.290241718292 0.290241718292 1 -decompress rosa expert rosaplus decompress:rosa 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 23.965 0.671751442127 23.965 23.49 24.44 23.72 0.215 0.166975674435 0.166975674435 553182 31.1126983722 553182 553160 553204 1199345 1199345 0.285946130753 0.285946130753 1 -decompress rosa expert rosaplus decompress:rosa 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 74.56 0.0282842712475 74.56 74.54 74.58 73.81 0.675 0.127906972241 0.127906972241 1304464 56.5685424949 1304464 1304424 1304504 2752778 2752778 0.2752778 0.2752778 1 -decompress rwkv expert rwkv decompress:rwkv 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.06 0 0.06 0.06 0.06 0.06 0 0.0651041666667 0.0651041666667 7650 25.4558441227 7650 7632 7668 3714 3714 0.90673828125 0.90673828125 1 -decompress rwkv expert rwkv decompress:rwkv 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.24 0 0.24 0.24 0.24 0.235 0 0.0651041666667 0.0651041666667 7840 33.941125497 7840 7816 7864 12000 12000 0.732421875 0.732421875 1 -decompress rwkv expert rwkv decompress:rwkv 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.945 0.00707106781187 0.945 0.94 0.95 0.935 0 0.0661394176932 0.0661394176932 7906 8.48528137424 7906 7900 7912 35340 35340 0.539245605469 0.539245605469 1 -decompress rwkv expert rwkv decompress:rwkv 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 3.815 0.0494974746831 3.815 3.78 3.85 3.81 0 0.0655363155363 0.0655363155363 7732 84.8528137424 7732 7672 7792 137010 137010 0.522651672363 0.522651672363 1 -decompress rwkv expert rwkv decompress:rwkv 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 15.78 0.0424264068712 15.78 15.75 15.81 15.77 0 0.0633715851932 0.0633715851932 8930 25.4558441227 8930 8912 8948 474275 474275 0.452303886414 0.452303886414 1 -decompress rwkv expert rwkv decompress:rwkv 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 31.655 0.30405591591 31.655 31.44 31.87 31.63 0 0.0631840867519 0.0631840867519 10342 8.48528137424 10342 10336 10348 917086 917086 0.437300682068 0.437300682068 1 -decompress rwkv expert rwkv decompress:rwkv 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 63.89 0.0424264068712 63.89 63.86 63.92 63.84 0 0.0626076206282 0.0626076206282 13160 16.9705627485 13160 13148 13172 1748887 1748887 0.416967153549 0.416967153549 1 -decompress rwkv expert rwkv decompress:rwkv 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 153.94 0.311126983722 153.94 153.72 154.16 153.82 0.01 0.0619511669613 0.0619511669613 20890 25.4558441227 20890 20872 20908 3968645 3968645 0.3968645 0.3968645 1 +operation subject subject_kind expert_kind series size_bytes repeats cpu compression_backend input_sha256 real_seconds_mean real_seconds_stdev real_seconds_median real_seconds_min real_seconds_max user_seconds_mean sys_seconds_mean throughput_mib_s_mean throughput_mib_s_median rss_kib_mean rss_kib_stdev rss_kib_median rss_kib_min rss_kib_max archive_bytes_mean archive_bytes_median archive_ratio_mean archive_ratio_median entropy_bpb_mean entropy_bpb_median verified_all +h ctw expert ctw h:ctw 4096 2 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.02 0 0.02 0.02 0.02 0.015 0 0.1953125 0.1953125 6768 22.627416998 6768 6752 6784 2.56549624542 2.56549624542 1 +h ctw expert ctw h:ctw 16384 2 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.095 0.00707106781187 0.095 0.09 0.1 0.09 0 0.164930555556 0.164930555556 12464 5.65685424949 12464 12460 12468 3.20340951209 3.20340951209 1 +h ctw expert ctw h:ctw 65536 2 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.48 0 0.48 0.48 0.48 0.475 0 0.130208333333 0.130208333333 22972 141.421356237 22972 22872 23072 2.77213895634 2.77213895634 1 +h ctw expert ctw h:ctw 262144 2 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 2.37 0.0141421356237 2.37 2.36 2.38 2.35 0.01 0.105487110098 0.105487110098 49240 96.1665222414 49240 49172 49308 2.44096714152 2.44096714152 1 +h ctw expert ctw h:ctw 1048576 2 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 12.105 0.0212132034356 12.105 12.09 12.12 12.04 0.05 0.0826106183819 0.0826106183819 121828 22.627416998 121828 121812 121844 2.30174274755 2.30174274755 1 +h ctw expert ctw h:ctw 2097152 2 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 26.985 0.0494974746831 26.985 26.95 27.02 26.85 0.105 0.0741153738933 0.0741153738933 198884 5.65685424949 198884 198880 198888 2.27731545661 2.27731545661 1 +h ctw expert ctw h:ctw 4194304 2 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 59.545 0.176776695297 59.545 59.42 59.67 59.335 0.15 0.0671763813507 0.0671763813507 326188 84.8528137424 326188 326128 326248 2.24674930722 2.24674930722 1 +h ctw expert ctw h:ctw 10000000 2 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 157.65 0.197989898732 157.65 157.51 157.79 157.205 0.305 0.0604931854421 0.0604931854421 608744 22.627416998 608744 608728 608760 2.19747398481 2.19747398481 1 +h match expert match h:match 4096 2 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0 0 0 0 0 0 0 5698 115.965512115 5698 5616 5780 5.57734993252 5.57734993252 1 +h match expert match h:match 16384 2 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0 0 0 0 0 0 0 5796 107.48023074 5796 5720 5872 6.58586286407 6.58586286407 1 +h match expert match h:match 65536 2 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.005 0.00707106781187 0.005 0 0.01 0.005 0 6618 14.1421356237 6618 6608 6628 6.5588255305 6.5588255305 1 +h match expert match h:match 262144 2 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 0.03 0 0.03 0.03 0.03 0.03 0 8.33333333333 8.33333333333 8152 16.9705627485 8152 8140 8164 6.29225944669 6.29225944669 1 +h match expert match h:match 1048576 2 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 0.13 0 0.13 0.13 0.13 0.13 0 7.69230769231 7.69230769231 11494 98.9949493661 11494 11424 11564 6.23692635511 6.23692635511 1 +h match expert match h:match 2097152 2 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 0.265 0.00707106781187 0.265 0.26 0.27 0.255 0 7.54985754986 7.54985754986 19376 96.1665222414 19376 19308 19444 6.2741772649 6.2741772649 1 +h match expert match h:match 4194304 2 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 0.53 0 0.53 0.53 0.53 0.525 0 7.54716981132 7.54716981132 21430 82.0243866176 21430 21372 21488 6.28038800899 6.28038800899 1 +h match expert match h:match 10000000 2 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 1.345 0.0353553390593 1.345 1.32 1.37 1.33 0.01 7.09296591222 7.09296591222 40576 169.705627485 40576 40456 40696 6.28744436593 6.28744436593 1 +h neural_mixture mixture neural-mixture h:neural_mixture 4096 2 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.12 0 0.12 0.12 0.12 0.12 0 0.0325520833333 0.0325520833333 12004 5.65685424949 12004 12000 12008 1.91857366907 1.91857366907 1 +h neural_mixture mixture neural-mixture h:neural_mixture 16384 2 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.52 0 0.52 0.52 0.52 0.505 0 0.0300480769231 0.0300480769231 28948 96.1665222414 28948 28880 29016 2.69245779825 2.69245779825 1 +h neural_mixture mixture neural-mixture h:neural_mixture 65536 2 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 2.305 0.0212132034356 2.305 2.29 2.32 2.265 0.03 0.0271161157958 0.0271161157958 80390 2.82842712475 80390 80388 80392 2.35902117116 2.35902117116 1 +h neural_mixture mixture neural-mixture h:neural_mixture 262144 2 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 9.97 0 9.97 9.97 9.97 9.865 0.085 0.025075225677 0.025075225677 254872 50.9116882454 254872 254836 254908 2.05795611601 2.05795611601 1 +h neural_mixture mixture neural-mixture h:neural_mixture 1048576 2 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 44.785 0.0777817459305 44.785 44.73 44.84 44.405 0.33 0.0223289384438 0.0223289384438 701906 31.1126983722 701906 701884 701928 1.93963669332 1.93963669332 1 +h neural_mixture mixture neural-mixture h:neural_mixture 2097152 2 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 94.55 0.141421356237 94.55 94.45 94.65 94.02 0.44 0.0211528528526 0.0211528528526 1014206 14.1421356237 1014206 1014196 1014216 1.92069854644 1.92069854644 1 +h neural_mixture mixture neural-mixture h:neural_mixture 4194304 2 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 200.01 0.0282842712475 200.01 199.99 200.03 199.115 0.7 0.01999900025 0.01999900025 1471332 11.313708499 1471332 1471324 1471340 1.88509670528 1.88509670528 1 +h neural_mixture mixture neural-mixture h:neural_mixture 10000000 2 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 513.375 1.13844191771 513.375 512.57 514.18 511.63 1.23 0.018576608937 0.018576608937 2658496 192.333044483 2658496 2658360 2658632 1.81617547371 1.81617547371 1 +h ppmd expert ppmd h:ppmd 4096 2 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.01 0 0.01 0.01 0.01 0.01 0 0.390625 0.390625 7284 16.9705627485 7284 7272 7296 2.02916276221 2.02916276221 1 +h ppmd expert ppmd h:ppmd 16384 2 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.03 0 0.03 0.03 0.03 0.03 0 0.520833333333 0.520833333333 16126 183.847763109 16126 15996 16256 3.05111741489 3.05111741489 1 +h ppmd expert ppmd h:ppmd 65536 2 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.19 0 0.19 0.19 0.19 0.175 0.01 0.328947368421 0.328947368421 45210 246.073159853 45210 45036 45384 2.82800309907 2.82800309907 1 +h ppmd expert ppmd h:ppmd 262144 2 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 0.94 0.0282842712475 0.94 0.92 0.96 0.87 0.065 0.266077898551 0.266077898551 144134 115.965512115 144134 144052 144216 2.54059561824 2.54059561824 1 +h ppmd expert ppmd h:ppmd 1048576 2 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 3.815 0.00707106781187 3.815 3.81 3.82 3.65 0.155 0.262123648157 0.262123648157 399028 67.8822509939 399028 398980 399076 2.48956475398 2.48956475398 1 +h ppmd expert ppmd h:ppmd 2097152 2 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 7.4 0.0424264068712 7.4 7.37 7.43 7.155 0.235 0.270274712331 0.270274712331 457128 141.421356237 457128 457028 457228 2.52508057302 2.52508057302 1 +h ppmd expert ppmd h:ppmd 4194304 2 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 14.67 0 14.67 14.67 14.67 14.475 0.175 0.27266530334 0.27266530334 459302 93.3380951166 459302 459236 459368 2.5353209835 2.5353209835 1 +h ppmd expert ppmd h:ppmd 10000000 2 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 35.64 0.0141421356237 35.64 35.63 35.65 35.33 0.27 0.267585407263 0.267585407263 557404 73.5391052434 557404 557352 557456 2.52355814767 2.52355814767 1 +h rosa expert rosaplus h:rosa 4096 2 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.02 0 0.02 0.02 0.02 0.02 0 0.1953125 0.1953125 6170 48.0832611207 6170 6136 6204 2.11172124286 2.11172124286 1 +h rosa expert rosaplus h:rosa 16384 2 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.13 0 0.13 0.13 0.13 0.125 0.005 0.120192307692 0.120192307692 7874 308.298556597 7874 7656 8092 3.48630622986 3.48630622986 1 +h rosa expert rosaplus h:rosa 65536 2 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.635 0.00707106781187 0.635 0.63 0.64 0.605 0.03 0.0984312996032 0.0984312996032 15582 53.7401153702 15582 15544 15620 3.19056696201 3.19056696201 1 +h rosa expert rosaplus h:rosa 262144 2 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 3.535 0.00707106781187 3.535 3.53 3.54 3.435 0.09 0.0707214993358 0.0707214993358 45186 110.308657865 45186 45108 45264 2.90567254487 2.90567254487 1 +h rosa expert rosaplus h:rosa 1048576 2 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 19.315 0.00707106781187 19.315 19.31 19.32 18.81 0.475 0.0517732367078 0.0517732367078 200130 19.7989898732 200130 200116 200144 2.80123027825 2.80123027825 1 +h rosa expert rosaplus h:rosa 2097152 2 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 46.355 0.0777817459305 46.355 46.3 46.41 44.615 1.68 0.0431453525088 0.0431453525088 324604 22.627416998 324604 324588 324620 2.70437138702 2.70437138702 1 +h rosa expert rosaplus h:rosa 4194304 2 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 112.08 0.0565685424949 112.08 112.04 112.12 107.095 4.84 0.0356887982644 0.0356887982644 641732 90.5096679919 641732 641668 641796 2.62099230176 2.62099230176 1 +h rosa expert rosaplus h:rosa 10000000 2 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 333.705 0.53033008589 333.705 333.33 334.08 320.715 12.59 0.0285784007047 0.0285784007047 1477922 8.48528137424 1477922 1477916 1477928 2.48481897791 2.48481897791 1 +h rwkv expert rwkv h:rwkv 4096 2 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.065 0.00707106781187 0.065 0.06 0.07 0.065 0 0.0604538690476 0.0604538690476 7982 82.0243866176 7982 7924 8040 7.21701437947 7.21701437947 1 +h rwkv expert rwkv h:rwkv 16384 2 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.23 0 0.23 0.23 0.23 0.225 0 0.0679347826087 0.0679347826087 8102 115.965512115 8102 8020 8184 5.85021072042 5.85021072042 1 +h rwkv expert rwkv h:rwkv 65536 2 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.91 0 0.91 0.91 0.91 0.9 0 0.0686813186813 0.0686813186813 8258 65.0538238692 8258 8212 8304 4.31173851819 4.31173851819 1 +h rwkv expert rwkv h:rwkv 262144 2 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 3.645 0.0353553390593 3.645 3.62 3.67 3.645 0 0.0685903322444 0.0685903322444 8200 39.5979797464 8200 8172 8228 4.18065126521 4.18065126521 1 +h rwkv expert rwkv h:rwkv 1048576 2 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 15.02 0.0848528137424 15.02 14.96 15.08 15.01 0 0.0665789585668 0.0665789585668 8700 28.2842712475 8700 8680 8720 3.61829048151 3.61829048151 1 +h rwkv expert rwkv h:rwkv 2097152 2 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 30.515 0.162634559673 30.515 30.4 30.63 30.485 0 0.0655424678248 0.0655424678248 9858 82.0243866176 9858 9800 9916 3.49833649966 3.49833649966 1 +h rwkv expert rwkv h:rwkv 4194304 2 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 62.84 1.00409162928 62.84 62.13 63.55 62.79 0 0.0636618506183 0.0636618506183 11954 87.6812408671 11954 11892 12016 3.33570098711 3.33570098711 1 +h rwkv expert rwkv h:rwkv 10000000 2 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 148.11 0.11313708499 148.11 148.03 148.19 148.005 0 0.0643896154643 0.0643896154643 17642 8.48528137424 17642 17636 17648 3.17490278117 3.17490278117 1 +compress ctw expert ctw compress:ctw 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.03 0 0.03 0.03 0.03 0.03 0 0.130208333333 0.130208333333 6822 110.308657865 6822 6744 6900 1332 1332 0.3251953125 0.3251953125 1 +compress ctw expert ctw compress:ctw 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.18 0 0.18 0.18 0.18 0.17 0 0.0868055555556 0.0868055555556 12502 25.4558441227 12502 12484 12520 6579 6579 0.401550292969 0.401550292969 1 +compress ctw expert ctw compress:ctw 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.815 0.00707106781187 0.815 0.81 0.82 0.805 0 0.0766900030111 0.0766900030111 23010 200.818325857 23010 22868 23152 22728 22728 0.346801757812 0.346801757812 1 +compress ctw expert ctw compress:ctw 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 3.51 0 3.51 3.51 3.51 3.5 0.005 0.0712250712251 0.0712250712251 49558 8.48528137424 49558 49552 49564 80004 80004 0.305191040039 0.305191040039 1 +compress ctw expert ctw compress:ctw 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 15.84 0.0565685424949 15.84 15.8 15.88 15.765 0.055 0.063131715716 0.063131715716 122616 5.65685424949 122616 122612 122620 301713 301713 0.287735939026 0.287735939026 1 +compress ctw expert ctw compress:ctw 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 33.56 0 33.56 33.56 33.56 33.435 0.085 0.0595947556615 0.0595947556615 200070 65.0538238692 200070 200024 200116 597003 597003 0.284673213959 0.284673213959 1 +compress ctw expert ctw compress:ctw 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 70.575 0.120208152802 70.575 70.49 70.66 70.36 0.15 0.0566773758732 0.0566773758732 328660 135.764501988 328660 328564 328756 1177962 1177962 0.280848026276 0.280848026276 1 +compress ctw expert ctw compress:ctw 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 179.265 0.671751442127 179.265 178.79 179.74 178.77 0.33 0.0531995097848 0.0531995097848 614158 53.7401153702 614158 614120 614196 2746861 2746861 0.2746861 0.2746861 1 +compress match expert match compress:match 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0 0 0 0 0 0 0 5296 73.5391052434 5296 5244 5348 2874 2874 0.70166015625 0.70166015625 1 +compress match expert match compress:match 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.01 0 0.01 0.01 0.01 0.005 0 1.5625 1.5625 5580 39.5979797464 5580 5552 5608 13506 13506 0.824340820312 0.824340820312 1 +compress match expert match compress:match 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.03 0 0.03 0.03 0.03 0.03 0 2.08333333333 2.08333333333 6428 265.872149726 6428 6240 6616 53748 53748 0.820129394531 0.820129394531 1 +compress match expert match compress:match 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 0.13 0 0.13 0.13 0.13 0.125 0 1.92307692308 1.92307692308 8156 50.9116882454 8156 8120 8192 206203 206203 0.786602020264 0.786602020264 1 +compress match expert match compress:match 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 0.52 0 0.52 0.52 0.52 0.515 0.005 1.92307692308 1.92307692308 12104 220.61731573 12104 11948 12260 817505 817505 0.779633522034 0.779633522034 1 +compress match expert match compress:match 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 1.04 0 1.04 1.04 1.04 1.04 0 1.92307692308 1.92307692308 22090 25.4558441227 22090 22072 22108 1644756 1644756 0.784280776978 0.784280776978 1 +compress match expert match compress:match 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 2.14 0.0141421356237 2.14 2.13 2.15 2.13 0 1.86919969429 1.86919969429 29032 147.078210487 29032 28928 29136 3292751 3292751 0.785053014755 0.785053014755 1 +compress match expert match compress:match 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 5.04 0.0141421356237 5.04 5.03 5.05 5.015 0.015 1.89221839446 1.89221839446 55600 130.107647738 55600 55508 55692 7859324 7859324 0.7859324 0.7859324 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.13 0 0.13 0.13 0.13 0.12 0 0.0300480769231 0.0300480769231 12186 70.7106781187 12186 12136 12236 1001 1001 0.244384765625 0.244384765625 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.57 0 0.57 0.57 0.57 0.555 0.005 0.0274122807018 0.0274122807018 29508 33.941125497 29508 29484 29532 5533 5533 0.337707519531 0.337707519531 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 2.485 0.00707106781187 2.485 2.48 2.49 2.445 0.035 0.0251510072548 0.0251510072548 80902 115.965512115 80902 80820 80984 19344 19344 0.295166015625 0.295166015625 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 10.88 0.0424264068712 10.88 10.85 10.91 10.76 0.1 0.022978115879 0.022978115879 257328 84.8528137424 257328 257268 257388 67454 67454 0.257316589355 0.257316589355 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 49.005 0.332340187158 49.005 48.77 49.24 48.63 0.325 0.020406550284 0.020406550284 694028 39.5979797464 694028 694000 694056 254251 254251 0.242472648621 0.242472648621 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 103.785 0.106066017178 103.785 103.71 103.86 103.205 0.47 0.0192706175694 0.0192706175694 1026486 3385.62726832 1026486 1024092 1028880 503520 503520 0.240097045898 0.240097045898 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 220.565 0.615182899632 220.565 220.13 221 219.64 0.695 0.0181353141182 0.0181353141182 1453754 82.0243866176 1453754 1453696 1453812 988355 988355 0.235642194748 0.235642194748 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 575.54 2.07889393669 575.54 574.07 577.01 573.66 1.265 0.0165701869164 0.0165701869164 2646600 5221.27647228 2646600 2642908 2650292 2270248 2270248 0.2270248 0.2270248 1 +compress ppmd expert ppmd compress:ppmd 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.01 0 0.01 0.01 0.01 0.005 0 0.390625 0.390625 7262 42.4264068712 7262 7232 7292 1058 1058 0.25830078125 0.25830078125 1 +compress ppmd expert ppmd compress:ppmd 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.04 0 0.04 0.04 0.04 0.04 0 0.390625 0.390625 15978 93.3380951166 15978 15912 16044 6267 6267 0.382507324219 0.382507324219 1 +compress ppmd expert ppmd compress:ppmd 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.21 0 0.21 0.21 0.21 0.19 0.01 0.297619047619 0.297619047619 45062 98.9949493661 45062 44992 45132 23186 23186 0.353790283203 0.353790283203 1 +compress ppmd expert ppmd compress:ppmd 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 1.01 0.0141421356237 1.01 1 1.02 0.965 0.04 0.247549019608 0.247549019608 143994 189.504617358 143994 143860 144128 83269 83269 0.317646026611 0.317646026611 1 +compress ppmd expert ppmd compress:ppmd 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 4.195 0.00707106781187 4.195 4.19 4.2 4.015 0.17 0.238379361291 0.238379361291 399382 104.651803616 399382 399308 399456 326331 326331 0.311213493347 0.311213493347 1 +compress ppmd expert ppmd compress:ppmd 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 8.16 0.0141421356237 8.16 8.15 8.17 7.94 0.205 0.245098407311 0.245098407311 457658 14.1421356237 457658 457648 457668 661953 661953 0.315643787384 0.315643787384 1 +compress ppmd expert ppmd compress:ppmd 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 16.215 0.0494974746831 16.215 16.18 16.25 15.98 0.215 0.246686317391 0.246686317391 459716 339.41125497 459716 459476 459956 1329257 1329257 0.316919565201 0.316919565201 1 +compress ppmd expert ppmd compress:ppmd 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 39.19 0.0565685424949 39.19 39.15 39.23 38.89 0.265 0.243346596048 0.243346596048 568906 31.1126983722 568906 568884 568928 3154465 3154465 0.3154465 0.3154465 1 +compress rosa expert rosaplus compress:rosa 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0 0 0 0 0 0 0 5736 56.5685424949 5736 5696 5776 1127 1127 0.275146484375 0.275146484375 1 +compress rosa expert rosaplus compress:rosa 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.02 0 0.02 0.02 0.02 0.02 0 0.78125 0.78125 8320 113.13708499 8320 8240 8400 6359 6359 0.388122558594 0.388122558594 1 +compress rosa expert rosaplus compress:rosa 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.12 0 0.12 0.12 0.12 0.115 0.005 0.520833333333 0.520833333333 17588 186.676190233 17588 17456 17720 22843 22843 0.348556518555 0.348556518555 1 +compress rosa expert rosaplus compress:rosa 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 0.695 0.00707106781187 0.695 0.69 0.7 0.68 0.01 0.359730848861 0.359730848861 55486 104.651803616 55486 55412 55560 80590 80590 0.307426452637 0.307426452637 1 +compress rosa expert rosaplus compress:rosa 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 3.84 0.0141421356237 3.84 3.83 3.85 3.78 0.05 0.260418432742 0.260418432742 146162 59.3969696197 146162 146120 146204 306522 306522 0.292322158813 0.292322158813 1 +compress rosa expert rosaplus compress:rosa 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 9.32 0.0141421356237 9.32 9.31 9.33 9.175 0.135 0.214592521727 0.214592521727 284848 67.8822509939 284848 284800 284896 608681 608681 0.290241718292 0.290241718292 1 +compress rosa expert rosaplus compress:rosa 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 23.395 0.106066017178 23.395 23.32 23.47 23.145 0.225 0.17097846161 0.17097846161 564220 22.627416998 564220 564204 564236 1199345 1199345 0.285946130753 0.285946130753 1 +compress rosa expert rosaplus compress:rosa 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 73.8 0.0848528137424 73.8 73.74 73.86 73.085 0.64 0.12922424753 0.12922424753 1306208 50.9116882454 1306208 1306172 1306244 2752778 2752778 0.2752778 0.2752778 1 +compress rwkv expert rwkv compress:rwkv 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.06 0 0.06 0.06 0.06 0.06 0 0.0651041666667 0.0651041666667 7710 65.0538238692 7710 7664 7756 3714 3714 0.90673828125 0.90673828125 1 +compress rwkv expert rwkv compress:rwkv 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.245 0.00707106781187 0.245 0.24 0.25 0.24 0 0.0638020833333 0.0638020833333 7666 36.7695526217 7666 7640 7692 12000 12000 0.732421875 0.732421875 1 +compress rwkv expert rwkv compress:rwkv 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.96 0.0282842712475 0.96 0.94 0.98 0.955 0 0.0651324359531 0.0651324359531 7722 104.651803616 7722 7648 7796 35340 35340 0.539245605469 0.539245605469 1 +compress rwkv expert rwkv compress:rwkv 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 3.81 0.0424264068712 3.81 3.78 3.84 3.805 0 0.0656208664021 0.0656208664021 7872 107.48023074 7872 7796 7948 137010 137010 0.522651672363 0.522651672363 1 +compress rwkv expert rwkv compress:rwkv 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 15.805 0.346482322781 15.805 15.56 16.05 15.71 0.085 0.0632863240676 0.0632863240676 9256 96.1665222414 9256 9188 9324 474275 474275 0.452303886414 0.452303886414 1 +compress rwkv expert rwkv compress:rwkv 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 32.195 0.0212132034356 32.195 32.18 32.21 31.66 0.5 0.0621214609146 0.0621214609146 11250 189.504617358 11250 11116 11384 917086 917086 0.437300682068 0.437300682068 1 +compress rwkv expert rwkv compress:rwkv 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 66.09 0.141421356237 66.09 65.99 66.19 64.365 1.66 0.0605236670868 0.0605236670868 14946 48.0832611207 14946 14912 14980 1748887 1748887 0.416967153549 0.416967153549 1 +compress rwkv expert rwkv compress:rwkv 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 155.31 0.537401153702 155.31 154.93 155.69 153.32 1.86 0.0614049337165 0.0614049337165 24942 149.906637612 24942 24836 25048 3968645 3968645 0.3968645 0.3968645 1 +decompress ctw expert ctw decompress:ctw 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.03 0 0.03 0.03 0.03 0.03 0 0.130208333333 0.130208333333 6874 115.965512115 6874 6792 6956 1332 1332 0.3251953125 0.3251953125 1 +decompress ctw expert ctw decompress:ctw 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.18 0 0.18 0.18 0.18 0.18 0 0.0868055555556 0.0868055555556 12706 70.7106781187 12706 12656 12756 6579 6579 0.401550292969 0.401550292969 1 +decompress ctw expert ctw decompress:ctw 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.815 0.00707106781187 0.815 0.81 0.82 0.805 0.005 0.0766900030111 0.0766900030111 22796 67.8822509939 22796 22748 22844 22728 22728 0.346801757812 0.346801757812 1 +decompress ctw expert ctw decompress:ctw 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 3.525 0.00707106781187 3.525 3.52 3.53 3.5 0.02 0.0709221285089 0.0709221285089 49508 62.2253967444 49508 49464 49552 80004 80004 0.305191040039 0.305191040039 1 +decompress ctw expert ctw decompress:ctw 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 15.89 0.0141421356237 15.89 15.88 15.9 15.82 0.05 0.0629326869762 0.0629326869762 122218 31.1126983722 122218 122196 122240 301713 301713 0.287735939026 0.287735939026 1 +decompress ctw expert ctw decompress:ctw 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 33.545 0.00707106781186 33.545 33.54 33.55 33.405 0.11 0.0596214054087 0.0596214054087 199622 48.0832611207 199622 199588 199656 597003 597003 0.284673213959 0.284673213959 1 +decompress ctw expert ctw decompress:ctw 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 70.815 0.00707106781186 70.815 70.81 70.82 70.605 0.15 0.0564852082178 0.0564852082178 327672 45.2548339959 327672 327640 327704 1177962 1177962 0.280848026276 0.280848026276 1 +decompress ctw expert ctw decompress:ctw 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 179.705 0.0919238815543 179.705 179.64 179.77 179.205 0.33 0.0530688874085 0.0530688874085 611636 90.5096679919 611636 611572 611700 2746861 2746861 0.2746861 0.2746861 1 +decompress match expert match decompress:match 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0 0 0 0 0 0 0 5498 42.4264068712 5498 5468 5528 2874 2874 0.70166015625 0.70166015625 1 +decompress match expert match decompress:match 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.01 0 0.01 0.01 0.01 0.01 0 1.5625 1.5625 5642 144.249783362 5642 5540 5744 13506 13506 0.824340820312 0.824340820312 1 +decompress match expert match decompress:match 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.03 0 0.03 0.03 0.03 0.03 0 2.08333333333 2.08333333333 6314 59.3969696197 6314 6272 6356 53748 53748 0.820129394531 0.820129394531 1 +decompress match expert match decompress:match 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 0.14 0 0.14 0.14 0.14 0.135 0 1.78571428571 1.78571428571 8110 36.7695526217 8110 8084 8136 206203 206203 0.786602020264 0.786602020264 1 +decompress match expert match decompress:match 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 0.56 0 0.56 0.56 0.56 0.56 0 1.78571428571 1.78571428571 11610 2.82842712475 11610 11608 11612 817505 817505 0.779633522034 0.779633522034 1 +decompress match expert match decompress:match 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 1.125 0.00707106781187 1.125 1.12 1.13 1.12 0 1.77781289507 1.77781289507 20102 115.965512115 20102 20020 20184 1644756 1644756 0.784280776978 0.784280776978 1 +decompress match expert match decompress:match 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 2.3 0.0141421356237 2.3 2.29 2.31 2.29 0 1.73916331122 1.73916331122 23962 161.220346111 23962 23848 24076 3292751 3292751 0.785053014755 0.785053014755 1 +decompress match expert match decompress:match 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 5.45 0.0141421356237 5.45 5.44 5.46 5.425 0.015 1.74986702233 1.74986702233 46162 76.3675323681 46162 46108 46216 7859324 7859324 0.7859324 0.7859324 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.13 0 0.13 0.13 0.13 0.125 0 0.0300480769231 0.0300480769231 11858 48.0832611207 11858 11824 11892 1001 1001 0.244384765625 0.244384765625 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.565 0.00707106781187 0.565 0.56 0.57 0.555 0.005 0.027657033208 0.027657033208 29926 8.48528137424 29926 29920 29932 5533 5533 0.337707519531 0.337707519531 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 2.5 0.0141421356237 2.5 2.49 2.51 2.465 0.025 0.0250004000064 0.0250004000064 80690 82.0243866176 80690 80632 80748 19344 19344 0.295166015625 0.295166015625 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 10.935 0.0494974746831 10.935 10.9 10.97 10.825 0.09 0.0228626027615 0.0228626027615 259300 22.627416998 259300 259284 259316 67454 67454 0.257316589355 0.257316589355 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 48.96 0.0141421356237 48.96 48.95 48.97 48.595 0.315 0.0204248374534 0.0204248374534 701542 104.651803616 701542 701468 701616 254251 254251 0.242472648621 0.242472648621 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 104.045 0.0777817459305 104.045 103.99 104.1 103.49 0.44 0.0192224571952 0.0192224571952 1013456 16.9705627485 1013456 1013444 1013468 503520 503520 0.240097045898 0.240097045898 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 221.8 0.381837661841 221.8 221.53 222.07 220.84 0.72 0.0180342918278 0.0180342918278 1471378 161.220346111 1471378 1471264 1471492 988355 988355 0.235642194748 0.235642194748 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 574.545 0.997020561473 574.545 573.84 575.25 572.645 1.265 0.0165987999605 0.0165987999605 2655396 33.941125497 2655396 2655372 2655420 2270248 2270248 0.2270248 0.2270248 1 +decompress ppmd expert ppmd decompress:ppmd 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.01 0 0.01 0.01 0.01 0.005 0 0.390625 0.390625 7364 11.313708499 7364 7356 7372 1058 1058 0.25830078125 0.25830078125 1 +decompress ppmd expert ppmd decompress:ppmd 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.04 0 0.04 0.04 0.04 0.03 0 0.390625 0.390625 16078 36.7695526217 16078 16052 16104 6267 6267 0.382507324219 0.382507324219 1 +decompress ppmd expert ppmd decompress:ppmd 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.21 0 0.21 0.21 0.21 0.185 0.02 0.297619047619 0.297619047619 45074 8.48528137424 45074 45068 45080 23186 23186 0.353790283203 0.353790283203 1 +decompress ppmd expert ppmd decompress:ppmd 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 1.02 0.0141421356237 1.02 1.01 1.03 0.945 0.07 0.245121599539 0.245121599539 144204 90.5096679919 144204 144140 144268 83269 83269 0.317646026611 0.317646026611 1 +decompress ppmd expert ppmd decompress:ppmd 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 4.245 0.0212132034356 4.245 4.23 4.26 4.06 0.175 0.235574201711 0.235574201711 399112 16.9705627485 399112 399100 399124 326331 326331 0.311213493347 0.311213493347 1 +decompress ppmd expert ppmd decompress:ppmd 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 8.25 0 8.25 8.25 8.25 8.035 0.205 0.242424242424 0.242424242424 457468 158.391918986 457468 457356 457580 661953 661953 0.315643787384 0.315643787384 1 +decompress ppmd expert ppmd decompress:ppmd 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 16.285 0.0353553390593 16.285 16.26 16.31 16.05 0.21 0.245625386971 0.245625386971 458390 121.622366364 458390 458304 458476 1329257 1329257 0.316919565201 0.316919565201 1 +decompress ppmd expert ppmd decompress:ppmd 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 39.55 0.0707106781187 39.55 39.5 39.6 39.225 0.285 0.241131691688 0.241131691688 558980 124.450793489 558980 558892 559068 3154465 3154465 0.3154465 0.3154465 1 +decompress rosa expert rosaplus decompress:rosa 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0 0 0 0 0 0 0 6016 33.941125497 6016 5992 6040 1127 1127 0.275146484375 0.275146484375 1 +decompress rosa expert rosaplus decompress:rosa 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.02 0 0.02 0.02 0.02 0.02 0 0.78125 0.78125 8344 62.2253967444 8344 8300 8388 6359 6359 0.388122558594 0.388122558594 1 +decompress rosa expert rosaplus decompress:rosa 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.13 0 0.13 0.13 0.13 0.115 0.005 0.480769230769 0.480769230769 17808 79.1959594929 17808 17752 17864 22843 22843 0.348556518555 0.348556518555 1 +decompress rosa expert rosaplus decompress:rosa 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 0.7 0 0.7 0.7 0.7 0.69 0.005 0.357142857143 0.357142857143 55634 42.4264068712 55634 55604 55664 80590 80590 0.307426452637 0.307426452637 1 +decompress rosa expert rosaplus decompress:rosa 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 3.855 0.00707106781187 3.855 3.85 3.86 3.8 0.045 0.259403808627 0.259403808627 146626 31.1126983722 146626 146604 146648 306522 306522 0.292322158813 0.292322158813 1 +decompress rosa expert rosaplus decompress:rosa 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 9.395 0.0636396103068 9.395 9.35 9.44 9.285 0.095 0.212884075048 0.212884075048 285428 118.793939239 285428 285344 285512 608681 608681 0.290241718292 0.290241718292 1 +decompress rosa expert rosaplus decompress:rosa 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 23.965 0.671751442127 23.965 23.49 24.44 23.72 0.215 0.166975674435 0.166975674435 553182 31.1126983722 553182 553160 553204 1199345 1199345 0.285946130753 0.285946130753 1 +decompress rosa expert rosaplus decompress:rosa 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 74.56 0.0282842712475 74.56 74.54 74.58 73.81 0.675 0.127906972241 0.127906972241 1304464 56.5685424949 1304464 1304424 1304504 2752778 2752778 0.2752778 0.2752778 1 +decompress rwkv expert rwkv decompress:rwkv 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.06 0 0.06 0.06 0.06 0.06 0 0.0651041666667 0.0651041666667 7650 25.4558441227 7650 7632 7668 3714 3714 0.90673828125 0.90673828125 1 +decompress rwkv expert rwkv decompress:rwkv 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.24 0 0.24 0.24 0.24 0.235 0 0.0651041666667 0.0651041666667 7840 33.941125497 7840 7816 7864 12000 12000 0.732421875 0.732421875 1 +decompress rwkv expert rwkv decompress:rwkv 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.945 0.00707106781187 0.945 0.94 0.95 0.935 0 0.0661394176932 0.0661394176932 7906 8.48528137424 7906 7900 7912 35340 35340 0.539245605469 0.539245605469 1 +decompress rwkv expert rwkv decompress:rwkv 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 3.815 0.0494974746831 3.815 3.78 3.85 3.81 0 0.0655363155363 0.0655363155363 7732 84.8528137424 7732 7672 7792 137010 137010 0.522651672363 0.522651672363 1 +decompress rwkv expert rwkv decompress:rwkv 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 15.78 0.0424264068712 15.78 15.75 15.81 15.77 0 0.0633715851932 0.0633715851932 8930 25.4558441227 8930 8912 8948 474275 474275 0.452303886414 0.452303886414 1 +decompress rwkv expert rwkv decompress:rwkv 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 31.655 0.30405591591 31.655 31.44 31.87 31.63 0 0.0631840867519 0.0631840867519 10342 8.48528137424 10342 10336 10348 917086 917086 0.437300682068 0.437300682068 1 +decompress rwkv expert rwkv decompress:rwkv 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 63.89 0.0424264068712 63.89 63.86 63.92 63.84 0 0.0626076206282 0.0626076206282 13160 16.9705627485 13160 13148 13172 1748887 1748887 0.416967153549 0.416967153549 1 +decompress rwkv expert rwkv decompress:rwkv 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 153.94 0.311126983722 153.94 153.72 154.16 153.82 0.01 0.0619511669613 0.0619511669613 20890 25.4558441227 20890 20872 20908 3968645 3968645 0.3968645 0.3968645 1 diff --git a/benchmarks/6f464811/infotheory-two-json-summary-full.tsv b/benchmarks/6f464811/infotheory-two-json-summary-full.tsv index aa1aaaf0..cf83d466 100644 --- a/benchmarks/6f464811/infotheory-two-json-summary-full.tsv +++ b/benchmarks/6f464811/infotheory-two-json-summary-full.tsv @@ -1,145 +1,145 @@ -operation subject subject_kind expert_kind series size_bytes repeats cpu compression_backend input_sha256 real_seconds_mean real_seconds_stdev real_seconds_median real_seconds_min real_seconds_max user_seconds_mean sys_seconds_mean throughput_mib_s_mean throughput_mib_s_median rss_kib_mean rss_kib_stdev rss_kib_median rss_kib_min rss_kib_max archive_bytes_mean archive_bytes_median archive_ratio_mean archive_ratio_median entropy_bpb_mean entropy_bpb_median verified_all -h ctw expert ctw h:ctw 4096 3 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.04 0 0.04 0.04 0.04 0.03 0 0.09765625 0.09765625 13386.6666667 80.133222407 13392 13304 13464 2.56549624542 2.56549624542 1 -h ctw expert ctw h:ctw 16384 3 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.176666666667 0.0057735026919 0.18 0.17 0.18 0.16 0.01 0.0885076252723 0.0868055555556 40813.3333333 9.23760430703 40808 40808 40824 3.20340951209 3.20340951209 1 -h ctw expert ctw h:ctw 65536 3 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.776666666667 0.0057735026919 0.78 0.77 0.78 0.743333333333 0.0333333333333 0.0804750804751 0.0801282051282 92276 68.2348884369 92256 92220 92352 2.77213895634 2.77213895634 1 -h ctw expert ctw h:ctw 262144 3 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 3.27666666667 0.0057735026919 3.28 3.27 3.28 3.19666666667 0.07 0.0762972079262 0.0762195121951 221774.666667 64.1664502161 221780 221708 221836 2.44096714152 2.44096714152 1 -h ctw expert ctw h:ctw 1048576 3 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 14.4566666667 0.0115470053838 14.45 14.45 14.47 14.23 0.206666666667 0.0691722682813 0.0692041522491 552056 80.8949936646 552088 551964 552116 2.30174274755 2.30174274755 1 -h ctw expert ctw h:ctw 2097152 3 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 30.5533333333 0.0404145188433 30.53 30.53 30.6 30.1633333333 0.356666666667 0.0654593824282 0.0655093350802 897202.666667 119.82209034 897168 897104 897336 2.27731545661 2.27731545661 1 -h ctw expert ctw h:ctw 4194304 3 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 64.7966666667 0.0152752523165 64.8 64.78 64.81 64.11 0.616666666667 0.0617315728413 0.0617283950617 1472845.33333 73.3575717519 1472820 1472788 1472928 2.24674930722 2.24674930722 1 -h ctw expert ctw h:ctw 10000000 3 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 165.963333333 0.175594229214 165.98 165.78 166.13 164.59 1.21 0.0574629955347 0.0574571825766 2773640 64.3739077577 2773648 2773572 2773700 2.19747398481 2.19747398481 1 -h match expert match h:match 4096 3 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0 0 0 0 0 0 0 5720 22.2710574513 5724 5696 5740 5.57734993252 5.57734993252 1 -h match expert match h:match 16384 3 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0 0 0 0 0 0 0 5665.33333333 154.108187107 5672 5508 5816 6.58586286407 6.58586286407 1 -h match expert match h:match 65536 3 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.01 0 0.01 0.01 0.01 0.00666666666667 0 6.25 6.25 6684 158.745078664 6624 6564 6864 6.5588255305 6.5588255305 1 -h match expert match h:match 262144 3 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 0.03 0 0.03 0.03 0.03 0.03 0 8.33333333333 8.33333333333 7868 69.7423830967 7836 7820 7948 6.29225944669 6.29225944669 1 -h match expert match h:match 1048576 3 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 0.133333333333 0.0057735026919 0.13 0.13 0.14 0.13 0 7.50915750916 7.69230769231 11425.3333333 135.784142422 11400 11304 11572 6.23692635511 6.23692635511 1 -h match expert match h:match 2097152 3 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 0.27 0 0.27 0.27 0.27 0.263333333333 0 7.40740740741 7.40740740741 19253.3333333 149.255932322 19208 19132 19420 6.2741772649 6.2741772649 1 -h match expert match h:match 4194304 3 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 0.53 0 0.53 0.53 0.53 0.52 0 7.54716981132 7.54716981132 21278.6666667 76.0350796234 21276 21204 21356 6.28038800899 6.28038800899 1 -h match expert match h:match 10000000 3 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 1.30666666667 0.0057735026919 1.31 1.3 1.31 1.28666666667 0.0133333333333 7.29862316921 7.27995661379 40453.3333333 27.2274371422 40444 40432 40484 6.28744436593 6.28744436593 1 -h neural_mixture mixture neural-mixture h:neural_mixture 4096 3 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.186666666667 0.0057735026919 0.19 0.18 0.19 0.18 0 0.0209399366472 0.0205592105263 19389.3333333 35.8515457593 19408 19348 19412 1.91856494114 1.91856494114 1 -h neural_mixture mixture neural-mixture h:neural_mixture 16384 3 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.806666666667 0.0115470053838 0.8 0.8 0.82 0.79 0.0133333333333 0.0193724593496 0.01953125 59222.6666667 121.15004471 59276 59084 59308 2.69245376114 2.69245376114 1 -h neural_mixture mixture neural-mixture h:neural_mixture 65536 3 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 3.44333333333 0.0057735026919 3.44 3.44 3.45 3.38 0.0533333333333 0.0181510504438 0.0181686046512 161166.666667 71.7030915187 161204 161084 161212 2.35901503943 2.35901503943 1 -h neural_mixture mixture neural-mixture h:neural_mixture 262144 3 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 14.71 0.01 14.71 14.7 14.72 14.5133333333 0.17 0.0169952465686 0.0169952413324 484545.333333 78.4176850802 484536 484472 484628 2.05794860665 2.05794860665 1 -h neural_mixture mixture neural-mixture h:neural_mixture 1048576 3 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 63.8266666667 0.0585946527708 63.85 63.76 63.87 63.2466666667 0.513333333333 0.0156674414369 0.0156617071261 1184440 69.3974062916 1184456 1184364 1184500 1.93962973278 1.93962973278 1 -h neural_mixture mixture neural-mixture h:neural_mixture 2097152 3 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 133.123333333 0.309246395829 132.96 132.93 133.48 132.146666667 0.84 0.0150237162454 0.0150421179302 1841770.66667 180.236881168 1841760 1841596 1841956 1.92069255467 1.92069255467 1 -h neural_mixture mixture neural-mixture h:neural_mixture 4194304 3 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 277.44 0.783134726596 277.31 276.73 278.28 275.99 1.17333333333 0.0144176082502 0.0144242905052 2806732 189.145446681 2806652 2806596 2806948 1.88509131394 1.88509131394 1 -h neural_mixture mixture neural-mixture h:neural_mixture 10000000 3 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 698.953333333 0.898906743402 698.48 698.39 699.99 695.94 2.30666666667 0.0136443353473 0.0136535665503 5058941.33333 164.406001513 5058928 5058784 5059112 1.81617083096 1.81617083096 1 -h ppmd expert ppmd h:ppmd 4096 3 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0 0 0 0 0 0 0 7326.66666667 53.2666249478 7340 7268 7372 2.02915198126 2.02915198126 1 -h ppmd expert ppmd h:ppmd 16384 3 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.03 0 0.03 0.03 0.03 0.0266666666667 0 0.520833333333 0.520833333333 16073.3333333 52.2047252012 16056 16032 16132 3.05111195443 3.05111195443 1 -h ppmd expert ppmd h:ppmd 65536 3 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.17 0 0.17 0.17 0.17 0.16 0.00666666666667 0.367647058824 0.367647058824 45270.6666667 80.0333263918 45268 45192 45352 2.82799347281 2.82799347281 1 -h ppmd expert ppmd h:ppmd 262144 3 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 0.836666666667 0.0057735026919 0.84 0.83 0.84 0.77 0.0566666666667 0.298814304838 0.297619047619 144156 116.824654932 144172 144032 144264 2.54058234497 2.54058234497 1 -h ppmd expert ppmd h:ppmd 1048576 3 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 3.52333333333 0.0057735026919 3.52 3.52 3.53 3.33333333333 0.18 0.283822645721 0.284090909091 398886.666667 30.2875111776 398900 398852 398908 2.489549756 2.489549756 1 -h ppmd expert ppmd h:ppmd 2097152 3 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 6.86333333333 0.0929157324318 6.82 6.8 6.97 6.63 0.22 0.291438941645 0.293255131965 457068 99.518842437 457088 456960 457156 2.52506560801 2.52506560801 1 -h ppmd expert ppmd h:ppmd 4194304 3 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 13.56 0.0854400374532 13.57 13.47 13.64 13.3166666667 0.22 0.294993067076 0.294767870302 459240 134.044768641 459236 459108 459376 2.53530592864 2.53530592864 1 -h ppmd expert ppmd h:ppmd 10000000 3 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 32.9466666667 0.283607710285 32.83 32.74 33.27 32.63 0.276666666667 0.289474268395 0.290488673898 557333.333333 121.083992886 557352 557204 557444 2.5235430657 2.5235430657 1 -h rosa expert rosaplus h:rosa 4096 3 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.02 0 0.02 0.02 0.02 0.02 0 0.1953125 0.1953125 5889.33333333 140.873465682 5952 5728 5988 2.11172124286 2.11172124286 1 -h rosa expert rosaplus h:rosa 16384 3 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.12 0 0.12 0.12 0.12 0.113333333333 0.00333333333333 0.130208333333 0.130208333333 8101.33333333 51.4328040586 8080 8064 8160 3.48630622986 3.48630622986 1 -h rosa expert rosaplus h:rosa 65536 3 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.603333333333 0.0057735026919 0.6 0.6 0.61 0.573333333333 0.0233333333333 0.103597449909 0.104166666667 16697.3333333 98.006802485 16696 16600 16796 3.19056696201 3.19056696201 1 -h rosa expert rosaplus h:rosa 262144 3 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 3.49666666667 0.0057735026919 3.5 3.49 3.5 3.38666666667 0.0966666666667 0.0714967935598 0.0714285714286 46952 90.0666419936 46900 46900 47056 2.90567254487 2.90567254487 1 -h rosa expert rosaplus h:rosa 1048576 3 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 20.5766666667 0.0057735026919 20.58 20.57 20.58 19.7833333333 0.76 0.048598738984 0.0485908649174 203844 47.1593044902 203832 203804 203896 2.80123027825 2.80123027825 1 -h rosa expert rosaplus h:rosa 2097152 3 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 49.1166666667 0.0550757054729 49.09 49.08 49.18 47.1133333333 1.94 0.0407194097478 0.0407414952129 337701.333333 64.2910050733 337728 337628 337748 2.70437138702 2.70437138702 1 -h rosa expert rosaplus h:rosa 4194304 3 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 117.806666667 0.117189305542 117.76 117.72 117.94 112.6 5.06666666667 0.0339539582152 0.0339673913043 643536 72.9931503636 643572 643452 643584 2.62099230176 2.62099230176 1 -h rosa expert rosaplus h:rosa 10000000 3 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 346.986666667 1.73367624813 346.15 345.83 348.98 333.436666667 13.1733333333 0.0274849219272 0.0275508974839 1551498.66667 175.377687672 1551524 1551312 1551660 2.48481897791 2.48481897791 1 -h rwkv expert rwkv h:rwkv 4096 3 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.06 0 0.06 0.06 0.06 0.0533333333333 0 0.0651041666667 0.0651041666667 8030.66666667 31.0698138606 8040 7996 8056 7.21701437947 7.21701437947 1 -h rwkv expert rwkv h:rwkv 16384 3 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.226666666667 0.0057735026919 0.23 0.22 0.23 0.226666666667 0 0.0689640974967 0.0679347826087 8052 77.1492060879 8020 7996 8140 5.85021072042 5.85021072042 1 -h rwkv expert rwkv h:rwkv 65536 3 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.906666666667 0.0115470053838 0.9 0.9 0.92 0.903333333333 0 0.0689412238325 0.0694444444444 8273.33333333 107.802288164 8296 8156 8368 4.31173851819 4.31173851819 1 -h rwkv expert rwkv h:rwkv 262144 3 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 3.63333333333 0.0152752523165 3.63 3.62 3.65 3.62666666667 0 0.0688081491939 0.068870523416 8264 69.2820323028 8304 8184 8304 4.18065126521 4.18065126521 1 -h rwkv expert rwkv h:rwkv 1048576 3 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 15.0133333333 0.136137185711 15.06 14.86 15.12 15 0 0.0666111265213 0.066401062417 8833.33333333 51.4328040586 8812 8796 8892 3.61829048151 3.61829048151 1 -h rwkv expert rwkv h:rwkv 2097152 3 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 30.2566666667 0.251064400769 30.32 29.98 30.47 30.23 0 0.0661041780776 0.065963060686 9836 155.897402159 9808 9696 10004 3.49833649966 3.49833649966 1 -h rwkv expert rwkv h:rwkv 4194304 3 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 61.0533333333 0.0351188458428 61.05 61.02 61.09 61.01 0 0.0655165027668 0.0655200655201 11856 154.350251053 11844 11708 12016 3.33570098711 3.33570098711 1 -h rwkv expert rwkv h:rwkv 10000000 3 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 147.966666667 1.31241507662 148.41 146.49 149 147.856666667 0 0.0644553640807 0.0642594378011 17626.6666667 43.1431724996 17608 17596 17676 3.17490278117 3.17490278117 1 -compress ctw expert ctw compress:ctw 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.05 0 0.05 0.05 0.05 0.0433333333333 0.00333333333333 0.078125 0.078125 13432 48.4974226119 13440 13380 13476 1332 1332 0.3251953125 0.3251953125 1 -compress ctw expert ctw compress:ctw 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.23 0 0.23 0.23 0.23 0.22 0 0.0679347826087 0.0679347826087 40992 139.484766193 41056 40832 41088 6579 6579 0.401550292969 0.401550292969 1 -compress ctw expert ctw compress:ctw 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 1.01 0.01 1.01 1 1.02 0.98 0.0233333333333 0.0618852326409 0.0618811881188 92406.6666667 64.6632301492 92396 92348 92476 22728 22728 0.346801757812 0.346801757812 1 -compress ctw expert ctw compress:ctw 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 4.20666666667 0.0115470053838 4.2 4.2 4.22 4.11666666667 0.0733333333333 0.0594297750696 0.0595238095238 221974.666667 126.258993079 221984 221844 222096 80004 80004 0.305191040039 0.305191040039 1 -compress ctw expert ctw compress:ctw 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 17.9866666667 0.0057735026919 17.99 17.98 17.99 17.76 0.203333333333 0.0555967421443 0.0555864369094 552538.666667 6.11010092661 552540 552532 552544 301713 301713 0.287735939026 0.287735939026 1 -compress ctw expert ctw compress:ctw 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 37.36 0.0346410161514 37.34 37.34 37.4 36.95 0.38 0.0535332212449 0.0535618639529 898394.666667 96.0277737602 898356 898324 898504 597003 597003 0.284673213959 0.284673213959 1 -compress ctw expert ctw compress:ctw 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 77.9033333333 0.0416333199893 77.89 77.87 77.95 77.1933333333 0.636666666667 0.0513456945973 0.0513544742586 1475212 85.5102333057 1475240 1475116 1475280 1177962 1177962 0.280848026276 0.280848026276 1 -compress ctw expert ctw compress:ctw 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 197.076666667 0.210317220725 197.09 196.86 197.28 195.666666667 1.23 0.0483910681408 0.0483877576948 2779017.33333 89.9184816005 2778996 2778940 2779116 2746861 2746861 0.2746861 0.2746861 1 -compress match expert match compress:match 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0 0 0 0 0 0 0 5464 139.312598138 5508 5308 5576 2874 2874 0.70166015625 0.70166015625 1 -compress match expert match compress:match 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.02 0 0.02 0.02 0.02 0.0166666666667 0 0.78125 0.78125 5546.66666667 63.7913264742 5516 5504 5620 13506 13506 0.824340820312 0.824340820312 1 -compress match expert match compress:match 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.07 0 0.07 0.07 0.07 0.07 0 0.892857142857 0.892857142857 6438.66666667 50.3322295685 6432 6392 6492 53748 53748 0.820129394531 0.820129394531 1 -compress match expert match compress:match 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 0.296666666667 0.0057735026919 0.3 0.29 0.3 0.293333333333 0 0.842911877395 0.833333333333 8230.66666667 48.8808074129 8220 8188 8284 206203 206203 0.786602020264 0.786602020264 1 -compress match expert match compress:match 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 1.17666666667 0.0057735026919 1.18 1.17 1.18 1.17666666667 0 0.849872036313 0.847457627119 12108 16 12108 12092 12124 817505 817505 0.779633522034 0.779633522034 1 -compress match expert match compress:match 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 2.36666666667 0.0208166599947 2.36 2.35 2.39 2.36 0.00333333333333 0.845113846863 0.847457627119 22012 107.628992377 22044 21892 22100 1644756 1644756 0.784280776978 0.784280776978 1 -compress match expert match compress:match 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 4.72333333333 0.0115470053838 4.73 4.71 4.73 4.70333333333 0.00666666666667 0.846862941367 0.845665961945 28905.3333333 64.2910050733 28932 28832 28952 3292751 3292751 0.785053014755 0.785053014755 1 -compress match expert match compress:match 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 11.31 0.03 11.31 11.28 11.34 11.2766666667 0.0233333333333 0.843217320706 0.843213365523 55496 116.824654932 55512 55372 55604 7859324 7859324 0.7859324 0.7859324 1 -compress neural_mixture mixture neural-mixture compress:neural_mixture 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.15 0 0.15 0.15 0.15 0.146666666667 0.00333333333333 0.0260416666667 0.0260416666667 19533.3333333 91.2432645916 19516 19452 19632 1001 1001 0.244384765625 0.244384765625 1 -compress neural_mixture mixture neural-mixture compress:neural_mixture 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.673333333333 0.0057735026919 0.67 0.67 0.68 0.646666666667 0.02 0.0232065774071 0.0233208955224 58921.3333333 100.5849558 58880 58848 59036 5533 5533 0.337707519531 0.337707519531 1 -compress neural_mixture mixture neural-mixture compress:neural_mixture 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 2.88 0.01 2.88 2.87 2.89 2.81666666667 0.0533333333333 0.021701563317 0.0217013888889 165472 62.3538290725 165436 165436 165544 19344 19344 0.295166015625 0.295166015625 1 -compress neural_mixture mixture neural-mixture compress:neural_mixture 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 12.4633333333 0.0251661147842 12.46 12.44 12.49 12.26 0.186666666667 0.0200588937634 0.0200642054575 471460 169.846989964 471368 471356 471656 67454 67454 0.257316589355 0.257316589355 1 -compress neural_mixture mixture neural-mixture compress:neural_mixture 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 55.08 0.0854400374532 55.09 54.99 55.16 54.5 0.516666666667 0.0181554394441 0.0181521147214 1184602.66667 46.3609030686 1184596 1184560 1184652 254251 254251 0.242472648621 0.242472648621 1 -compress neural_mixture mixture neural-mixture compress:neural_mixture 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 115.833333333 0.0642910050733 115.86 115.76 115.88 114.92 0.793333333333 0.0172661905974 0.0172622130157 1834400 32 1834400 1834368 1834432 503519 503519 0.240096569061 0.240096569061 1 -compress neural_mixture mixture neural-mixture compress:neural_mixture 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 244.6 0.113578166916 244.55 244.52 244.73 243.136666667 1.21333333333 0.016353232113 0.0163565732979 2763934.66667 85.5414129725 2763980 2763836 2763988 988355 988355 0.235642194748 0.235642194748 1 -compress neural_mixture mixture neural-mixture compress:neural_mixture 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 628.033333333 2.47508249021 626.83 626.39 630.88 625.04 2.34666666667 0.0151852476444 0.0152142417626 5066150.66667 96.4434203735 5066140 5066060 5066252 2270248 2270248 0.2270248 0.2270248 1 -compress ppmd expert ppmd compress:ppmd 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.01 0 0.01 0.01 0.01 0.01 0 0.390625 0.390625 7142.66666667 23.0940107676 7156 7116 7156 1058 1058 0.25830078125 0.25830078125 1 -compress ppmd expert ppmd compress:ppmd 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.05 0 0.05 0.05 0.05 0.0466666666667 0 0.3125 0.3125 16113.3333333 95.0228042805 16160 16004 16176 6267 6267 0.382507324219 0.382507324219 1 -compress ppmd expert ppmd compress:ppmd 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.243333333333 0.0057735026919 0.24 0.24 0.25 0.22 0.0166666666667 0.256944444444 0.260416666667 44936 158.037970121 44872 44820 45116 23186 23186 0.353790283203 0.353790283203 1 -compress ppmd expert ppmd compress:ppmd 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 1.12666666667 0.0057735026919 1.13 1.12 1.13 1.06333333333 0.0566666666667 0.221897387273 0.221238938053 144042.666667 25.7164020293 144032 144024 144072 83269 83269 0.317646026611 0.317646026611 1 -compress ppmd expert ppmd compress:ppmd 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 4.67666666667 0.0152752523165 4.68 4.66 4.69 4.48 0.186666666667 0.213829034853 0.213675213675 399384 108.885260711 399428 399260 399464 326331 326331 0.311213493347 0.311213493347 1 -compress ppmd expert ppmd compress:ppmd 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 9.09333333333 0.0907377172588 9.13 8.99 9.16 8.86666666667 0.213333333333 0.219956024064 0.219058050383 457573.333333 127.268744526 457508 457492 457720 661953 661953 0.315643787384 0.315643787384 1 -compress ppmd expert ppmd compress:ppmd 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 18.2566666667 0.179536440127 18.19 18.12 18.46 17.9966666667 0.236666666667 0.219112106711 0.21990104453 459750.666667 91.2432645916 459700 459696 459856 1329257 1329257 0.316919565201 0.316919565201 1 -compress ppmd expert ppmd compress:ppmd 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 44.0866666667 0.0873689494805 44.11 43.99 44.16 43.7566666667 0.28 0.216318648541 0.216203653685 568782.666667 112.095197637 568788 568668 568892 3154465 3154465 0.3154465 0.3154465 1 -compress rosa expert rosaplus compress:rosa 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0 0 0 0 0 0 0 5694.66666667 41.6333199893 5708 5648 5728 1127 1127 0.275146484375 0.275146484375 1 -compress rosa expert rosaplus compress:rosa 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.03 0 0.03 0.03 0.03 0.03 0 0.520833333333 0.520833333333 8620 99.518842437 8600 8532 8728 6359 6359 0.388122558594 0.388122558594 1 -compress rosa expert rosaplus compress:rosa 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.17 0 0.17 0.17 0.17 0.17 0 0.367647058824 0.367647058824 19241.3333333 124.343609942 19184 19156 19384 22843 22843 0.348556518555 0.348556518555 1 -compress rosa expert rosaplus compress:rosa 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 0.94 0 0.94 0.94 0.94 0.92 0.0133333333333 0.265957446809 0.265957446809 61928 112.641022723 61972 61800 62012 80590 80590 0.307426452637 0.307426452637 1 -compress rosa expert rosaplus compress:rosa 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 5.11666666667 0.0057735026919 5.12 5.11 5.12 5.04333333333 0.0633333333333 0.195439905414 0.1953125 172198.666667 79.0274214013 172184 172128 172284 306522 306522 0.292322158813 0.292322158813 1 -compress rosa expert rosaplus compress:rosa 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 12.11 0.0264575131106 12.12 12.08 12.13 11.9633333333 0.133333333333 0.165153292408 0.16501650165 337314.666667 50.013331556 337292 337280 337372 608681 608681 0.290241718292 0.290241718292 1 -compress rosa expert rosaplus compress:rosa 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 28.6833333333 0.030550504633 28.69 28.65 28.71 28.37 0.28 0.139453911429 0.139421401185 669321.333333 68.3910325506 669288 669276 669400 1199345 1199345 0.285946130753 0.285946130753 1 -compress rosa expert rosaplus compress:rosa 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 83.6766666667 0.18009256879 83.67 83.5 83.86 82.95 0.653333333333 0.113971707897 0.113980437003 1556698.66667 143.182866759 1556664 1556576 1556856 2752778 2752778 0.2752778 0.2752778 1 -compress rwkv expert rwkv compress:rwkv 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.06 0 0.06 0.06 0.06 0.06 0 0.0651041666667 0.0651041666667 7632 98.3056458196 7672 7520 7704 3714 3714 0.90673828125 0.90673828125 1 -compress rwkv expert rwkv compress:rwkv 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.24 0 0.24 0.24 0.24 0.24 0 0.0651041666667 0.0651041666667 7646.66666667 117.598185927 7680 7516 7744 12000 12000 0.732421875 0.732421875 1 -compress rwkv expert rwkv compress:rwkv 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.963333333333 0.0057735026919 0.96 0.96 0.97 0.963333333333 0 0.064880441008 0.0651041666667 7661.33333333 56.0475988186 7664 7604 7716 35340 35340 0.539245605469 0.539245605469 1 -compress rwkv expert rwkv compress:rwkv 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 3.86333333333 0.0057735026919 3.86 3.86 3.87 3.86333333333 0 0.0647110539869 0.0647668393782 7780 49.9599839872 7764 7740 7836 137010 137010 0.522651672363 0.522651672363 1 -compress rwkv expert rwkv compress:rwkv 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 16.02 0.0529150262213 16.04 15.96 16.06 15.92 0.08 0.0624224272925 0.0623441396509 9266.66666667 82.0081296783 9312 9172 9316 474275 474275 0.452303886414 0.452303886414 1 -compress rwkv expert rwkv compress:rwkv 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 33.54 0.465080638169 33.55 33.07 34 32.98 0.523333333333 0.0596379408195 0.0596125186289 11234.6666667 132.926044601 11164 11152 11388 917086 917086 0.437300682068 0.437300682068 1 -compress rwkv expert rwkv compress:rwkv 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 67.9233333333 0.479200723427 68.2 67.37 68.2 66.1366666667 1.72 0.0588918870723 0.058651026393 14912 27.7128129211 14896 14896 14944 1748887 1748887 0.416967153549 0.416967153549 1 -compress rwkv expert rwkv compress:rwkv 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 160.75 0.457055795281 160.7 160.32 161.23 158.64 1.97666666667 0.0593268711582 0.0593450103551 24826.6666667 42.3949682549 24820 24788 24872 3968645 3968645 0.3968645 0.3968645 1 -decompress ctw expert ctw decompress:ctw 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.05 0 0.05 0.05 0.05 0.0433333333333 0 0.078125 0.078125 13414.6666667 78.0085465403 13380 13360 13504 1332 1332 0.3251953125 0.3251953125 1 -decompress ctw expert ctw decompress:ctw 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.23 0 0.23 0.23 0.23 0.216666666667 0.00333333333333 0.0679347826087 0.0679347826087 40950.6666667 90.9798512492 40924 40876 41052 6579 6579 0.401550292969 0.401550292969 1 -decompress ctw expert ctw decompress:ctw 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 1.01333333333 0.0057735026919 1.01 1.01 1.02 0.973333333333 0.0333333333333 0.0616789620138 0.0618811881188 92393.3333333 38.0175398117 92392 92356 92432 22728 22728 0.346801757812 0.346801757812 1 -decompress ctw expert ctw decompress:ctw 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 4.22666666667 0.0057735026919 4.23 4.22 4.23 4.14666666667 0.0733333333333 0.0591483386179 0.0591016548463 222052 59.1945943478 222024 222012 222120 80004 80004 0.305191040039 0.305191040039 1 -decompress ctw expert ctw decompress:ctw 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 18.1633333333 0.0750555349947 18.12 18.12 18.25 17.9266666667 0.22 0.0550565988287 0.0551876379691 552485.333333 110.151410946 552492 552372 552592 301713 301713 0.287735939026 0.287735939026 1 -decompress ctw expert ctw decompress:ctw 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 37.5766666667 0.0702376916857 37.57 37.51 37.65 37.1866666667 0.35 0.0532246427016 0.0532339632686 897960 64.3739077577 897968 897892 898020 597003 597003 0.284673213959 0.284673213959 1 -decompress ctw expert ctw decompress:ctw 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 78.3933333333 0.0321455025366 78.38 78.37 78.43 77.7133333333 0.606666666667 0.0510247527208 0.0510334268946 1474226.66667 111.163543185 1474272 1474100 1474308 1177962 1177962 0.280848026276 0.280848026276 1 -decompress ctw expert ctw decompress:ctw 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 198.19 0.127671453348 198.16 198.08 198.33 196.81 1.19333333333 0.0481192078402 0.0481264794311 2776474.66667 88.5738862946 2776436 2776412 2776576 2746861 2746861 0.2746861 0.2746861 1 -decompress match expert match decompress:match 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0 0 0 0 0 0 0 5441.33333333 77.5972508104 5480 5352 5492 2874 2874 0.70166015625 0.70166015625 1 -decompress match expert match decompress:match 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.02 0 0.02 0.02 0.02 0.0166666666667 0 0.78125 0.78125 5502.66666667 66.6133119829 5524 5428 5556 13506 13506 0.824340820312 0.824340820312 1 -decompress match expert match decompress:match 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.07 0 0.07 0.07 0.07 0.07 0 0.892857142857 0.892857142857 6333.33333333 54.0123442681 6332 6280 6388 53748 53748 0.820129394531 0.820129394531 1 -decompress match expert match decompress:match 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 0.3 0 0.3 0.3 0.3 0.296666666667 0 0.833333333333 0.833333333333 7748 31.2409987036 7764 7712 7768 206203 206203 0.786602020264 0.786602020264 1 -decompress match expert match decompress:match 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 1.20333333333 0.0057735026919 1.2 1.2 1.21 1.19666666667 0 0.831037649219 0.833333333333 11472 120.199833611 11464 11356 11596 817505 817505 0.779633522034 0.779633522034 1 -decompress match expert match decompress:match 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 2.42333333333 0.0115470053838 2.43 2.41 2.43 2.41666666667 0 0.825322017884 0.82304526749 19988 138.564064606 19908 19908 20148 1644756 1644756 0.784280776978 0.784280776978 1 -decompress match expert match decompress:match 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 4.82 0 4.82 4.82 4.82 4.81333333333 0.00333333333333 0.829875518672 0.829875518672 23849.3333333 189.750713657 23752 23728 24068 3292751 3292751 0.785053014755 0.785053014755 1 -decompress match expert match decompress:match 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 11.5866666667 0.037859388972 11.57 11.56 11.63 11.5566666667 0.01 0.823084947606 0.824264750567 46058.6666667 55.1845388975 46080 45996 46100 7859324 7859324 0.7859324 0.7859324 1 -decompress neural_mixture mixture neural-mixture decompress:neural_mixture 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.153333333333 0.0057735026919 0.15 0.15 0.16 0.14 0.00333333333333 0.0254991319444 0.0260416666667 19297.3333333 104.102513578 19292 19196 19404 1001 1001 0.244384765625 0.244384765625 1 -decompress neural_mixture mixture neural-mixture decompress:neural_mixture 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.666666666667 0.0115470053838 0.66 0.66 0.68 0.646666666667 0.02 0.0234421420083 0.0236742424242 59745.3333333 195.400443534 59848 59520 59868 5533 5533 0.337707519531 0.337707519531 1 -decompress neural_mixture mixture neural-mixture decompress:neural_mixture 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 2.88333333333 0.0057735026919 2.88 2.88 2.89 2.82 0.05 0.0216763584519 0.0217013888889 163630.666667 44.0605643783 163628 163588 163676 19344 19344 0.295166015625 0.295166015625 1 -decompress neural_mixture mixture neural-mixture decompress:neural_mixture 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 12.49 0.01 12.49 12.48 12.5 12.3 0.176666666667 0.0200160213641 0.0200160128102 468437.333333 88.1211287566 468432 468352 468528 67454 67454 0.257316589355 0.257316589355 1 -decompress neural_mixture mixture neural-mixture decompress:neural_mixture 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 55.05 0.0458257569496 55.06 55 55.09 54.4533333333 0.536666666667 0.0181653126628 0.0181620050854 1182212 88.2723059629 1182220 1182120 1182296 254251 254251 0.242472648621 0.242472648621 1 -decompress neural_mixture mixture neural-mixture decompress:neural_mixture 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 115.816666667 0.228108161479 115.86 115.57 116.02 114.876666667 0.816666666667 0.0172687164345 0.0172622130157 1835429.33333 82.2030007563 1835436 1835344 1835508 503519 503519 0.240096569061 0.240096569061 1 -decompress neural_mixture mixture neural-mixture decompress:neural_mixture 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 244.606666667 0.549302588136 244.44 244.16 245.22 243.133333333 1.22 0.0163528389885 0.0163639338897 2780732 98.3056458196 2780692 2780660 2780844 988355 988355 0.235642194748 0.235642194748 1 -decompress neural_mixture mixture neural-mixture decompress:neural_mixture 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 626.176666667 0.318956632371 626.09 625.91 626.53 623.146666667 2.39666666667 0.015230118465 0.0152322240637 5068066.66667 24.1108550934 5068064 5068044 5068092 2270248 2270248 0.2270248 0.2270248 1 -decompress ppmd expert ppmd decompress:ppmd 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.01 0 0.01 0.01 0.01 0.01 0 0.390625 0.390625 7314.66666667 100.5849558 7356 7200 7388 1058 1058 0.25830078125 0.25830078125 1 -decompress ppmd expert ppmd decompress:ppmd 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.05 0 0.05 0.05 0.05 0.0433333333333 0 0.3125 0.3125 16074.6666667 32.3316150746 16056 16056 16112 6267 6267 0.382507324219 0.382507324219 1 -decompress ppmd expert ppmd decompress:ppmd 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.24 0 0.24 0.24 0.24 0.22 0.0133333333333 0.260416666667 0.260416666667 44953.3333333 96.5263349213 44928 44872 45060 23186 23186 0.353790283203 0.353790283203 1 -decompress ppmd expert ppmd decompress:ppmd 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 1.12 0 1.12 1.12 1.12 1.04 0.0766666666667 0.223214285714 0.223214285714 144074.666667 53.1162247654 144044 144044 144136 83269 83269 0.317646026611 0.317646026611 1 -decompress ppmd expert ppmd decompress:ppmd 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 4.63333333333 0.0152752523165 4.63 4.62 4.65 4.43666666667 0.183333333333 0.215828900424 0.215982721382 399077.333333 172.340747745 398988 398968 399276 326331 326331 0.311213493347 0.311213493347 1 -decompress ppmd expert ppmd decompress:ppmd 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 9.02 0.0264575131106 9.03 8.99 9.04 8.79333333333 0.206666666667 0.221730763641 0.221483942414 457549.333333 40.0666112035 457552 457508 457588 661953 661953 0.315643787384 0.315643787384 1 -decompress ppmd expert ppmd decompress:ppmd 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 17.9166666667 0.0493288286232 17.94 17.86 17.95 17.6733333333 0.223333333333 0.223256943906 0.222965440357 458150.666667 28.0950766743 458148 458124 458180 1329257 1329257 0.316919565201 0.316919565201 1 -decompress ppmd expert ppmd decompress:ppmd 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 43.5166666667 0.227229692895 43.48 43.31 43.76 43.2033333333 0.266666666667 0.219155487712 0.219336319321 559126.666667 68.8573404463 559156 559048 559176 3154465 3154465 0.3154465 0.3154465 1 -decompress rosa expert rosaplus decompress:rosa 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0 0 0 0 0 0 0 5778.66666667 70.0095231617 5780 5708 5848 1127 1127 0.275146484375 0.275146484375 1 -decompress rosa expert rosaplus decompress:rosa 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.03 0 0.03 0.03 0.03 0.03 0 0.520833333333 0.520833333333 8576 60.3986754822 8568 8520 8640 6359 6359 0.388122558594 0.388122558594 1 -decompress rosa expert rosaplus decompress:rosa 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.173333333333 0.0057735026919 0.17 0.17 0.18 0.17 0.00333333333333 0.360838779956 0.367647058824 19366.6666667 171.643040445 19284 19252 19564 22843 22843 0.348556518555 0.348556518555 1 -decompress rosa expert rosaplus decompress:rosa 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 0.943333333333 0.0057735026919 0.94 0.94 0.95 0.926666666667 0.01 0.265024262785 0.265957446809 61973.3333333 77.1837634048 61948 61912 62060 80590 80590 0.307426452637 0.307426452637 1 -decompress rosa expert rosaplus decompress:rosa 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 5.10666666667 0.0057735026919 5.11 5.1 5.11 5.04 0.0566666666667 0.195822621286 0.195694716243 172676 36.6606055596 172668 172644 172716 306522 306522 0.292322158813 0.292322158813 1 -decompress rosa expert rosaplus decompress:rosa 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 12.1 0.0458257569496 12.09 12.06 12.15 11.9533333333 0.133333333333 0.165290834882 0.165425971878 338014.666667 153.118690346 338036 337852 338156 608681 608681 0.290241718292 0.290241718292 1 -decompress rosa expert rosaplus decompress:rosa 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 28.7033333333 0.0152752523165 28.7 28.69 28.72 28.37 0.303333333333 0.139356663167 0.1393728223 658261.333333 54.4548742844 658280 658200 658304 1199345 1199345 0.285946130753 0.285946130753 1 -decompress rosa expert rosaplus decompress:rosa 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 83.9566666667 0.208166599947 83.89 83.79 84.19 83.1766666667 0.7 0.113591720433 0.113681525379 1555033.33333 93.8367376529 1555064 1554928 1555108 2752778 2752778 0.2752778 0.2752778 1 -decompress rwkv expert rwkv decompress:rwkv 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.06 0 0.06 0.06 0.06 0.06 0 0.0651041666667 0.0651041666667 7540 72 7540 7468 7612 3714 3714 0.90673828125 0.90673828125 1 -decompress rwkv expert rwkv decompress:rwkv 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.24 0 0.24 0.24 0.24 0.24 0 0.0651041666667 0.0651041666667 7628 50.1198563446 7632 7576 7676 12000 12000 0.732421875 0.732421875 1 -decompress rwkv expert rwkv decompress:rwkv 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.953333333333 0.0057735026919 0.95 0.95 0.96 0.953333333333 0 0.0655610380117 0.0657894736842 7729.33333333 144.904566296 7696 7604 7888 35340 35340 0.539245605469 0.539245605469 1 -decompress rwkv expert rwkv decompress:rwkv 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 3.84666666667 0.0585946527708 3.87 3.78 3.89 3.84 0 0.0650014671756 0.0645994832041 7853.33333333 56.1901533485 7848 7800 7912 137010 137010 0.522651672363 0.522651672363 1 -decompress rwkv expert rwkv decompress:rwkv 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 16.0133333333 0.0665832811848 15.98 15.97 16.09 16 0 0.0624486781318 0.0625782227785 8768 138.390751136 8756 8636 8912 474275 474275 0.452303886414 0.452303886414 1 -decompress rwkv expert rwkv decompress:rwkv 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 32.5466666667 0.220302821891 32.44 32.4 32.8 32.52 0 0.0614520953174 0.0616522811344 10202.6666667 37.1662929727 10220 10160 10228 917086 917086 0.437300682068 0.437300682068 1 -decompress rwkv expert rwkv decompress:rwkv 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 65.11 0.163707055437 65.15 64.93 65.25 65.0566666667 0 0.0614347546121 0.0613967766692 13210.6666667 6.11010092661 13212 13204 13216 1748887 1748887 0.416967153549 0.416967153549 1 -decompress rwkv expert rwkv decompress:rwkv 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 157.906666667 0.937567775328 157.73 157.07 158.92 157.786666667 0.00666666666667 0.0603962273874 0.060462455868 20888 62.3538290725 20924 20816 20924 3968645 3968645 0.3968645 0.3968645 1 +operation subject subject_kind expert_kind series size_bytes repeats cpu compression_backend input_sha256 real_seconds_mean real_seconds_stdev real_seconds_median real_seconds_min real_seconds_max user_seconds_mean sys_seconds_mean throughput_mib_s_mean throughput_mib_s_median rss_kib_mean rss_kib_stdev rss_kib_median rss_kib_min rss_kib_max archive_bytes_mean archive_bytes_median archive_ratio_mean archive_ratio_median entropy_bpb_mean entropy_bpb_median verified_all +h ctw expert ctw h:ctw 4096 3 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.04 0 0.04 0.04 0.04 0.03 0 0.09765625 0.09765625 13386.6666667 80.133222407 13392 13304 13464 2.56549624542 2.56549624542 1 +h ctw expert ctw h:ctw 16384 3 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.176666666667 0.0057735026919 0.18 0.17 0.18 0.16 0.01 0.0885076252723 0.0868055555556 40813.3333333 9.23760430703 40808 40808 40824 3.20340951209 3.20340951209 1 +h ctw expert ctw h:ctw 65536 3 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.776666666667 0.0057735026919 0.78 0.77 0.78 0.743333333333 0.0333333333333 0.0804750804751 0.0801282051282 92276 68.2348884369 92256 92220 92352 2.77213895634 2.77213895634 1 +h ctw expert ctw h:ctw 262144 3 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 3.27666666667 0.0057735026919 3.28 3.27 3.28 3.19666666667 0.07 0.0762972079262 0.0762195121951 221774.666667 64.1664502161 221780 221708 221836 2.44096714152 2.44096714152 1 +h ctw expert ctw h:ctw 1048576 3 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 14.4566666667 0.0115470053838 14.45 14.45 14.47 14.23 0.206666666667 0.0691722682813 0.0692041522491 552056 80.8949936646 552088 551964 552116 2.30174274755 2.30174274755 1 +h ctw expert ctw h:ctw 2097152 3 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 30.5533333333 0.0404145188433 30.53 30.53 30.6 30.1633333333 0.356666666667 0.0654593824282 0.0655093350802 897202.666667 119.82209034 897168 897104 897336 2.27731545661 2.27731545661 1 +h ctw expert ctw h:ctw 4194304 3 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 64.7966666667 0.0152752523165 64.8 64.78 64.81 64.11 0.616666666667 0.0617315728413 0.0617283950617 1472845.33333 73.3575717519 1472820 1472788 1472928 2.24674930722 2.24674930722 1 +h ctw expert ctw h:ctw 10000000 3 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 165.963333333 0.175594229214 165.98 165.78 166.13 164.59 1.21 0.0574629955347 0.0574571825766 2773640 64.3739077577 2773648 2773572 2773700 2.19747398481 2.19747398481 1 +h match expert match h:match 4096 3 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0 0 0 0 0 0 0 5720 22.2710574513 5724 5696 5740 5.57734993252 5.57734993252 1 +h match expert match h:match 16384 3 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0 0 0 0 0 0 0 5665.33333333 154.108187107 5672 5508 5816 6.58586286407 6.58586286407 1 +h match expert match h:match 65536 3 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.01 0 0.01 0.01 0.01 0.00666666666667 0 6.25 6.25 6684 158.745078664 6624 6564 6864 6.5588255305 6.5588255305 1 +h match expert match h:match 262144 3 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 0.03 0 0.03 0.03 0.03 0.03 0 8.33333333333 8.33333333333 7868 69.7423830967 7836 7820 7948 6.29225944669 6.29225944669 1 +h match expert match h:match 1048576 3 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 0.133333333333 0.0057735026919 0.13 0.13 0.14 0.13 0 7.50915750916 7.69230769231 11425.3333333 135.784142422 11400 11304 11572 6.23692635511 6.23692635511 1 +h match expert match h:match 2097152 3 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 0.27 0 0.27 0.27 0.27 0.263333333333 0 7.40740740741 7.40740740741 19253.3333333 149.255932322 19208 19132 19420 6.2741772649 6.2741772649 1 +h match expert match h:match 4194304 3 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 0.53 0 0.53 0.53 0.53 0.52 0 7.54716981132 7.54716981132 21278.6666667 76.0350796234 21276 21204 21356 6.28038800899 6.28038800899 1 +h match expert match h:match 10000000 3 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 1.30666666667 0.0057735026919 1.31 1.3 1.31 1.28666666667 0.0133333333333 7.29862316921 7.27995661379 40453.3333333 27.2274371422 40444 40432 40484 6.28744436593 6.28744436593 1 +h neural_mixture mixture neural-mixture h:neural_mixture 4096 3 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.186666666667 0.0057735026919 0.19 0.18 0.19 0.18 0 0.0209399366472 0.0205592105263 19389.3333333 35.8515457593 19408 19348 19412 1.91856494114 1.91856494114 1 +h neural_mixture mixture neural-mixture h:neural_mixture 16384 3 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.806666666667 0.0115470053838 0.8 0.8 0.82 0.79 0.0133333333333 0.0193724593496 0.01953125 59222.6666667 121.15004471 59276 59084 59308 2.69245376114 2.69245376114 1 +h neural_mixture mixture neural-mixture h:neural_mixture 65536 3 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 3.44333333333 0.0057735026919 3.44 3.44 3.45 3.38 0.0533333333333 0.0181510504438 0.0181686046512 161166.666667 71.7030915187 161204 161084 161212 2.35901503943 2.35901503943 1 +h neural_mixture mixture neural-mixture h:neural_mixture 262144 3 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 14.71 0.01 14.71 14.7 14.72 14.5133333333 0.17 0.0169952465686 0.0169952413324 484545.333333 78.4176850802 484536 484472 484628 2.05794860665 2.05794860665 1 +h neural_mixture mixture neural-mixture h:neural_mixture 1048576 3 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 63.8266666667 0.0585946527708 63.85 63.76 63.87 63.2466666667 0.513333333333 0.0156674414369 0.0156617071261 1184440 69.3974062916 1184456 1184364 1184500 1.93962973278 1.93962973278 1 +h neural_mixture mixture neural-mixture h:neural_mixture 2097152 3 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 133.123333333 0.309246395829 132.96 132.93 133.48 132.146666667 0.84 0.0150237162454 0.0150421179302 1841770.66667 180.236881168 1841760 1841596 1841956 1.92069255467 1.92069255467 1 +h neural_mixture mixture neural-mixture h:neural_mixture 4194304 3 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 277.44 0.783134726596 277.31 276.73 278.28 275.99 1.17333333333 0.0144176082502 0.0144242905052 2806732 189.145446681 2806652 2806596 2806948 1.88509131394 1.88509131394 1 +h neural_mixture mixture neural-mixture h:neural_mixture 10000000 3 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 698.953333333 0.898906743402 698.48 698.39 699.99 695.94 2.30666666667 0.0136443353473 0.0136535665503 5058941.33333 164.406001513 5058928 5058784 5059112 1.81617083096 1.81617083096 1 +h ppmd expert ppmd h:ppmd 4096 3 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0 0 0 0 0 0 0 7326.66666667 53.2666249478 7340 7268 7372 2.02915198126 2.02915198126 1 +h ppmd expert ppmd h:ppmd 16384 3 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.03 0 0.03 0.03 0.03 0.0266666666667 0 0.520833333333 0.520833333333 16073.3333333 52.2047252012 16056 16032 16132 3.05111195443 3.05111195443 1 +h ppmd expert ppmd h:ppmd 65536 3 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.17 0 0.17 0.17 0.17 0.16 0.00666666666667 0.367647058824 0.367647058824 45270.6666667 80.0333263918 45268 45192 45352 2.82799347281 2.82799347281 1 +h ppmd expert ppmd h:ppmd 262144 3 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 0.836666666667 0.0057735026919 0.84 0.83 0.84 0.77 0.0566666666667 0.298814304838 0.297619047619 144156 116.824654932 144172 144032 144264 2.54058234497 2.54058234497 1 +h ppmd expert ppmd h:ppmd 1048576 3 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 3.52333333333 0.0057735026919 3.52 3.52 3.53 3.33333333333 0.18 0.283822645721 0.284090909091 398886.666667 30.2875111776 398900 398852 398908 2.489549756 2.489549756 1 +h ppmd expert ppmd h:ppmd 2097152 3 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 6.86333333333 0.0929157324318 6.82 6.8 6.97 6.63 0.22 0.291438941645 0.293255131965 457068 99.518842437 457088 456960 457156 2.52506560801 2.52506560801 1 +h ppmd expert ppmd h:ppmd 4194304 3 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 13.56 0.0854400374532 13.57 13.47 13.64 13.3166666667 0.22 0.294993067076 0.294767870302 459240 134.044768641 459236 459108 459376 2.53530592864 2.53530592864 1 +h ppmd expert ppmd h:ppmd 10000000 3 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 32.9466666667 0.283607710285 32.83 32.74 33.27 32.63 0.276666666667 0.289474268395 0.290488673898 557333.333333 121.083992886 557352 557204 557444 2.5235430657 2.5235430657 1 +h rosa expert rosaplus h:rosa 4096 3 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.02 0 0.02 0.02 0.02 0.02 0 0.1953125 0.1953125 5889.33333333 140.873465682 5952 5728 5988 2.11172124286 2.11172124286 1 +h rosa expert rosaplus h:rosa 16384 3 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.12 0 0.12 0.12 0.12 0.113333333333 0.00333333333333 0.130208333333 0.130208333333 8101.33333333 51.4328040586 8080 8064 8160 3.48630622986 3.48630622986 1 +h rosa expert rosaplus h:rosa 65536 3 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.603333333333 0.0057735026919 0.6 0.6 0.61 0.573333333333 0.0233333333333 0.103597449909 0.104166666667 16697.3333333 98.006802485 16696 16600 16796 3.19056696201 3.19056696201 1 +h rosa expert rosaplus h:rosa 262144 3 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 3.49666666667 0.0057735026919 3.5 3.49 3.5 3.38666666667 0.0966666666667 0.0714967935598 0.0714285714286 46952 90.0666419936 46900 46900 47056 2.90567254487 2.90567254487 1 +h rosa expert rosaplus h:rosa 1048576 3 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 20.5766666667 0.0057735026919 20.58 20.57 20.58 19.7833333333 0.76 0.048598738984 0.0485908649174 203844 47.1593044902 203832 203804 203896 2.80123027825 2.80123027825 1 +h rosa expert rosaplus h:rosa 2097152 3 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 49.1166666667 0.0550757054729 49.09 49.08 49.18 47.1133333333 1.94 0.0407194097478 0.0407414952129 337701.333333 64.2910050733 337728 337628 337748 2.70437138702 2.70437138702 1 +h rosa expert rosaplus h:rosa 4194304 3 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 117.806666667 0.117189305542 117.76 117.72 117.94 112.6 5.06666666667 0.0339539582152 0.0339673913043 643536 72.9931503636 643572 643452 643584 2.62099230176 2.62099230176 1 +h rosa expert rosaplus h:rosa 10000000 3 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 346.986666667 1.73367624813 346.15 345.83 348.98 333.436666667 13.1733333333 0.0274849219272 0.0275508974839 1551498.66667 175.377687672 1551524 1551312 1551660 2.48481897791 2.48481897791 1 +h rwkv expert rwkv h:rwkv 4096 3 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.06 0 0.06 0.06 0.06 0.0533333333333 0 0.0651041666667 0.0651041666667 8030.66666667 31.0698138606 8040 7996 8056 7.21701437947 7.21701437947 1 +h rwkv expert rwkv h:rwkv 16384 3 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.226666666667 0.0057735026919 0.23 0.22 0.23 0.226666666667 0 0.0689640974967 0.0679347826087 8052 77.1492060879 8020 7996 8140 5.85021072042 5.85021072042 1 +h rwkv expert rwkv h:rwkv 65536 3 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.906666666667 0.0115470053838 0.9 0.9 0.92 0.903333333333 0 0.0689412238325 0.0694444444444 8273.33333333 107.802288164 8296 8156 8368 4.31173851819 4.31173851819 1 +h rwkv expert rwkv h:rwkv 262144 3 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 3.63333333333 0.0152752523165 3.63 3.62 3.65 3.62666666667 0 0.0688081491939 0.068870523416 8264 69.2820323028 8304 8184 8304 4.18065126521 4.18065126521 1 +h rwkv expert rwkv h:rwkv 1048576 3 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 15.0133333333 0.136137185711 15.06 14.86 15.12 15 0 0.0666111265213 0.066401062417 8833.33333333 51.4328040586 8812 8796 8892 3.61829048151 3.61829048151 1 +h rwkv expert rwkv h:rwkv 2097152 3 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 30.2566666667 0.251064400769 30.32 29.98 30.47 30.23 0 0.0661041780776 0.065963060686 9836 155.897402159 9808 9696 10004 3.49833649966 3.49833649966 1 +h rwkv expert rwkv h:rwkv 4194304 3 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 61.0533333333 0.0351188458428 61.05 61.02 61.09 61.01 0 0.0655165027668 0.0655200655201 11856 154.350251053 11844 11708 12016 3.33570098711 3.33570098711 1 +h rwkv expert rwkv h:rwkv 10000000 3 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 147.966666667 1.31241507662 148.41 146.49 149 147.856666667 0 0.0644553640807 0.0642594378011 17626.6666667 43.1431724996 17608 17596 17676 3.17490278117 3.17490278117 1 +compress ctw expert ctw compress:ctw 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.05 0 0.05 0.05 0.05 0.0433333333333 0.00333333333333 0.078125 0.078125 13432 48.4974226119 13440 13380 13476 1332 1332 0.3251953125 0.3251953125 1 +compress ctw expert ctw compress:ctw 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.23 0 0.23 0.23 0.23 0.22 0 0.0679347826087 0.0679347826087 40992 139.484766193 41056 40832 41088 6579 6579 0.401550292969 0.401550292969 1 +compress ctw expert ctw compress:ctw 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 1.01 0.01 1.01 1 1.02 0.98 0.0233333333333 0.0618852326409 0.0618811881188 92406.6666667 64.6632301492 92396 92348 92476 22728 22728 0.346801757812 0.346801757812 1 +compress ctw expert ctw compress:ctw 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 4.20666666667 0.0115470053838 4.2 4.2 4.22 4.11666666667 0.0733333333333 0.0594297750696 0.0595238095238 221974.666667 126.258993079 221984 221844 222096 80004 80004 0.305191040039 0.305191040039 1 +compress ctw expert ctw compress:ctw 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 17.9866666667 0.0057735026919 17.99 17.98 17.99 17.76 0.203333333333 0.0555967421443 0.0555864369094 552538.666667 6.11010092661 552540 552532 552544 301713 301713 0.287735939026 0.287735939026 1 +compress ctw expert ctw compress:ctw 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 37.36 0.0346410161514 37.34 37.34 37.4 36.95 0.38 0.0535332212449 0.0535618639529 898394.666667 96.0277737602 898356 898324 898504 597003 597003 0.284673213959 0.284673213959 1 +compress ctw expert ctw compress:ctw 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 77.9033333333 0.0416333199893 77.89 77.87 77.95 77.1933333333 0.636666666667 0.0513456945973 0.0513544742586 1475212 85.5102333057 1475240 1475116 1475280 1177962 1177962 0.280848026276 0.280848026276 1 +compress ctw expert ctw compress:ctw 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 197.076666667 0.210317220725 197.09 196.86 197.28 195.666666667 1.23 0.0483910681408 0.0483877576948 2779017.33333 89.9184816005 2778996 2778940 2779116 2746861 2746861 0.2746861 0.2746861 1 +compress match expert match compress:match 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0 0 0 0 0 0 0 5464 139.312598138 5508 5308 5576 2874 2874 0.70166015625 0.70166015625 1 +compress match expert match compress:match 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.02 0 0.02 0.02 0.02 0.0166666666667 0 0.78125 0.78125 5546.66666667 63.7913264742 5516 5504 5620 13506 13506 0.824340820312 0.824340820312 1 +compress match expert match compress:match 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.07 0 0.07 0.07 0.07 0.07 0 0.892857142857 0.892857142857 6438.66666667 50.3322295685 6432 6392 6492 53748 53748 0.820129394531 0.820129394531 1 +compress match expert match compress:match 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 0.296666666667 0.0057735026919 0.3 0.29 0.3 0.293333333333 0 0.842911877395 0.833333333333 8230.66666667 48.8808074129 8220 8188 8284 206203 206203 0.786602020264 0.786602020264 1 +compress match expert match compress:match 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 1.17666666667 0.0057735026919 1.18 1.17 1.18 1.17666666667 0 0.849872036313 0.847457627119 12108 16 12108 12092 12124 817505 817505 0.779633522034 0.779633522034 1 +compress match expert match compress:match 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 2.36666666667 0.0208166599947 2.36 2.35 2.39 2.36 0.00333333333333 0.845113846863 0.847457627119 22012 107.628992377 22044 21892 22100 1644756 1644756 0.784280776978 0.784280776978 1 +compress match expert match compress:match 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 4.72333333333 0.0115470053838 4.73 4.71 4.73 4.70333333333 0.00666666666667 0.846862941367 0.845665961945 28905.3333333 64.2910050733 28932 28832 28952 3292751 3292751 0.785053014755 0.785053014755 1 +compress match expert match compress:match 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 11.31 0.03 11.31 11.28 11.34 11.2766666667 0.0233333333333 0.843217320706 0.843213365523 55496 116.824654932 55512 55372 55604 7859324 7859324 0.7859324 0.7859324 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.15 0 0.15 0.15 0.15 0.146666666667 0.00333333333333 0.0260416666667 0.0260416666667 19533.3333333 91.2432645916 19516 19452 19632 1001 1001 0.244384765625 0.244384765625 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.673333333333 0.0057735026919 0.67 0.67 0.68 0.646666666667 0.02 0.0232065774071 0.0233208955224 58921.3333333 100.5849558 58880 58848 59036 5533 5533 0.337707519531 0.337707519531 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 2.88 0.01 2.88 2.87 2.89 2.81666666667 0.0533333333333 0.021701563317 0.0217013888889 165472 62.3538290725 165436 165436 165544 19344 19344 0.295166015625 0.295166015625 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 12.4633333333 0.0251661147842 12.46 12.44 12.49 12.26 0.186666666667 0.0200588937634 0.0200642054575 471460 169.846989964 471368 471356 471656 67454 67454 0.257316589355 0.257316589355 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 55.08 0.0854400374532 55.09 54.99 55.16 54.5 0.516666666667 0.0181554394441 0.0181521147214 1184602.66667 46.3609030686 1184596 1184560 1184652 254251 254251 0.242472648621 0.242472648621 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 115.833333333 0.0642910050733 115.86 115.76 115.88 114.92 0.793333333333 0.0172661905974 0.0172622130157 1834400 32 1834400 1834368 1834432 503519 503519 0.240096569061 0.240096569061 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 244.6 0.113578166916 244.55 244.52 244.73 243.136666667 1.21333333333 0.016353232113 0.0163565732979 2763934.66667 85.5414129725 2763980 2763836 2763988 988355 988355 0.235642194748 0.235642194748 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 628.033333333 2.47508249021 626.83 626.39 630.88 625.04 2.34666666667 0.0151852476444 0.0152142417626 5066150.66667 96.4434203735 5066140 5066060 5066252 2270248 2270248 0.2270248 0.2270248 1 +compress ppmd expert ppmd compress:ppmd 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.01 0 0.01 0.01 0.01 0.01 0 0.390625 0.390625 7142.66666667 23.0940107676 7156 7116 7156 1058 1058 0.25830078125 0.25830078125 1 +compress ppmd expert ppmd compress:ppmd 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.05 0 0.05 0.05 0.05 0.0466666666667 0 0.3125 0.3125 16113.3333333 95.0228042805 16160 16004 16176 6267 6267 0.382507324219 0.382507324219 1 +compress ppmd expert ppmd compress:ppmd 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.243333333333 0.0057735026919 0.24 0.24 0.25 0.22 0.0166666666667 0.256944444444 0.260416666667 44936 158.037970121 44872 44820 45116 23186 23186 0.353790283203 0.353790283203 1 +compress ppmd expert ppmd compress:ppmd 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 1.12666666667 0.0057735026919 1.13 1.12 1.13 1.06333333333 0.0566666666667 0.221897387273 0.221238938053 144042.666667 25.7164020293 144032 144024 144072 83269 83269 0.317646026611 0.317646026611 1 +compress ppmd expert ppmd compress:ppmd 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 4.67666666667 0.0152752523165 4.68 4.66 4.69 4.48 0.186666666667 0.213829034853 0.213675213675 399384 108.885260711 399428 399260 399464 326331 326331 0.311213493347 0.311213493347 1 +compress ppmd expert ppmd compress:ppmd 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 9.09333333333 0.0907377172588 9.13 8.99 9.16 8.86666666667 0.213333333333 0.219956024064 0.219058050383 457573.333333 127.268744526 457508 457492 457720 661953 661953 0.315643787384 0.315643787384 1 +compress ppmd expert ppmd compress:ppmd 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 18.2566666667 0.179536440127 18.19 18.12 18.46 17.9966666667 0.236666666667 0.219112106711 0.21990104453 459750.666667 91.2432645916 459700 459696 459856 1329257 1329257 0.316919565201 0.316919565201 1 +compress ppmd expert ppmd compress:ppmd 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 44.0866666667 0.0873689494805 44.11 43.99 44.16 43.7566666667 0.28 0.216318648541 0.216203653685 568782.666667 112.095197637 568788 568668 568892 3154465 3154465 0.3154465 0.3154465 1 +compress rosa expert rosaplus compress:rosa 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0 0 0 0 0 0 0 5694.66666667 41.6333199893 5708 5648 5728 1127 1127 0.275146484375 0.275146484375 1 +compress rosa expert rosaplus compress:rosa 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.03 0 0.03 0.03 0.03 0.03 0 0.520833333333 0.520833333333 8620 99.518842437 8600 8532 8728 6359 6359 0.388122558594 0.388122558594 1 +compress rosa expert rosaplus compress:rosa 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.17 0 0.17 0.17 0.17 0.17 0 0.367647058824 0.367647058824 19241.3333333 124.343609942 19184 19156 19384 22843 22843 0.348556518555 0.348556518555 1 +compress rosa expert rosaplus compress:rosa 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 0.94 0 0.94 0.94 0.94 0.92 0.0133333333333 0.265957446809 0.265957446809 61928 112.641022723 61972 61800 62012 80590 80590 0.307426452637 0.307426452637 1 +compress rosa expert rosaplus compress:rosa 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 5.11666666667 0.0057735026919 5.12 5.11 5.12 5.04333333333 0.0633333333333 0.195439905414 0.1953125 172198.666667 79.0274214013 172184 172128 172284 306522 306522 0.292322158813 0.292322158813 1 +compress rosa expert rosaplus compress:rosa 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 12.11 0.0264575131106 12.12 12.08 12.13 11.9633333333 0.133333333333 0.165153292408 0.16501650165 337314.666667 50.013331556 337292 337280 337372 608681 608681 0.290241718292 0.290241718292 1 +compress rosa expert rosaplus compress:rosa 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 28.6833333333 0.030550504633 28.69 28.65 28.71 28.37 0.28 0.139453911429 0.139421401185 669321.333333 68.3910325506 669288 669276 669400 1199345 1199345 0.285946130753 0.285946130753 1 +compress rosa expert rosaplus compress:rosa 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 83.6766666667 0.18009256879 83.67 83.5 83.86 82.95 0.653333333333 0.113971707897 0.113980437003 1556698.66667 143.182866759 1556664 1556576 1556856 2752778 2752778 0.2752778 0.2752778 1 +compress rwkv expert rwkv compress:rwkv 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.06 0 0.06 0.06 0.06 0.06 0 0.0651041666667 0.0651041666667 7632 98.3056458196 7672 7520 7704 3714 3714 0.90673828125 0.90673828125 1 +compress rwkv expert rwkv compress:rwkv 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.24 0 0.24 0.24 0.24 0.24 0 0.0651041666667 0.0651041666667 7646.66666667 117.598185927 7680 7516 7744 12000 12000 0.732421875 0.732421875 1 +compress rwkv expert rwkv compress:rwkv 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.963333333333 0.0057735026919 0.96 0.96 0.97 0.963333333333 0 0.064880441008 0.0651041666667 7661.33333333 56.0475988186 7664 7604 7716 35340 35340 0.539245605469 0.539245605469 1 +compress rwkv expert rwkv compress:rwkv 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 3.86333333333 0.0057735026919 3.86 3.86 3.87 3.86333333333 0 0.0647110539869 0.0647668393782 7780 49.9599839872 7764 7740 7836 137010 137010 0.522651672363 0.522651672363 1 +compress rwkv expert rwkv compress:rwkv 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 16.02 0.0529150262213 16.04 15.96 16.06 15.92 0.08 0.0624224272925 0.0623441396509 9266.66666667 82.0081296783 9312 9172 9316 474275 474275 0.452303886414 0.452303886414 1 +compress rwkv expert rwkv compress:rwkv 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 33.54 0.465080638169 33.55 33.07 34 32.98 0.523333333333 0.0596379408195 0.0596125186289 11234.6666667 132.926044601 11164 11152 11388 917086 917086 0.437300682068 0.437300682068 1 +compress rwkv expert rwkv compress:rwkv 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 67.9233333333 0.479200723427 68.2 67.37 68.2 66.1366666667 1.72 0.0588918870723 0.058651026393 14912 27.7128129211 14896 14896 14944 1748887 1748887 0.416967153549 0.416967153549 1 +compress rwkv expert rwkv compress:rwkv 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 160.75 0.457055795281 160.7 160.32 161.23 158.64 1.97666666667 0.0593268711582 0.0593450103551 24826.6666667 42.3949682549 24820 24788 24872 3968645 3968645 0.3968645 0.3968645 1 +decompress ctw expert ctw decompress:ctw 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.05 0 0.05 0.05 0.05 0.0433333333333 0 0.078125 0.078125 13414.6666667 78.0085465403 13380 13360 13504 1332 1332 0.3251953125 0.3251953125 1 +decompress ctw expert ctw decompress:ctw 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.23 0 0.23 0.23 0.23 0.216666666667 0.00333333333333 0.0679347826087 0.0679347826087 40950.6666667 90.9798512492 40924 40876 41052 6579 6579 0.401550292969 0.401550292969 1 +decompress ctw expert ctw decompress:ctw 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 1.01333333333 0.0057735026919 1.01 1.01 1.02 0.973333333333 0.0333333333333 0.0616789620138 0.0618811881188 92393.3333333 38.0175398117 92392 92356 92432 22728 22728 0.346801757812 0.346801757812 1 +decompress ctw expert ctw decompress:ctw 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 4.22666666667 0.0057735026919 4.23 4.22 4.23 4.14666666667 0.0733333333333 0.0591483386179 0.0591016548463 222052 59.1945943478 222024 222012 222120 80004 80004 0.305191040039 0.305191040039 1 +decompress ctw expert ctw decompress:ctw 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 18.1633333333 0.0750555349947 18.12 18.12 18.25 17.9266666667 0.22 0.0550565988287 0.0551876379691 552485.333333 110.151410946 552492 552372 552592 301713 301713 0.287735939026 0.287735939026 1 +decompress ctw expert ctw decompress:ctw 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 37.5766666667 0.0702376916857 37.57 37.51 37.65 37.1866666667 0.35 0.0532246427016 0.0532339632686 897960 64.3739077577 897968 897892 898020 597003 597003 0.284673213959 0.284673213959 1 +decompress ctw expert ctw decompress:ctw 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 78.3933333333 0.0321455025366 78.38 78.37 78.43 77.7133333333 0.606666666667 0.0510247527208 0.0510334268946 1474226.66667 111.163543185 1474272 1474100 1474308 1177962 1177962 0.280848026276 0.280848026276 1 +decompress ctw expert ctw decompress:ctw 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 198.19 0.127671453348 198.16 198.08 198.33 196.81 1.19333333333 0.0481192078402 0.0481264794311 2776474.66667 88.5738862946 2776436 2776412 2776576 2746861 2746861 0.2746861 0.2746861 1 +decompress match expert match decompress:match 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0 0 0 0 0 0 0 5441.33333333 77.5972508104 5480 5352 5492 2874 2874 0.70166015625 0.70166015625 1 +decompress match expert match decompress:match 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.02 0 0.02 0.02 0.02 0.0166666666667 0 0.78125 0.78125 5502.66666667 66.6133119829 5524 5428 5556 13506 13506 0.824340820312 0.824340820312 1 +decompress match expert match decompress:match 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.07 0 0.07 0.07 0.07 0.07 0 0.892857142857 0.892857142857 6333.33333333 54.0123442681 6332 6280 6388 53748 53748 0.820129394531 0.820129394531 1 +decompress match expert match decompress:match 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 0.3 0 0.3 0.3 0.3 0.296666666667 0 0.833333333333 0.833333333333 7748 31.2409987036 7764 7712 7768 206203 206203 0.786602020264 0.786602020264 1 +decompress match expert match decompress:match 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 1.20333333333 0.0057735026919 1.2 1.2 1.21 1.19666666667 0 0.831037649219 0.833333333333 11472 120.199833611 11464 11356 11596 817505 817505 0.779633522034 0.779633522034 1 +decompress match expert match decompress:match 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 2.42333333333 0.0115470053838 2.43 2.41 2.43 2.41666666667 0 0.825322017884 0.82304526749 19988 138.564064606 19908 19908 20148 1644756 1644756 0.784280776978 0.784280776978 1 +decompress match expert match decompress:match 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 4.82 0 4.82 4.82 4.82 4.81333333333 0.00333333333333 0.829875518672 0.829875518672 23849.3333333 189.750713657 23752 23728 24068 3292751 3292751 0.785053014755 0.785053014755 1 +decompress match expert match decompress:match 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 11.5866666667 0.037859388972 11.57 11.56 11.63 11.5566666667 0.01 0.823084947606 0.824264750567 46058.6666667 55.1845388975 46080 45996 46100 7859324 7859324 0.7859324 0.7859324 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.153333333333 0.0057735026919 0.15 0.15 0.16 0.14 0.00333333333333 0.0254991319444 0.0260416666667 19297.3333333 104.102513578 19292 19196 19404 1001 1001 0.244384765625 0.244384765625 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.666666666667 0.0115470053838 0.66 0.66 0.68 0.646666666667 0.02 0.0234421420083 0.0236742424242 59745.3333333 195.400443534 59848 59520 59868 5533 5533 0.337707519531 0.337707519531 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 2.88333333333 0.0057735026919 2.88 2.88 2.89 2.82 0.05 0.0216763584519 0.0217013888889 163630.666667 44.0605643783 163628 163588 163676 19344 19344 0.295166015625 0.295166015625 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 12.49 0.01 12.49 12.48 12.5 12.3 0.176666666667 0.0200160213641 0.0200160128102 468437.333333 88.1211287566 468432 468352 468528 67454 67454 0.257316589355 0.257316589355 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 55.05 0.0458257569496 55.06 55 55.09 54.4533333333 0.536666666667 0.0181653126628 0.0181620050854 1182212 88.2723059629 1182220 1182120 1182296 254251 254251 0.242472648621 0.242472648621 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 115.816666667 0.228108161479 115.86 115.57 116.02 114.876666667 0.816666666667 0.0172687164345 0.0172622130157 1835429.33333 82.2030007563 1835436 1835344 1835508 503519 503519 0.240096569061 0.240096569061 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 244.606666667 0.549302588136 244.44 244.16 245.22 243.133333333 1.22 0.0163528389885 0.0163639338897 2780732 98.3056458196 2780692 2780660 2780844 988355 988355 0.235642194748 0.235642194748 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 626.176666667 0.318956632371 626.09 625.91 626.53 623.146666667 2.39666666667 0.015230118465 0.0152322240637 5068066.66667 24.1108550934 5068064 5068044 5068092 2270248 2270248 0.2270248 0.2270248 1 +decompress ppmd expert ppmd decompress:ppmd 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.01 0 0.01 0.01 0.01 0.01 0 0.390625 0.390625 7314.66666667 100.5849558 7356 7200 7388 1058 1058 0.25830078125 0.25830078125 1 +decompress ppmd expert ppmd decompress:ppmd 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.05 0 0.05 0.05 0.05 0.0433333333333 0 0.3125 0.3125 16074.6666667 32.3316150746 16056 16056 16112 6267 6267 0.382507324219 0.382507324219 1 +decompress ppmd expert ppmd decompress:ppmd 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.24 0 0.24 0.24 0.24 0.22 0.0133333333333 0.260416666667 0.260416666667 44953.3333333 96.5263349213 44928 44872 45060 23186 23186 0.353790283203 0.353790283203 1 +decompress ppmd expert ppmd decompress:ppmd 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 1.12 0 1.12 1.12 1.12 1.04 0.0766666666667 0.223214285714 0.223214285714 144074.666667 53.1162247654 144044 144044 144136 83269 83269 0.317646026611 0.317646026611 1 +decompress ppmd expert ppmd decompress:ppmd 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 4.63333333333 0.0152752523165 4.63 4.62 4.65 4.43666666667 0.183333333333 0.215828900424 0.215982721382 399077.333333 172.340747745 398988 398968 399276 326331 326331 0.311213493347 0.311213493347 1 +decompress ppmd expert ppmd decompress:ppmd 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 9.02 0.0264575131106 9.03 8.99 9.04 8.79333333333 0.206666666667 0.221730763641 0.221483942414 457549.333333 40.0666112035 457552 457508 457588 661953 661953 0.315643787384 0.315643787384 1 +decompress ppmd expert ppmd decompress:ppmd 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 17.9166666667 0.0493288286232 17.94 17.86 17.95 17.6733333333 0.223333333333 0.223256943906 0.222965440357 458150.666667 28.0950766743 458148 458124 458180 1329257 1329257 0.316919565201 0.316919565201 1 +decompress ppmd expert ppmd decompress:ppmd 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 43.5166666667 0.227229692895 43.48 43.31 43.76 43.2033333333 0.266666666667 0.219155487712 0.219336319321 559126.666667 68.8573404463 559156 559048 559176 3154465 3154465 0.3154465 0.3154465 1 +decompress rosa expert rosaplus decompress:rosa 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0 0 0 0 0 0 0 5778.66666667 70.0095231617 5780 5708 5848 1127 1127 0.275146484375 0.275146484375 1 +decompress rosa expert rosaplus decompress:rosa 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.03 0 0.03 0.03 0.03 0.03 0 0.520833333333 0.520833333333 8576 60.3986754822 8568 8520 8640 6359 6359 0.388122558594 0.388122558594 1 +decompress rosa expert rosaplus decompress:rosa 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.173333333333 0.0057735026919 0.17 0.17 0.18 0.17 0.00333333333333 0.360838779956 0.367647058824 19366.6666667 171.643040445 19284 19252 19564 22843 22843 0.348556518555 0.348556518555 1 +decompress rosa expert rosaplus decompress:rosa 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 0.943333333333 0.0057735026919 0.94 0.94 0.95 0.926666666667 0.01 0.265024262785 0.265957446809 61973.3333333 77.1837634048 61948 61912 62060 80590 80590 0.307426452637 0.307426452637 1 +decompress rosa expert rosaplus decompress:rosa 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 5.10666666667 0.0057735026919 5.11 5.1 5.11 5.04 0.0566666666667 0.195822621286 0.195694716243 172676 36.6606055596 172668 172644 172716 306522 306522 0.292322158813 0.292322158813 1 +decompress rosa expert rosaplus decompress:rosa 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 12.1 0.0458257569496 12.09 12.06 12.15 11.9533333333 0.133333333333 0.165290834882 0.165425971878 338014.666667 153.118690346 338036 337852 338156 608681 608681 0.290241718292 0.290241718292 1 +decompress rosa expert rosaplus decompress:rosa 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 28.7033333333 0.0152752523165 28.7 28.69 28.72 28.37 0.303333333333 0.139356663167 0.1393728223 658261.333333 54.4548742844 658280 658200 658304 1199345 1199345 0.285946130753 0.285946130753 1 +decompress rosa expert rosaplus decompress:rosa 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 83.9566666667 0.208166599947 83.89 83.79 84.19 83.1766666667 0.7 0.113591720433 0.113681525379 1555033.33333 93.8367376529 1555064 1554928 1555108 2752778 2752778 0.2752778 0.2752778 1 +decompress rwkv expert rwkv decompress:rwkv 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.06 0 0.06 0.06 0.06 0.06 0 0.0651041666667 0.0651041666667 7540 72 7540 7468 7612 3714 3714 0.90673828125 0.90673828125 1 +decompress rwkv expert rwkv decompress:rwkv 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.24 0 0.24 0.24 0.24 0.24 0 0.0651041666667 0.0651041666667 7628 50.1198563446 7632 7576 7676 12000 12000 0.732421875 0.732421875 1 +decompress rwkv expert rwkv decompress:rwkv 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.953333333333 0.0057735026919 0.95 0.95 0.96 0.953333333333 0 0.0655610380117 0.0657894736842 7729.33333333 144.904566296 7696 7604 7888 35340 35340 0.539245605469 0.539245605469 1 +decompress rwkv expert rwkv decompress:rwkv 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 3.84666666667 0.0585946527708 3.87 3.78 3.89 3.84 0 0.0650014671756 0.0645994832041 7853.33333333 56.1901533485 7848 7800 7912 137010 137010 0.522651672363 0.522651672363 1 +decompress rwkv expert rwkv decompress:rwkv 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 16.0133333333 0.0665832811848 15.98 15.97 16.09 16 0 0.0624486781318 0.0625782227785 8768 138.390751136 8756 8636 8912 474275 474275 0.452303886414 0.452303886414 1 +decompress rwkv expert rwkv decompress:rwkv 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 32.5466666667 0.220302821891 32.44 32.4 32.8 32.52 0 0.0614520953174 0.0616522811344 10202.6666667 37.1662929727 10220 10160 10228 917086 917086 0.437300682068 0.437300682068 1 +decompress rwkv expert rwkv decompress:rwkv 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 65.11 0.163707055437 65.15 64.93 65.25 65.0566666667 0 0.0614347546121 0.0613967766692 13210.6666667 6.11010092661 13212 13204 13216 1748887 1748887 0.416967153549 0.416967153549 1 +decompress rwkv expert rwkv decompress:rwkv 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 157.906666667 0.937567775328 157.73 157.07 158.92 157.786666667 0.00666666666667 0.0603962273874 0.060462455868 20888 62.3538290725 20924 20816 20924 3968645 3968645 0.3968645 0.3968645 1 diff --git a/benchmarks/77b5f61f/infotheory-two-json-raw-20260510-132339.tsv b/benchmarks/77b5f61f/infotheory-two-json-raw-20260510-132339.tsv new file mode 100644 index 00000000..f53cc9e6 --- /dev/null +++ b/benchmarks/77b5f61f/infotheory-two-json-raw-20260510-132339.tsv @@ -0,0 +1,289 @@ +operation subject subject_kind expert_kind series size_bytes repetition cpu compression_backend input_sha256 suite_spec_path suite_spec_sha256 build_mode build_features archive_bytes entropy_bpb real_seconds user_seconds sys_seconds rss_kib verified +h neural_mixture mixture neural-mixture h:neural_mixture 4096 1 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1.918573669072413 0.12 0.12 0.00 12348 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1001 0.13 0.12 0.00 12124 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1001 0.13 0.13 0.00 12564 1 +h neural_mixture mixture neural-mixture h:neural_mixture 4096 2 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1.918573669072413 0.12 0.12 0.00 12368 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1001 0.13 0.13 0.00 12220 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1001 0.13 0.13 0.00 12444 1 +h ctw expert ctw h:ctw 4096 1 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.565496245416876 0.02 0.01 0.00 7096 1 +compress ctw expert ctw compress:ctw 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1332 0.04 0.03 0.00 7168 1 +decompress ctw expert ctw decompress:ctw 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1332 0.03 0.03 0.00 7236 1 +h ctw expert ctw h:ctw 4096 2 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.565496245416876 0.02 0.02 0.00 7144 1 +compress ctw expert ctw compress:ctw 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1332 0.04 0.04 0.00 7164 1 +decompress ctw expert ctw decompress:ctw 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1332 0.03 0.03 0.00 7296 1 +h ppmd expert ppmd h:ppmd 4096 1 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.029162762208123 0.01 0.00 0.00 7636 1 +compress ppmd expert ppmd compress:ppmd 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1058 0.01 0.01 0.00 7612 1 +decompress ppmd expert ppmd decompress:ppmd 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1058 0.01 0.01 0.00 7552 1 +h ppmd expert ppmd h:ppmd 4096 2 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.029162762208123 0.01 0.01 0.00 7864 1 +compress ppmd expert ppmd compress:ppmd 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1058 0.01 0.00 0.00 7548 1 +decompress ppmd expert ppmd decompress:ppmd 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1058 0.01 0.01 0.00 7808 1 +h rosa expert rosaplus h:rosa 4096 1 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.111721242855073 0.03 0.03 0.00 6744 1 +compress rosa expert rosaplus compress:rosa 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1127 0.00 0.00 0.00 6104 1 +decompress rosa expert rosaplus decompress:rosa 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1127 0.00 0.00 0.00 6332 1 +h rosa expert rosaplus h:rosa 4096 2 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.111721242855073 0.03 0.03 0.00 6600 1 +compress rosa expert rosaplus compress:rosa 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1127 0.00 0.00 0.00 6240 1 +decompress rosa expert rosaplus decompress:rosa 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1127 0.00 0.00 0.00 6036 1 +h match expert match h:match 4096 1 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 5.577349932524691 0.00 0.00 0.00 5984 1 +compress match expert match compress:match 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2874 0.00 0.00 0.00 5808 1 +decompress match expert match decompress:match 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2874 0.01 0.00 0.00 5996 1 +h match expert match h:match 4096 2 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 5.577349932524691 0.00 0.00 0.00 5952 1 +compress match expert match compress:match 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2874 0.00 0.00 0.00 5840 1 +decompress match expert match decompress:match 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2874 0.00 0.00 0.00 5760 1 +h rwkv7 expert rwkv7 h:rwkv7 4096 1 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 7.217014379468224 0.09 0.09 0.00 8440 1 +compress rwkv7 expert rwkv7 compress:rwkv7 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3714 0.06 0.06 0.00 8096 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3714 0.06 0.06 0.00 8112 1 +h rwkv7 expert rwkv7 h:rwkv7 4096 2 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 7.217014379468224 0.06 0.06 0.00 8384 1 +compress rwkv7 expert rwkv7 compress:rwkv7 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3714 0.06 0.06 0.00 8000 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3714 0.06 0.06 0.00 8060 1 +h neural_mixture mixture neural-mixture h:neural_mixture 16384 1 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.6924577982523554 0.52 0.51 0.00 29884 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 5533 0.57 0.55 0.01 30560 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 5533 0.57 0.57 0.00 30196 1 +h neural_mixture mixture neural-mixture h:neural_mixture 16384 2 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.6924577982523554 0.52 0.51 0.01 30064 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 5533 0.57 0.56 0.00 30520 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 5533 0.57 0.55 0.01 30188 1 +h ctw expert ctw h:ctw 16384 1 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3.2034095120908512 0.10 0.10 0.00 12916 1 +compress ctw expert ctw compress:ctw 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 6579 0.18 0.17 0.00 12872 1 +decompress ctw expert ctw decompress:ctw 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 6579 0.18 0.17 0.00 13048 1 +h ctw expert ctw h:ctw 16384 2 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3.2034095120908512 0.10 0.10 0.00 12640 1 +compress ctw expert ctw compress:ctw 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 6579 0.18 0.18 0.00 12948 1 +decompress ctw expert ctw decompress:ctw 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 6579 0.18 0.18 0.00 13056 1 +h ppmd expert ppmd h:ppmd 16384 1 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3.05111741488576 0.03 0.02 0.00 16560 1 +compress ppmd expert ppmd compress:ppmd 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 6267 0.04 0.02 0.01 16644 1 +decompress ppmd expert ppmd decompress:ppmd 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 6267 0.04 0.03 0.00 16460 1 +h ppmd expert ppmd h:ppmd 16384 2 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3.05111741488576 0.03 0.02 0.00 16552 1 +compress ppmd expert ppmd compress:ppmd 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 6267 0.04 0.03 0.01 16256 1 +decompress ppmd expert ppmd decompress:ppmd 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 6267 0.04 0.04 0.00 16572 1 +h rosa expert rosaplus h:rosa 16384 1 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3.486306229863964 0.13 0.12 0.01 8272 1 +compress rosa expert rosaplus compress:rosa 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 6359 0.02 0.02 0.00 8912 1 +decompress rosa expert rosaplus decompress:rosa 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 6359 0.02 0.02 0.00 8608 1 +h rosa expert rosaplus h:rosa 16384 2 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3.486306229863964 0.14 0.12 0.01 8044 1 +compress rosa expert rosaplus compress:rosa 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 6359 0.02 0.02 0.00 8636 1 +decompress rosa expert rosaplus decompress:rosa 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 6359 0.02 0.02 0.00 8640 1 +h match expert match h:match 16384 1 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 6.5858628640659385 0.00 0.00 0.00 6272 1 +compress match expert match compress:match 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 13506 0.01 0.01 0.00 5760 1 +decompress match expert match decompress:match 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 13506 0.01 0.01 0.00 5844 1 +h match expert match h:match 16384 2 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 6.5858628640659385 0.00 0.00 0.00 6068 1 +compress match expert match compress:match 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 13506 0.01 0.00 0.00 5852 1 +decompress match expert match decompress:match 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 13506 0.01 0.01 0.00 5788 1 +h rwkv7 expert rwkv7 h:rwkv7 16384 1 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 5.850210720420741 0.27 0.26 0.00 8428 1 +compress rwkv7 expert rwkv7 compress:rwkv7 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 12000 0.24 0.24 0.00 8008 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 12000 0.24 0.24 0.00 8040 1 +h rwkv7 expert rwkv7 h:rwkv7 16384 2 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 5.850210720420741 0.23 0.23 0.00 8336 1 +compress rwkv7 expert rwkv7 compress:rwkv7 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 12000 0.24 0.24 0.00 8064 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 12000 0.24 0.24 0.00 8028 1 +h neural_mixture mixture neural-mixture h:neural_mixture 65536 1 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.3590211711615408 2.30 2.27 0.02 80828 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 19344 2.48 2.43 0.04 81128 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 19344 2.49 2.46 0.02 81608 1 +h neural_mixture mixture neural-mixture h:neural_mixture 65536 2 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.3590211711615408 2.29 2.26 0.02 80980 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 19344 2.48 2.45 0.03 81056 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 19344 2.48 2.46 0.01 81640 1 +h ctw expert ctw h:ctw 65536 1 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.7721389563404757 0.49 0.48 0.00 23324 1 +compress ctw expert ctw compress:ctw 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 22728 0.82 0.80 0.01 23480 1 +decompress ctw expert ctw decompress:ctw 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 22728 0.81 0.81 0.00 23372 1 +h ctw expert ctw h:ctw 65536 2 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.7721389563404757 0.48 0.47 0.01 23564 1 +compress ctw expert ctw compress:ctw 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 22728 0.81 0.80 0.00 23364 1 +decompress ctw expert ctw decompress:ctw 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 22728 0.82 0.81 0.00 23160 1 +h ppmd expert ppmd h:ppmd 65536 1 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.828003099069916 0.19 0.16 0.03 45576 1 +compress ppmd expert ppmd compress:ppmd 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 23186 0.21 0.20 0.00 45484 1 +decompress ppmd expert ppmd decompress:ppmd 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 23186 0.21 0.19 0.01 45408 1 +h ppmd expert ppmd h:ppmd 65536 2 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.828003099069916 0.19 0.17 0.02 45748 1 +compress ppmd expert ppmd compress:ppmd 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 23186 0.21 0.20 0.01 45496 1 +decompress ppmd expert ppmd decompress:ppmd 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 23186 0.22 0.19 0.02 45476 1 +h rosa expert rosaplus h:rosa 65536 1 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3.1905669620146786 0.65 0.63 0.02 15668 1 +compress rosa expert rosaplus compress:rosa 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 22843 0.12 0.12 0.00 18112 1 +decompress rosa expert rosaplus decompress:rosa 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 22843 0.12 0.12 0.00 18024 1 +h rosa expert rosaplus h:rosa 65536 2 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3.1905669620146786 0.66 0.65 0.01 15784 1 +compress rosa expert rosaplus compress:rosa 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 22843 0.12 0.12 0.00 17920 1 +decompress rosa expert rosaplus decompress:rosa 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 22843 0.12 0.12 0.00 18120 1 +h match expert match h:match 65536 1 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 6.558825530499182 0.01 0.01 0.00 6792 1 +compress match expert match compress:match 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 53748 0.04 0.04 0.00 6716 1 +decompress match expert match decompress:match 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 53748 0.03 0.03 0.00 7000 1 +h match expert match h:match 65536 2 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 6.558825530499182 0.01 0.00 0.00 7080 1 +compress match expert match compress:match 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 53748 0.03 0.03 0.00 6772 1 +decompress match expert match decompress:match 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 53748 0.03 0.03 0.00 6884 1 +h rwkv7 expert rwkv7 h:rwkv7 65536 1 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 4.311738518194067 0.90 0.90 0.00 8428 1 +compress rwkv7 expert rwkv7 compress:rwkv7 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 35340 0.96 0.95 0.00 8088 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 35340 0.96 0.95 0.00 8204 1 +h rwkv7 expert rwkv7 h:rwkv7 65536 2 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 4.311738518194067 0.92 0.91 0.00 8444 1 +compress rwkv7 expert rwkv7 compress:rwkv7 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 35340 0.97 0.97 0.00 8240 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 35340 0.95 0.94 0.00 8084 1 +h neural_mixture mixture neural-mixture h:neural_mixture 262144 1 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.0579561160084685 9.97 9.87 0.08 258432 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 67454 10.86 10.75 0.09 259072 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 67454 10.87 10.78 0.07 257784 1 +h neural_mixture mixture neural-mixture h:neural_mixture 262144 2 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.0579561160084685 9.99 9.87 0.10 258468 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 67454 10.86 10.75 0.09 259104 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 67454 10.92 10.83 0.08 257728 1 +h ctw expert ctw h:ctw 262144 1 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.4409671415182586 2.37 2.34 0.02 49564 1 +compress ctw expert ctw compress:ctw 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 80004 3.52 3.48 0.03 49872 1 +decompress ctw expert ctw decompress:ctw 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 80004 3.53 3.50 0.02 50116 1 +h ctw expert ctw h:ctw 262144 2 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.4409671415182586 2.38 2.36 0.01 49744 1 +compress ctw expert ctw compress:ctw 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 80004 3.51 3.50 0.01 49768 1 +decompress ctw expert ctw decompress:ctw 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 80004 3.52 3.51 0.01 49932 1 +h ppmd expert ppmd h:ppmd 262144 1 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.5405956182399834 0.91 0.85 0.05 144404 1 +compress ppmd expert ppmd compress:ppmd 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 83269 1.02 0.96 0.05 144524 1 +decompress ppmd expert ppmd decompress:ppmd 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 83269 1.04 0.98 0.06 144508 1 +h ppmd expert ppmd h:ppmd 262144 2 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.5405956182399834 0.92 0.85 0.06 144480 1 +compress ppmd expert ppmd compress:ppmd 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 83269 1.00 0.93 0.06 144240 1 +decompress ppmd expert ppmd decompress:ppmd 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 83269 1.02 0.96 0.06 144560 1 +h rosa expert rosaplus h:rosa 262144 1 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.905672544873225 3.63 3.52 0.10 45176 1 +compress rosa expert rosaplus compress:rosa 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 80590 0.69 0.69 0.00 55704 1 +decompress rosa expert rosaplus decompress:rosa 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 80590 0.71 0.70 0.01 56176 1 +h rosa expert rosaplus h:rosa 262144 2 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.905672544873225 3.61 3.53 0.07 45156 1 +compress rosa expert rosaplus compress:rosa 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 80590 0.70 0.68 0.01 55744 1 +decompress rosa expert rosaplus decompress:rosa 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 80590 0.71 0.69 0.01 55936 1 +h match expert match h:match 262144 1 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 6.292259446690552 0.03 0.03 0.00 8184 1 +compress match expert match compress:match 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 206203 0.13 0.13 0.00 8328 1 +decompress match expert match decompress:match 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 206203 0.14 0.13 0.00 8324 1 +h match expert match h:match 262144 2 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 6.292259446690552 0.03 0.03 0.00 8248 1 +compress match expert match compress:match 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 206203 0.13 0.13 0.00 8716 1 +decompress match expert match decompress:match 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 206203 0.14 0.13 0.00 8320 1 +h rwkv7 expert rwkv7 h:rwkv7 262144 1 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 4.180651265210767 3.70 3.69 0.00 8488 1 +compress rwkv7 expert rwkv7 compress:rwkv7 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 137010 3.82 3.81 0.00 8156 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 137010 3.89 3.89 0.00 8168 1 +h rwkv7 expert rwkv7 h:rwkv7 262144 2 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 4.180651265210767 3.67 3.66 0.00 8504 1 +compress rwkv7 expert rwkv7 compress:rwkv7 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 137010 3.84 3.83 0.00 8024 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 137010 3.82 3.81 0.00 8340 1 +h neural_mixture mixture neural-mixture h:neural_mixture 1048576 1 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1.939636693324927 44.68 44.34 0.29 697616 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 254251 48.94 48.56 0.33 697952 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 254251 49.00 48.59 0.36 694276 1 +h neural_mixture mixture neural-mixture h:neural_mixture 1048576 2 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1.939636693324927 44.65 44.24 0.36 697692 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 254251 49.20 48.82 0.32 697936 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 254251 49.05 48.64 0.36 694464 1 +h ctw expert ctw h:ctw 1048576 1 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.301742747548954 12.13 12.05 0.06 122672 1 +compress ctw expert ctw compress:ctw 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 301713 15.78 15.68 0.08 123132 1 +decompress ctw expert ctw decompress:ctw 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 301713 15.83 15.76 0.05 122944 1 +h ctw expert ctw h:ctw 1048576 2 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.301742747548954 12.12 12.07 0.03 122320 1 +compress ctw expert ctw compress:ctw 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 301713 15.80 15.74 0.04 122952 1 +decompress ctw expert ctw decompress:ctw 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 301713 15.87 15.81 0.04 122836 1 +h ppmd expert ppmd h:ppmd 1048576 1 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.489564753975732 3.82 3.62 0.19 399172 1 +compress ppmd expert ppmd compress:ppmd 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 326331 4.20 3.99 0.20 399868 1 +decompress ppmd expert ppmd decompress:ppmd 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 326331 4.25 4.07 0.17 399724 1 +h ppmd expert ppmd h:ppmd 1048576 2 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.489564753975732 3.81 3.59 0.20 399536 1 +compress ppmd expert ppmd compress:ppmd 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 326331 4.19 3.98 0.19 399704 1 +decompress ppmd expert ppmd decompress:ppmd 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 326331 4.26 4.08 0.17 399568 1 +h rosa expert rosaplus h:rosa 1048576 1 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.801230278247886 19.75 19.28 0.45 164524 1 +compress rosa expert rosaplus compress:rosa 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 306522 3.82 3.76 0.05 146684 1 +decompress rosa expert rosaplus decompress:rosa 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 306522 3.93 3.88 0.04 146936 1 +h rosa expert rosaplus h:rosa 1048576 2 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.801230278247886 19.75 19.26 0.46 164596 1 +compress rosa expert rosaplus compress:rosa 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 306522 3.84 3.76 0.07 146728 1 +decompress rosa expert rosaplus decompress:rosa 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 306522 3.89 3.84 0.04 146880 1 +h match expert match h:match 1048576 1 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 6.236926355106452 0.13 0.13 0.00 11880 1 +compress match expert match compress:match 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 817505 0.53 0.52 0.00 12452 1 +decompress match expert match decompress:match 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 817505 0.55 0.55 0.00 11960 1 +h match expert match h:match 1048576 2 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 6.236926355106452 0.13 0.12 0.00 11780 1 +compress match expert match compress:match 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 817505 0.53 0.52 0.00 12576 1 +decompress match expert match decompress:match 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 817505 0.55 0.55 0.00 11720 1 +h rwkv7 expert rwkv7 h:rwkv7 1048576 1 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3.6182904815139514 15.28 15.27 0.00 9060 1 +compress rwkv7 expert rwkv7 compress:rwkv7 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 474275 15.73 15.64 0.07 9592 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 474275 15.80 15.78 0.00 9212 1 +h rwkv7 expert rwkv7 h:rwkv7 1048576 2 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3.6182904815139514 14.84 14.83 0.00 9092 1 +compress rwkv7 expert rwkv7 compress:rwkv7 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 474275 15.91 15.82 0.07 9804 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 474275 15.76 15.74 0.00 9340 1 +h neural_mixture mixture neural-mixture h:neural_mixture 2097152 1 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1.9206985464400697 94.37 93.80 0.48 1013276 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 2097152 1 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 503520 103.89 103.28 0.50 1013616 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 2097152 1 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 503520 104.20 103.62 0.47 1024488 1 +h neural_mixture mixture neural-mixture h:neural_mixture 2097152 2 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1.9206985464400697 94.44 93.89 0.45 1013092 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 503520 103.58 103.01 0.46 1013660 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 503520 104.22 103.60 0.51 1019696 1 +h ctw expert ctw h:ctw 2097152 1 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.277315456609421 26.98 26.89 0.07 199208 1 +compress ctw expert ctw compress:ctw 2097152 1 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 597003 33.39 33.24 0.11 200784 1 +decompress ctw expert ctw decompress:ctw 2097152 1 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 597003 33.64 33.54 0.07 199972 1 +h ctw expert ctw h:ctw 2097152 2 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.277315456609421 27.05 26.94 0.08 199140 1 +compress ctw expert ctw compress:ctw 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 597003 33.37 33.23 0.11 200572 1 +decompress ctw expert ctw decompress:ctw 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 597003 33.66 33.56 0.07 200144 1 +h ppmd expert ppmd h:ppmd 2097152 1 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.525080573022069 7.35 7.15 0.19 457504 1 +compress ppmd expert ppmd compress:ppmd 2097152 1 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 661953 8.16 7.91 0.24 457876 1 +decompress ppmd expert ppmd decompress:ppmd 2097152 1 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 661953 8.24 8.04 0.19 457888 1 +h ppmd expert ppmd h:ppmd 2097152 2 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.525080573022069 7.36 7.13 0.22 457504 1 +compress ppmd expert ppmd compress:ppmd 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 661953 8.15 7.93 0.21 458112 1 +decompress ppmd expert ppmd decompress:ppmd 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 661953 8.25 8.01 0.22 458068 1 +h rosa expert rosaplus h:rosa 2097152 1 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.704371387016519 47.41 45.52 1.83 328348 1 +compress rosa expert rosaplus compress:rosa 2097152 1 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 608681 9.39 9.26 0.12 285184 1 +decompress rosa expert rosaplus decompress:rosa 2097152 1 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 608681 9.52 9.42 0.09 285684 1 +h rosa expert rosaplus h:rosa 2097152 2 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.704371387016519 47.46 45.73 1.67 328472 1 +compress rosa expert rosaplus compress:rosa 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 608681 9.30 9.17 0.11 285048 1 +decompress rosa expert rosaplus decompress:rosa 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 608681 9.41 9.25 0.15 285816 1 +h match expert match h:match 2097152 1 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 6.274177264896603 0.27 0.26 0.01 19584 1 +compress match expert match compress:match 2097152 1 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1644756 1.06 1.06 0.00 22396 1 +decompress match expert match decompress:match 2097152 1 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1644756 1.10 1.09 0.00 20532 1 +h match expert match h:match 2097152 2 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 6.274177264896603 0.28 0.28 0.00 19772 1 +compress match expert match compress:match 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1644756 1.07 1.06 0.00 22652 1 +decompress match expert match decompress:match 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1644756 1.11 1.10 0.00 20380 1 +h rwkv7 expert rwkv7 h:rwkv7 2097152 1 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3.498336499657132 30.07 30.05 0.00 10280 1 +compress rwkv7 expert rwkv7 compress:rwkv7 2097152 1 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 917086 32.93 32.31 0.59 11696 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 2097152 1 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 917086 31.77 31.74 0.00 10760 1 +h rwkv7 expert rwkv7 h:rwkv7 2097152 2 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3.498336499657132 30.41 30.39 0.00 10164 1 +compress rwkv7 expert rwkv7 compress:rwkv7 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 917086 32.52 31.95 0.54 11648 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 917086 31.74 31.72 0.00 10752 1 +h neural_mixture mixture neural-mixture h:neural_mixture 4194304 1 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1.885096705283187 200.05 199.26 0.59 1472544 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 4194304 1 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 988355 221.34 220.53 0.59 1440560 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 4194304 1 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 988355 221.50 220.73 0.55 1462616 1 +h neural_mixture mixture neural-mixture h:neural_mixture 4194304 2 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1.885096705283187 200.04 199.23 0.61 1472480 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 988355 220.51 219.69 0.60 1440512 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 988355 221.56 220.74 0.60 1455332 1 +h ctw expert ctw h:ctw 4194304 1 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.246749307224066 59.39 59.17 0.16 326648 1 +compress ctw expert ctw compress:ctw 4194304 1 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1177962 70.38 70.17 0.15 329148 1 +decompress ctw expert ctw decompress:ctw 4194304 1 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1177962 70.62 70.42 0.13 328000 1 +h ctw expert ctw h:ctw 4194304 2 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.246749307224066 59.57 59.37 0.14 326508 1 +compress ctw expert ctw compress:ctw 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1177962 70.89 70.67 0.15 329088 1 +decompress ctw expert ctw decompress:ctw 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1177962 70.98 70.72 0.20 327928 1 +h ppmd expert ppmd h:ppmd 4194304 1 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.5353209835043775 14.66 14.39 0.25 459572 1 +compress ppmd expert ppmd compress:ppmd 4194304 1 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1329257 16.40 16.17 0.21 459908 1 +decompress ppmd expert ppmd decompress:ppmd 4194304 1 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1329257 16.34 16.13 0.19 458520 1 +h ppmd expert ppmd h:ppmd 4194304 2 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.5353209835043775 14.70 14.47 0.21 459616 1 +compress ppmd expert ppmd compress:ppmd 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1329257 16.08 15.82 0.24 460260 1 +decompress ppmd expert ppmd decompress:ppmd 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1329257 16.50 16.25 0.22 458516 1 +h rosa expert rosaplus h:rosa 4194304 1 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.62099230176433 113.72 108.74 4.84 641936 1 +compress rosa expert rosaplus compress:rosa 4194304 1 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1199345 23.27 22.99 0.26 564572 1 +decompress rosa expert rosaplus decompress:rosa 4194304 1 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1199345 23.58 23.32 0.23 553580 1 +h rosa expert rosaplus h:rosa 4194304 2 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.62099230176433 113.62 108.55 4.93 641864 1 +compress rosa expert rosaplus compress:rosa 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1199345 23.48 23.20 0.25 564796 1 +decompress rosa expert rosaplus decompress:rosa 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1199345 23.46 23.18 0.25 553472 1 +h match expert match h:match 4194304 1 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 6.280388008990573 0.55 0.54 0.00 21592 1 +compress match expert match compress:match 4194304 1 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3292751 2.12 2.11 0.01 29284 1 +decompress match expert match decompress:match 4194304 1 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3292751 2.21 2.20 0.00 24216 1 +h match expert match h:match 4194304 2 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 6.280388008990573 0.56 0.55 0.00 21532 1 +compress match expert match compress:match 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3292751 2.14 2.12 0.01 29444 1 +decompress match expert match decompress:match 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3292751 2.21 2.21 0.00 24140 1 +h rwkv7 expert rwkv7 h:rwkv7 4194304 1 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3.3357009871104677 62.07 62.02 0.00 12180 1 +compress rwkv7 expert rwkv7 compress:rwkv7 4194304 1 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1748887 66.32 64.55 1.72 15128 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 4194304 1 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1748887 64.69 64.64 0.00 13484 1 +h rwkv7 expert rwkv7 h:rwkv7 4194304 2 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3.3357009871104677 61.68 61.63 0.00 12356 1 +compress rwkv7 expert rwkv7 compress:rwkv7 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1748887 66.14 64.45 1.64 15188 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1748887 64.10 64.06 0.00 13568 1 +h neural_mixture mixture neural-mixture h:neural_mixture 10000000 1 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1.816175473708421 512.81 510.94 1.34 2653560 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 10000000 1 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2270248 574.12 572.24 1.29 2660716 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 10000000 1 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2270248 574.84 572.96 1.26 2655488 1 +h neural_mixture mixture neural-mixture h:neural_mixture 10000000 2 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1.816175473708421 512.83 510.97 1.35 2653564 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2270248 573.90 571.95 1.34 2660544 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2270248 574.42 572.48 1.35 2662604 1 +h ctw expert ctw h:ctw 10000000 1 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.1974739848061873 157.79 157.35 0.30 609276 1 +compress ctw expert ctw compress:ctw 10000000 1 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2746861 178.50 178.01 0.32 614476 1 +decompress ctw expert ctw decompress:ctw 10000000 1 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2746861 179.89 179.36 0.36 611972 1 +h ctw expert ctw h:ctw 10000000 2 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.1974739848061873 157.79 157.28 0.36 609104 1 +compress ctw expert ctw compress:ctw 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2746861 178.49 178.04 0.29 614524 1 +decompress ctw expert ctw decompress:ctw 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2746861 178.66 178.19 0.31 611808 1 +h ppmd expert ppmd h:ppmd 10000000 1 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.523558147667331 35.44 35.09 0.31 557548 1 +compress ppmd expert ppmd compress:ppmd 10000000 1 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3154465 39.40 39.12 0.24 569112 1 +decompress ppmd expert ppmd decompress:ppmd 10000000 1 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3154465 39.71 39.41 0.26 559492 1 +h ppmd expert ppmd h:ppmd 10000000 2 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.523558147667331 35.62 35.29 0.29 557532 1 +compress ppmd expert ppmd compress:ppmd 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3154465 39.00 38.69 0.27 569300 1 +decompress ppmd expert ppmd decompress:ppmd 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3154465 39.74 39.40 0.30 559392 1 +h rosa expert rosaplus h:rosa 10000000 1 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.484818977912099 337.29 324.35 12.55 1458136 1 +compress rosa expert rosaplus compress:rosa 10000000 1 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2752778 73.54 72.84 0.63 1302308 1 +decompress rosa expert rosaplus decompress:rosa 10000000 1 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2752778 73.96 73.19 0.70 1304496 1 +h rosa expert rosaplus h:rosa 10000000 2 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.484818977912099 337.08 323.93 12.76 1457964 1 +compress rosa expert rosaplus compress:rosa 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2752778 73.71 73.03 0.60 1302236 1 +decompress rosa expert rosaplus decompress:rosa 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2752778 74.80 74.15 0.57 1304616 1 +h match expert match h:match 10000000 1 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 6.287444365927943 1.35 1.33 0.01 40752 1 +compress match expert match compress:match 10000000 1 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 7859324 5.12 5.09 0.02 55756 1 +decompress match expert match decompress:match 10000000 1 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 7859324 5.36 5.33 0.02 46508 1 +h match expert match h:match 10000000 2 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 6.287444365927943 1.39 1.37 0.02 40812 1 +compress match expert match compress:match 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 7859324 5.20 5.17 0.02 55760 1 +decompress match expert match decompress:match 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 7859324 5.31 5.28 0.02 46496 1 +h rwkv7 expert rwkv7 h:rwkv7 10000000 1 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3.174902781170517 147.81 147.70 0.00 17856 1 +compress rwkv7 expert rwkv7 compress:rwkv7 10000000 1 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3968645 158.38 156.27 1.99 25204 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 10000000 1 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3968645 152.62 152.51 0.00 21536 1 +h rwkv7 expert rwkv7 h:rwkv7 10000000 2 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3.174902781170517 146.56 146.46 0.00 17784 1 +compress rwkv7 expert rwkv7 compress:rwkv7 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3968645 156.35 154.44 1.78 25088 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3968645 153.68 153.57 0.00 21484 1 diff --git a/benchmarks/77b5f61f/infotheory-two-json-summary-20260510-132339.tsv b/benchmarks/77b5f61f/infotheory-two-json-summary-20260510-132339.tsv new file mode 100644 index 00000000..53df0bc4 --- /dev/null +++ b/benchmarks/77b5f61f/infotheory-two-json-summary-20260510-132339.tsv @@ -0,0 +1,145 @@ +operation subject subject_kind expert_kind series size_bytes repeats cpu compression_backend input_sha256 suite_spec_path suite_spec_sha256 build_mode build_features real_seconds_mean real_seconds_stdev real_seconds_median real_seconds_min real_seconds_max user_seconds_mean sys_seconds_mean throughput_mib_s_mean throughput_mib_s_median rss_kib_mean rss_kib_stdev rss_kib_median rss_kib_min rss_kib_max archive_bytes_mean archive_bytes_median archive_ratio_mean archive_ratio_median entropy_bpb_mean entropy_bpb_median verified_all +h ctw expert ctw h:ctw 4096 2 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.02 0 0.02 0.02 0.02 0.015 0 0.1953125 0.1953125 7120 33.941125497 7120 7096 7144 2.56549624542 2.56549624542 1 +h ctw expert ctw h:ctw 16384 2 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.1 0 0.1 0.1 0.1 0.1 0 0.15625 0.15625 12778 195.161471607 12778 12640 12916 3.20340951209 3.20340951209 1 +h ctw expert ctw h:ctw 65536 2 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.485 0.00707106781187 0.485 0.48 0.49 0.475 0.005 0.128879676871 0.128879676871 23444 169.705627485 23444 23324 23564 2.77213895634 2.77213895634 1 +h ctw expert ctw h:ctw 262144 2 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.375 0.00707106781187 2.375 2.37 2.38 2.35 0.015 0.105263624437 0.105263624437 49654 127.279220614 49654 49564 49744 2.44096714152 2.44096714152 1 +h ctw expert ctw h:ctw 1048576 2 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 12.125 0.00707106781187 12.125 12.12 12.13 12.06 0.045 0.0824742408289 0.0824742408289 122496 248.901586978 122496 122320 122672 2.30174274755 2.30174274755 1 +h ctw expert ctw h:ctw 2097152 2 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 27.015 0.0494974746831 27.015 26.98 27.05 26.915 0.075 0.0740330689263 0.0740330689263 199174 48.0832611207 199174 199140 199208 2.27731545661 2.27731545661 1 +h ctw expert ctw h:ctw 4194304 2 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 59.48 0.127279220614 59.48 59.39 59.57 59.27 0.15 0.0672496495977 0.0672496495977 326578 98.9949493661 326578 326508 326648 2.24674930722 2.24674930722 1 +h ctw expert ctw h:ctw 10000000 2 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 157.79 0 157.79 157.79 157.79 157.315 0.33 0.0604394648841 0.0604394648841 609190 121.622366364 609190 609104 609276 2.19747398481 2.19747398481 1 +h match expert match h:match 4096 2 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0 0 0 0 0 0 0 5968 22.627416998 5968 5952 5984 5.57734993252 5.57734993252 1 +h match expert match h:match 16384 2 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0 0 0 0 0 0 0 6170 144.249783362 6170 6068 6272 6.58586286407 6.58586286407 1 +h match expert match h:match 65536 2 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.01 0 0.01 0.01 0.01 0.005 0 6.25 6.25 6936 203.646752982 6936 6792 7080 6.5588255305 6.5588255305 1 +h match expert match h:match 262144 2 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.03 0 0.03 0.03 0.03 0.03 0 8.33333333333 8.33333333333 8216 45.2548339959 8216 8184 8248 6.29225944669 6.29225944669 1 +h match expert match h:match 1048576 2 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.13 0 0.13 0.13 0.13 0.125 0 7.69230769231 7.69230769231 11830 70.7106781187 11830 11780 11880 6.23692635511 6.23692635511 1 +h match expert match h:match 2097152 2 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.275 0.00707106781187 0.275 0.27 0.28 0.27 0.005 7.27513227513 7.27513227513 19678 132.936074863 19678 19584 19772 6.2741772649 6.2741772649 1 +h match expert match h:match 4194304 2 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.555 0.00707106781187 0.555 0.55 0.56 0.545 0 7.20779220779 7.20779220779 21562 42.4264068712 21562 21532 21592 6.28038800899 6.28038800899 1 +h match expert match h:match 10000000 2 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1.37 0.0282842712475 1.37 1.35 1.39 1.35 0.015 6.96261025034 6.96261025034 40782 42.4264068712 40782 40752 40812 6.28744436593 6.28744436593 1 +h neural_mixture mixture neural-mixture h:neural_mixture 4096 2 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.12 0 0.12 0.12 0.12 0.12 0 0.0325520833333 0.0325520833333 12358 14.1421356237 12358 12348 12368 1.91857366907 1.91857366907 1 +h neural_mixture mixture neural-mixture h:neural_mixture 16384 2 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.52 0 0.52 0.52 0.52 0.51 0.005 0.0300480769231 0.0300480769231 29974 127.279220614 29974 29884 30064 2.69245779825 2.69245779825 1 +h neural_mixture mixture neural-mixture h:neural_mixture 65536 2 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.295 0.00707106781187 2.295 2.29 2.3 2.265 0.02 0.0272332447313 0.0272332447313 80904 107.48023074 80904 80828 80980 2.35902117116 2.35902117116 1 +h neural_mixture mixture neural-mixture h:neural_mixture 262144 2 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 9.98 0.0141421356237 9.98 9.97 9.99 9.87 0.09 0.025050125351 0.025050125351 258450 25.4558441227 258450 258432 258468 2.05795611601 2.05795611601 1 +h neural_mixture mixture neural-mixture h:neural_mixture 1048576 2 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 44.665 0.0212132034356 44.665 44.65 44.68 44.29 0.325 0.0223888976331 0.0223888976331 697654 53.7401153702 697654 697616 697692 1.93963669332 1.93963669332 1 +h neural_mixture mixture neural-mixture h:neural_mixture 2097152 2 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 94.405 0.0494974746831 94.405 94.37 94.44 93.845 0.465 0.0211853214862 0.0211853214862 1013184 130.107647738 1013184 1013092 1013276 1.92069854644 1.92069854644 1 +h neural_mixture mixture neural-mixture h:neural_mixture 4194304 2 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 200.045 0.00707106781188 200.045 200.04 200.05 199.245 0.6 0.0199955010248 0.0199955010248 1472512 45.2548339959 1472512 1472480 1472544 1.88509670528 1.88509670528 1 +h neural_mixture mixture neural-mixture h:neural_mixture 10000000 2 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 512.82 0.0141421356238 512.82 512.81 512.83 510.955 1.345 0.0185966677737 0.0185966677737 2653562 2.82842712475 2653562 2653560 2653564 1.81617547371 1.81617547371 1 +h ppmd expert ppmd h:ppmd 4096 2 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.01 0 0.01 0.01 0.01 0.005 0 0.390625 0.390625 7750 161.220346111 7750 7636 7864 2.02916276221 2.02916276221 1 +h ppmd expert ppmd h:ppmd 16384 2 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.03 0 0.03 0.03 0.03 0.02 0 0.520833333333 0.520833333333 16556 5.65685424949 16556 16552 16560 3.05111741489 3.05111741489 1 +h ppmd expert ppmd h:ppmd 65536 2 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.19 0 0.19 0.19 0.19 0.165 0.025 0.328947368421 0.328947368421 45662 121.622366364 45662 45576 45748 2.82800309907 2.82800309907 1 +h ppmd expert ppmd h:ppmd 262144 2 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.915 0.00707106781187 0.915 0.91 0.92 0.85 0.055 0.27323220258 0.27323220258 144442 53.7401153702 144442 144404 144480 2.54059561824 2.54059561824 1 +h ppmd expert ppmd h:ppmd 1048576 2 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3.815 0.00707106781187 3.815 3.81 3.82 3.605 0.195 0.262123648157 0.262123648157 399354 257.386868352 399354 399172 399536 2.48956475398 2.48956475398 1 +h ppmd expert ppmd h:ppmd 2097152 2 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 7.355 0.00707106781187 7.355 7.35 7.36 7.14 0.205 0.271923986986 0.271923986986 457504 0 457504 457504 457504 2.52508057302 2.52508057302 1 +h ppmd expert ppmd h:ppmd 4194304 2 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 14.68 0.0282842712475 14.68 14.66 14.7 14.43 0.23 0.272480069791 0.272480069791 459594 31.1126983722 459594 459572 459616 2.5353209835 2.5353209835 1 +h ppmd expert ppmd h:ppmd 10000000 2 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 35.53 0.127279220614 35.53 35.44 35.62 35.19 0.3 0.268415546199 0.268415546199 557540 11.313708499 557540 557532 557548 2.52355814767 2.52355814767 1 +h rosa expert rosaplus h:rosa 4096 2 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.03 0 0.03 0.03 0.03 0.03 0 0.130208333333 0.130208333333 6672 101.823376491 6672 6600 6744 2.11172124286 2.11172124286 1 +h rosa expert rosaplus h:rosa 16384 2 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.135 0.00707106781187 0.135 0.13 0.14 0.12 0.01 0.115899725275 0.115899725275 8158 161.220346111 8158 8044 8272 3.48630622986 3.48630622986 1 +h rosa expert rosaplus h:rosa 65536 2 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.655 0.00707106781187 0.655 0.65 0.66 0.64 0.015 0.0954254079254 0.0954254079254 15726 82.0243866176 15726 15668 15784 3.19056696201 3.19056696201 1 +h rosa expert rosaplus h:rosa 262144 2 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3.62 0.0141421356237 3.62 3.61 3.63 3.525 0.085 0.0690613004892 0.0690613004892 45166 14.1421356237 45166 45156 45176 2.90567254487 2.90567254487 1 +h rosa expert rosaplus h:rosa 1048576 2 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 19.75 0 19.75 19.75 19.75 19.27 0.455 0.0506329113924 0.0506329113924 164560 50.9116882454 164560 164524 164596 2.80123027825 2.80123027825 1 +h rosa expert rosaplus h:rosa 2097152 2 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 47.435 0.0353553390593 47.435 47.41 47.46 45.625 1.75 0.0421629715513 0.0421629715513 328410 87.6812408671 328410 328348 328472 2.70437138702 2.70437138702 1 +h rosa expert rosaplus h:rosa 4194304 2 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 113.67 0.0707106781187 113.67 113.62 113.72 108.645 4.885 0.0351895906918 0.0351895906918 641900 50.9116882454 641900 641864 641936 2.62099230176 2.62099230176 1 +h rosa expert rosaplus h:rosa 10000000 2 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 337.185 0.148492424049 337.185 337.08 337.29 324.14 12.655 0.0282834173787 0.0282834173787 1458050 121.622366364 1458050 1457964 1458136 2.48481897791 2.48481897791 1 +h rwkv7 expert rwkv7 h:rwkv7 4096 2 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.075 0.0212132034356 0.075 0.06 0.09 0.075 0 0.0542534722222 0.0542534722222 8412 39.5979797464 8412 8384 8440 7.21701437947 7.21701437947 1 +h rwkv7 expert rwkv7 h:rwkv7 16384 2 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.25 0.0282842712475 0.25 0.23 0.27 0.245 0 0.0629025764895 0.0629025764895 8382 65.0538238692 8382 8336 8428 5.85021072042 5.85021072042 1 +h rwkv7 expert rwkv7 h:rwkv7 65536 2 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.91 0.0141421356237 0.91 0.9 0.92 0.905 0 0.0686896135266 0.0686896135266 8436 11.313708499 8436 8428 8444 4.31173851819 4.31173851819 1 +h rwkv7 expert rwkv7 h:rwkv7 262144 2 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3.685 0.0212132034356 3.685 3.67 3.7 3.675 0 0.0678437292879 0.0678437292879 8496 11.313708499 8496 8488 8504 4.18065126521 4.18065126521 1 +h rwkv7 expert rwkv7 h:rwkv7 1048576 2 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 15.06 0.311126983722 15.06 14.84 15.28 15.05 0 0.066415235461 0.066415235461 9076 22.627416998 9076 9060 9092 3.61829048151 3.61829048151 1 +h rwkv7 expert rwkv7 h:rwkv7 2097152 2 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 30.24 0.240416305603 30.24 30.07 30.41 30.22 0 0.0661396563778 0.0661396563778 10222 82.0243866176 10222 10164 10280 3.49833649966 3.49833649966 1 +h rwkv7 expert rwkv7 h:rwkv7 4194304 2 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 61.875 0.275771644663 61.875 61.68 62.07 61.825 0 0.0646471067246 0.0646471067246 12268 124.450793489 12268 12180 12356 3.33570098711 3.33570098711 1 +h rwkv7 expert rwkv7 h:rwkv7 10000000 2 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 147.185 0.883883476483 147.185 146.56 147.81 147.08 0 0.0647954284022 0.0647954284022 17820 50.9116882454 17820 17784 17856 3.17490278117 3.17490278117 1 +compress ctw expert ctw compress:ctw 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.04 0 0.04 0.04 0.04 0.035 0 0.09765625 0.09765625 7166 2.82842712475 7166 7164 7168 1332 1332 0.3251953125 0.3251953125 1 +compress ctw expert ctw compress:ctw 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.18 0 0.18 0.18 0.18 0.175 0 0.0868055555556 0.0868055555556 12910 53.7401153702 12910 12872 12948 6579 6579 0.401550292969 0.401550292969 1 +compress ctw expert ctw compress:ctw 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.815 0.00707106781187 0.815 0.81 0.82 0.8 0.005 0.0766900030111 0.0766900030111 23422 82.0243866176 23422 23364 23480 22728 22728 0.346801757812 0.346801757812 1 +compress ctw expert ctw compress:ctw 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3.515 0.00707106781187 3.515 3.51 3.52 3.49 0.02 0.0711238992489 0.0711238992489 49820 73.5391052434 49820 49768 49872 80004 80004 0.305191040039 0.305191040039 1 +compress ctw expert ctw compress:ctw 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 15.79 0.0141421356237 15.79 15.78 15.8 15.71 0.06 0.0633312476938 0.0633312476938 123042 127.279220614 123042 122952 123132 301713 301713 0.287735939026 0.287735939026 1 +compress ctw expert ctw compress:ctw 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 33.38 0.0141421356237 33.38 33.37 33.39 33.235 0.11 0.059916122813 0.059916122813 200678 149.906637612 200678 200572 200784 597003 597003 0.284673213959 0.284673213959 1 +compress ctw expert ctw compress:ctw 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 70.635 0.360624458405 70.635 70.38 70.89 70.42 0.15 0.0566298879055 0.0566298879055 329118 42.4264068712 329118 329088 329148 1177962 1177962 0.280848026276 0.280848026276 1 +compress ctw expert ctw compress:ctw 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 178.495 0.00707106781186 178.495 178.49 178.5 178.025 0.305 0.053428629214 0.053428629214 614500 33.941125497 614500 614476 614524 2746861 2746861 0.2746861 0.2746861 1 +compress match expert match compress:match 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0 0 0 0 0 0 0 5824 22.627416998 5824 5808 5840 2874 2874 0.70166015625 0.70166015625 1 +compress match expert match compress:match 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.01 0 0.01 0.01 0.01 0.005 0 1.5625 1.5625 5806 65.0538238692 5806 5760 5852 13506 13506 0.824340820312 0.824340820312 1 +compress match expert match compress:match 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.035 0.00707106781187 0.035 0.03 0.04 0.035 0 1.82291666667 1.82291666667 6744 39.5979797464 6744 6716 6772 53748 53748 0.820129394531 0.820129394531 1 +compress match expert match compress:match 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.13 0 0.13 0.13 0.13 0.13 0 1.92307692308 1.92307692308 8522 274.3574311 8522 8328 8716 206203 206203 0.786602020264 0.786602020264 1 +compress match expert match compress:match 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.53 0 0.53 0.53 0.53 0.52 0 1.88679245283 1.88679245283 12514 87.6812408671 12514 12452 12576 817505 817505 0.779633522034 0.779633522034 1 +compress match expert match compress:match 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1.065 0.00707106781187 1.065 1.06 1.07 1.06 0 1.87797566567 1.87797566567 22524 181.019335984 22524 22396 22652 1644756 1644756 0.784280776978 0.784280776978 1 +compress match expert match compress:match 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.13 0.0141421356237 2.13 2.12 2.14 2.115 0.01 1.87797566567 1.87797566567 29364 113.13708499 29364 29284 29444 3292751 3292751 0.785053014755 0.785053014755 1 +compress match expert match compress:match 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 5.16 0.0565685424949 5.16 5.12 5.2 5.13 0.02 1.84831710962 1.84831710962 55758 2.82842712475 55758 55756 55760 7859324 7859324 0.7859324 0.7859324 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.13 0 0.13 0.13 0.13 0.125 0 0.0300480769231 0.0300480769231 12172 67.8822509939 12172 12124 12220 1001 1001 0.244384765625 0.244384765625 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.57 0 0.57 0.57 0.57 0.555 0.005 0.0274122807018 0.0274122807018 30540 28.2842712475 30540 30520 30560 5533 5533 0.337707519531 0.337707519531 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.48 0 2.48 2.48 2.48 2.44 0.035 0.0252016129032 0.0252016129032 81092 50.9116882454 81092 81056 81128 19344 19344 0.295166015625 0.295166015625 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 10.86 0 10.86 10.86 10.86 10.75 0.09 0.0230202578269 0.0230202578269 259088 22.627416998 259088 259072 259104 67454 67454 0.257316589355 0.257316589355 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 49.07 0.183847763109 49.07 48.94 49.2 48.69 0.325 0.020379193371 0.020379193371 697944 11.313708499 697944 697936 697952 254251 254251 0.242472648621 0.242472648621 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 103.735 0.219203102168 103.735 103.58 103.89 103.145 0.48 0.0192799389331 0.0192799389331 1013638 31.1126983722 1013638 1013616 1013660 503520 503520 0.240097045898 0.240097045898 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 220.925 0.586898628385 220.925 220.51 221.34 220.11 0.595 0.0181057558655 0.0181057558655 1440536 33.941125497 1440536 1440512 1440560 988355 988355 0.235642194748 0.235642194748 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 574.01 0.155563491861 574.01 573.9 574.12 572.095 1.315 0.0166142462924 0.0166142462924 2660630 121.622366364 2660630 2660544 2660716 2270248 2270248 0.2270248 0.2270248 1 +compress ppmd expert ppmd compress:ppmd 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.01 0 0.01 0.01 0.01 0.005 0 0.390625 0.390625 7580 45.2548339959 7580 7548 7612 1058 1058 0.25830078125 0.25830078125 1 +compress ppmd expert ppmd compress:ppmd 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.04 0 0.04 0.04 0.04 0.025 0.01 0.390625 0.390625 16450 274.3574311 16450 16256 16644 6267 6267 0.382507324219 0.382507324219 1 +compress ppmd expert ppmd compress:ppmd 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.21 0 0.21 0.21 0.21 0.2 0.005 0.297619047619 0.297619047619 45490 8.48528137424 45490 45484 45496 23186 23186 0.353790283203 0.353790283203 1 +compress ppmd expert ppmd compress:ppmd 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1.01 0.0141421356237 1.01 1 1.02 0.945 0.055 0.247549019608 0.247549019608 144382 200.818325857 144382 144240 144524 83269 83269 0.317646026611 0.317646026611 1 +compress ppmd expert ppmd compress:ppmd 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 4.195 0.00707106781187 4.195 4.19 4.2 3.985 0.195 0.238379361291 0.238379361291 399786 115.965512115 399786 399704 399868 326331 326331 0.311213493347 0.311213493347 1 +compress ppmd expert ppmd compress:ppmd 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 8.155 0.00707106781187 8.155 8.15 8.16 7.92 0.225 0.245248406111 0.245248406111 457994 166.87720036 457994 457876 458112 661953 661953 0.315643787384 0.315643787384 1 +compress ppmd expert ppmd compress:ppmd 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 16.24 0.22627416998 16.24 16.08 16.4 15.995 0.225 0.246329328965 0.246329328965 460084 248.901586978 460084 459908 460260 1329257 1329257 0.316919565201 0.316919565201 1 +compress ppmd expert ppmd compress:ppmd 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 39.2 0.282842712475 39.2 39 39.4 38.905 0.255 0.243290597443 0.243290597443 569206 132.936074863 569206 569112 569300 3154465 3154465 0.3154465 0.3154465 1 +compress rosa expert rosaplus compress:rosa 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0 0 0 0 0 0 0 6172 96.1665222414 6172 6104 6240 1127 1127 0.275146484375 0.275146484375 1 +compress rosa expert rosaplus compress:rosa 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.02 0 0.02 0.02 0.02 0.02 0 0.78125 0.78125 8774 195.161471607 8774 8636 8912 6359 6359 0.388122558594 0.388122558594 1 +compress rosa expert rosaplus compress:rosa 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.12 0 0.12 0.12 0.12 0.12 0 0.520833333333 0.520833333333 18016 135.764501988 18016 17920 18112 22843 22843 0.348556518555 0.348556518555 1 +compress rosa expert rosaplus compress:rosa 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.695 0.00707106781187 0.695 0.69 0.7 0.685 0.005 0.359730848861 0.359730848861 55724 28.2842712475 55724 55704 55744 80590 80590 0.307426452637 0.307426452637 1 +compress rosa expert rosaplus compress:rosa 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3.83 0.0141421356237 3.83 3.82 3.84 3.76 0.06 0.261098385689 0.261098385689 146706 31.1126983722 146706 146684 146728 306522 306522 0.292322158813 0.292322158813 1 +compress rosa expert rosaplus compress:rosa 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 9.345 0.0636396103068 9.345 9.3 9.39 9.215 0.115 0.214023154351 0.214023154351 285116 96.1665222414 285116 285048 285184 608681 608681 0.290241718292 0.290241718292 1 +compress rosa expert rosaplus compress:rosa 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 23.375 0.148492424049 23.375 23.27 23.48 23.095 0.255 0.17112644762 0.17112644762 564684 158.391918986 564684 564572 564796 1199345 1199345 0.285946130753 0.285946130753 1 +compress rosa expert rosaplus compress:rosa 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 73.625 0.120208152802 73.625 73.54 73.71 72.935 0.615 0.129531488968 0.129531488968 1302272 50.9116882454 1302272 1302236 1302308 2752778 2752778 0.2752778 0.2752778 1 +compress rwkv7 expert rwkv7 compress:rwkv7 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.06 0 0.06 0.06 0.06 0.06 0 0.0651041666667 0.0651041666667 8048 67.8822509939 8048 8000 8096 3714 3714 0.90673828125 0.90673828125 1 +compress rwkv7 expert rwkv7 compress:rwkv7 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.24 0 0.24 0.24 0.24 0.24 0 0.0651041666667 0.0651041666667 8036 39.5979797464 8036 8008 8064 12000 12000 0.732421875 0.732421875 1 +compress rwkv7 expert rwkv7 compress:rwkv7 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.965 0.00707106781187 0.965 0.96 0.97 0.96 0 0.0647685781787 0.0647685781787 8164 107.48023074 8164 8088 8240 35340 35340 0.539245605469 0.539245605469 1 +compress rwkv7 expert rwkv7 compress:rwkv7 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3.83 0.0141421356237 3.83 3.82 3.84 3.82 0 0.0652745964223 0.0652745964223 8090 93.3380951166 8090 8024 8156 137010 137010 0.522651672363 0.522651672363 1 +compress rwkv7 expert rwkv7 compress:rwkv7 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 15.82 0.127279220614 15.82 15.73 15.91 15.73 0.07 0.0632131710356 0.0632131710356 9698 149.906637612 9698 9592 9804 474275 474275 0.452303886414 0.452303886414 1 +compress rwkv7 expert rwkv7 compress:rwkv7 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 32.725 0.289913780286 32.725 32.52 32.93 32.13 0.565 0.0611177536009 0.0611177536009 11672 33.941125497 11672 11648 11696 917086 917086 0.437300682068 0.437300682068 1 +compress rwkv7 expert rwkv7 compress:rwkv7 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 66.23 0.127279220614 66.23 66.14 66.32 64.5 1.68 0.0603957026492 0.0603957026492 15158 42.4264068712 15158 15128 15188 1748887 1748887 0.416967153549 0.416967153549 1 +compress rwkv7 expert rwkv7 compress:rwkv7 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 157.365 1.43542676581 157.365 156.35 158.38 155.355 1.885 0.0606052167229 0.0606052167229 25146 82.0243866176 25146 25088 25204 3968645 3968645 0.3968645 0.3968645 1 +decompress ctw expert ctw decompress:ctw 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.03 0 0.03 0.03 0.03 0.03 0 0.130208333333 0.130208333333 7266 42.4264068712 7266 7236 7296 1332 1332 0.3251953125 0.3251953125 1 +decompress ctw expert ctw decompress:ctw 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.18 0 0.18 0.18 0.18 0.175 0 0.0868055555556 0.0868055555556 13052 5.65685424949 13052 13048 13056 6579 6579 0.401550292969 0.401550292969 1 +decompress ctw expert ctw decompress:ctw 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.815 0.00707106781187 0.815 0.81 0.82 0.81 0 0.0766900030111 0.0766900030111 23266 149.906637612 23266 23160 23372 22728 22728 0.346801757812 0.346801757812 1 +decompress ctw expert ctw decompress:ctw 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3.525 0.00707106781187 3.525 3.52 3.53 3.505 0.015 0.0709221285089 0.0709221285089 50024 130.107647738 50024 49932 50116 80004 80004 0.305191040039 0.305191040039 1 +decompress ctw expert ctw decompress:ctw 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 15.85 0.0282842712475 15.85 15.83 15.87 15.785 0.045 0.0630915831051 0.0630915831051 122890 76.3675323681 122890 122836 122944 301713 301713 0.287735939026 0.287735939026 1 +decompress ctw expert ctw decompress:ctw 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 33.65 0.0141421356237 33.65 33.64 33.66 33.55 0.07 0.0594353692906 0.0594353692906 200058 121.622366364 200058 199972 200144 597003 597003 0.284673213959 0.284673213959 1 +decompress ctw expert ctw decompress:ctw 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 70.8 0.254558441227 70.8 70.62 70.98 70.57 0.165 0.0564975403221 0.0564975403221 327964 50.9116882454 327964 327928 328000 1177962 1177962 0.280848026276 0.280848026276 1 +decompress ctw expert ctw decompress:ctw 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 179.275 0.869741340859 179.275 178.66 179.89 178.775 0.335 0.0531967948452 0.0531967948452 611890 115.965512115 611890 611808 611972 2746861 2746861 0.2746861 0.2746861 1 +decompress match expert match decompress:match 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.005 0.00707106781187 0.005 0 0.01 0 0 5878 166.87720036 5878 5760 5996 2874 2874 0.70166015625 0.70166015625 1 +decompress match expert match decompress:match 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.01 0 0.01 0.01 0.01 0.01 0 1.5625 1.5625 5816 39.5979797464 5816 5788 5844 13506 13506 0.824340820312 0.824340820312 1 +decompress match expert match decompress:match 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.03 0 0.03 0.03 0.03 0.03 0 2.08333333333 2.08333333333 6942 82.0243866176 6942 6884 7000 53748 53748 0.820129394531 0.820129394531 1 +decompress match expert match decompress:match 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.14 0 0.14 0.14 0.14 0.13 0 1.78571428571 1.78571428571 8322 2.82842712475 8322 8320 8324 206203 206203 0.786602020264 0.786602020264 1 +decompress match expert match decompress:match 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.55 0 0.55 0.55 0.55 0.55 0 1.81818181818 1.81818181818 11840 169.705627485 11840 11720 11960 817505 817505 0.779633522034 0.779633522034 1 +decompress match expert match decompress:match 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1.105 0.00707106781187 1.105 1.1 1.11 1.095 0 1.80999180999 1.80999180999 20456 107.48023074 20456 20380 20532 1644756 1644756 0.784280776978 0.784280776978 1 +decompress match expert match decompress:match 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.21 0 2.21 2.21 2.21 2.205 0 1.80995475113 1.80995475113 24178 53.7401153702 24178 24140 24216 3292751 3292751 0.785053014755 0.785053014755 1 +decompress match expert match decompress:match 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 5.335 0.0353553390593 5.335 5.31 5.36 5.305 0.02 1.78761997851 1.78761997851 46502 8.48528137424 46502 46496 46508 7859324 7859324 0.7859324 0.7859324 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.13 0 0.13 0.13 0.13 0.13 0 0.0300480769231 0.0300480769231 12504 84.8528137424 12504 12444 12564 1001 1001 0.244384765625 0.244384765625 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.57 0 0.57 0.57 0.57 0.56 0.005 0.0274122807018 0.0274122807018 30192 5.65685424949 30192 30188 30196 5533 5533 0.337707519531 0.337707519531 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.485 0.00707106781187 2.485 2.48 2.49 2.46 0.015 0.0251510072548 0.0251510072548 81624 22.627416998 81624 81608 81640 19344 19344 0.295166015625 0.295166015625 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 10.895 0.0353553390593 10.895 10.87 10.92 10.805 0.075 0.0229464264653 0.0229464264653 257756 39.5979797464 257756 257728 257784 67454 67454 0.257316589355 0.257316589355 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 49.025 0.0353553390593 49.025 49 49.05 48.615 0.36 0.0203977615511 0.0203977615511 694370 132.936074863 694370 694276 694464 254251 254251 0.242472648621 0.242472648621 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 104.21 0.0141421356237 104.21 104.2 104.22 103.61 0.49 0.019192016298 0.019192016298 1022092 3388.45569545 1022092 1019696 1024488 503520 503520 0.240097045898 0.240097045898 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 221.53 0.0424264068712 221.53 221.5 221.56 220.735 0.575 0.0180562455349 0.0180562455349 1458974 5150.56579416 1458974 1455332 1462616 988355 988355 0.235642194748 0.235642194748 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 574.63 0.296984848098 574.63 574.42 574.84 572.72 1.305 0.0165963218728 0.0165963218728 2659046 5031.77185492 2659046 2655488 2662604 2270248 2270248 0.2270248 0.2270248 1 +decompress ppmd expert ppmd decompress:ppmd 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.01 0 0.01 0.01 0.01 0.01 0 0.390625 0.390625 7680 181.019335984 7680 7552 7808 1058 1058 0.25830078125 0.25830078125 1 +decompress ppmd expert ppmd decompress:ppmd 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.04 0 0.04 0.04 0.04 0.035 0 0.390625 0.390625 16516 79.1959594929 16516 16460 16572 6267 6267 0.382507324219 0.382507324219 1 +decompress ppmd expert ppmd decompress:ppmd 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.215 0.00707106781187 0.215 0.21 0.22 0.19 0.015 0.290854978355 0.290854978355 45442 48.0832611207 45442 45408 45476 23186 23186 0.353790283203 0.353790283203 1 +decompress ppmd expert ppmd decompress:ppmd 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1.03 0.0141421356237 1.03 1.02 1.04 0.97 0.06 0.2427413273 0.2427413273 144534 36.7695526217 144534 144508 144560 83269 83269 0.317646026611 0.317646026611 1 +decompress ppmd expert ppmd decompress:ppmd 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 4.255 0.00707106781187 4.255 4.25 4.26 4.075 0.17 0.235017950842 0.235017950842 399646 110.308657865 399646 399568 399724 326331 326331 0.311213493347 0.311213493347 1 +decompress ppmd expert ppmd decompress:ppmd 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 8.245 0.00707106781187 8.245 8.24 8.25 8.025 0.205 0.242571344513 0.242571344513 457978 127.279220614 457978 457888 458068 661953 661953 0.315643787384 0.315643787384 1 +decompress ppmd expert ppmd decompress:ppmd 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 16.42 0.11313708499 16.42 16.34 16.5 16.19 0.205 0.24361114202 0.24361114202 458518 2.82842712475 458518 458516 458520 1329257 1329257 0.316919565201 0.316919565201 1 +decompress ppmd expert ppmd decompress:ppmd 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 39.725 0.0212132034356 39.725 39.71 39.74 39.405 0.28 0.240069088076 0.240069088076 559442 70.7106781187 559442 559392 559492 3154465 3154465 0.3154465 0.3154465 1 +decompress rosa expert rosaplus decompress:rosa 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0 0 0 0 0 0 0 6184 209.303607231 6184 6036 6332 1127 1127 0.275146484375 0.275146484375 1 +decompress rosa expert rosaplus decompress:rosa 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.02 0 0.02 0.02 0.02 0.02 0 0.78125 0.78125 8624 22.627416998 8624 8608 8640 6359 6359 0.388122558594 0.388122558594 1 +decompress rosa expert rosaplus decompress:rosa 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.12 0 0.12 0.12 0.12 0.12 0 0.520833333333 0.520833333333 18072 67.8822509939 18072 18024 18120 22843 22843 0.348556518555 0.348556518555 1 +decompress rosa expert rosaplus decompress:rosa 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.71 0 0.71 0.71 0.71 0.695 0.01 0.352112676056 0.352112676056 56056 169.705627485 56056 55936 56176 80590 80590 0.307426452637 0.307426452637 1 +decompress rosa expert rosaplus decompress:rosa 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3.91 0.0282842712475 3.91 3.89 3.93 3.86 0.04 0.255761167475 0.255761167475 146908 39.5979797464 146908 146880 146936 306522 306522 0.292322158813 0.292322158813 1 +decompress rosa expert rosaplus decompress:rosa 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 9.465 0.0777817459305 9.465 9.41 9.52 9.335 0.12 0.211311942418 0.211311942418 285750 93.3380951166 285750 285684 285816 608681 608681 0.290241718292 0.290241718292 1 +decompress rosa expert rosaplus decompress:rosa 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 23.52 0.0848528137424 23.52 23.46 23.58 23.25 0.24 0.170069133971 0.170069133971 553526 76.3675323681 553526 553472 553580 1199345 1199345 0.285946130753 0.285946130753 1 +decompress rosa expert rosaplus decompress:rosa 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 74.38 0.593969696197 74.38 73.96 74.8 73.67 0.635 0.128220586887 0.128220586887 1304556 84.8528137424 1304556 1304496 1304616 2752778 2752778 0.2752778 0.2752778 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.06 0 0.06 0.06 0.06 0.06 0 0.0651041666667 0.0651041666667 8086 36.7695526217 8086 8060 8112 3714 3714 0.90673828125 0.90673828125 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.24 0 0.24 0.24 0.24 0.24 0 0.0651041666667 0.0651041666667 8034 8.48528137424 8034 8028 8040 12000 12000 0.732421875 0.732421875 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.955 0.00707106781187 0.955 0.95 0.96 0.945 0 0.0654468201754 0.0654468201754 8144 84.8528137424 8144 8084 8204 35340 35340 0.539245605469 0.539245605469 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3.855 0.0494974746831 3.855 3.82 3.89 3.85 0 0.0648561891816 0.0648561891816 8254 121.622366364 8254 8168 8340 137010 137010 0.522651672363 0.522651672363 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 15.78 0.0282842712475 15.78 15.76 15.8 15.76 0 0.0633714579451 0.0633714579451 9276 90.5096679919 9276 9212 9340 474275 474275 0.452303886414 0.452303886414 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 31.755 0.0212132034356 31.755 31.74 31.77 31.73 0 0.0629822215796 0.0629822215796 10756 5.65685424949 10756 10752 10760 917086 917086 0.437300682068 0.437300682068 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 64.395 0.4171930009 64.395 64.1 64.69 64.35 0 0.0621179275985 0.0621179275985 13526 59.3969696197 13526 13484 13568 1748887 1748887 0.416967153549 0.416967153549 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 153.15 0.749533188058 153.15 152.62 153.68 153.04 0 0.0622713508262 0.0622713508262 21510 36.7695526217 21510 21484 21536 3968645 3968645 0.3968645 0.3968645 1 diff --git a/benchmarks/8d3701d/infotheory-extra-summary-20260315-154128.tsv b/benchmarks/8d3701d/infotheory-extra-summary-20260315-154128.tsv index b9b8576a..9a5f8dc2 100644 --- a/benchmarks/8d3701d/infotheory-extra-summary-20260315-154128.tsv +++ b/benchmarks/8d3701d/infotheory-extra-summary-20260315-154128.tsv @@ -1,61 +1,61 @@ -operation subject subject_kind expert_kind series size_bytes repeats cpu compression_backend input_sha256 real_seconds_mean real_seconds_stdev real_seconds_median real_seconds_min real_seconds_max user_seconds_mean sys_seconds_mean throughput_mib_s_mean throughput_mib_s_median rss_kib_mean rss_kib_stdev rss_kib_median rss_kib_min rss_kib_max archive_bytes_mean archive_bytes_median archive_ratio_mean archive_ratio_median entropy_bpb_mean entropy_bpb_median verified_all -h mamba expert mamba h:mamba 4096 1 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.13 0 0.13 0.13 0.13 0.13 0 0.0300480769231 0.0300480769231 8072 0 8072 8072 8072 5.68299885572 5.68299885572 1 -h mamba expert mamba h:mamba 16384 1 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.54 0 0.54 0.54 0.54 0.54 0 0.0289351851852 0.0289351851852 8264 0 8264 8264 8264 5.6590588313 5.6590588313 1 -h mamba expert mamba h:mamba 65536 1 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 2.13 0 2.13 2.13 2.13 2.13 0 0.0293427230047 0.0293427230047 8184 0 8184 8184 8184 4.0634890812 4.0634890812 1 -h mamba expert mamba h:mamba 262144 1 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 8.49 0 8.49 8.49 8.49 8.48 0 0.0294464075383 0.0294464075383 8328 0 8328 8328 8328 4.02980188788 4.02980188788 1 -h mamba expert mamba h:mamba 1048576 1 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 34.52 0 34.52 34.52 34.52 34.49 0 0.0289687137891 0.0289687137891 8896 0 8896 8896 8896 3.89348085266 3.89348085266 1 -h neural_mixture mixture neural-mixture h:neural_mixture 4096 1 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.17 0 0.17 0.17 0.17 0.17 0 0.0229779411765 0.0229779411765 7264 0 7264 7264 7264 5.80859336517 5.80859336517 1 -h neural_mixture mixture neural-mixture h:neural_mixture 16384 1 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.69 0 0.69 0.69 0.69 0.69 0 0.0226449275362 0.0226449275362 7252 0 7252 7252 7252 5.29204448749 5.29204448749 1 -h neural_mixture mixture neural-mixture h:neural_mixture 65536 1 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 2.71 0 2.71 2.71 2.71 2.7 0 0.0230627306273 0.0230627306273 8992 0 8992 8992 8992 4.99650919052 4.99650919052 1 -h neural_mixture mixture neural-mixture h:neural_mixture 262144 1 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 10.76 0 10.76 10.76 10.76 10.74 0 0.0232342007435 0.0232342007435 11708 0 11708 11708 11708 4.88773353096 4.88773353096 1 -h neural_mixture mixture neural-mixture h:neural_mixture 1048576 1 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 43.19 0 43.19 43.19 43.19 43.16 0 0.0231535077564 0.0231535077564 17956 0 17956 17956 17956 4.86654569805 4.86654569805 1 -h particle expert particle h:particle 4096 1 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.08 0 0.08 0.08 0.08 0.08 0 0.048828125 0.048828125 5480 0 5480 5480 5480 5.70387590178 5.70387590178 1 -h particle expert particle h:particle 16384 1 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.35 0 0.35 0.35 0.35 0.35 0 0.0446428571429 0.0446428571429 5548 0 5548 5548 5548 5.21845840387 5.21845840387 1 -h particle expert particle h:particle 65536 1 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 1.39 0 1.39 1.39 1.39 1.39 0 0.044964028777 0.044964028777 5552 0 5552 5552 5552 4.98633070488 4.98633070488 1 -h particle expert particle h:particle 262144 1 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 5.83 0 5.83 5.83 5.83 5.82 0 0.0428816466552 0.0428816466552 5456 0 5456 5456 5456 4.90839121162 4.90839121162 1 -h particle expert particle h:particle 1048576 1 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 23.29 0 23.29 23.29 23.29 23.28 0 0.0429368827823 0.0429368827823 6168 0 6168 6168 6168 4.90130162409 4.90130162409 1 -h sparse-match expert sparse-match h:sparse-match 4096 1 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0 0 0 0 0 0 0 5620 0 5620 5620 5620 7.7675225889 7.7675225889 1 -h sparse-match expert sparse-match h:sparse-match 16384 1 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0 0 0 0 0 0 0 5888 0 5888 5888 5888 7.9458951059 7.9458951059 1 -h sparse-match expert sparse-match h:sparse-match 65536 1 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.01 0 0.01 0.01 0.01 0.01 0 6.25 6.25 7144 0 7144 7144 7144 7.97635753927 7.97635753927 1 -h sparse-match expert sparse-match h:sparse-match 262144 1 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 0.04 0 0.04 0.04 0.04 0.03 0 6.25 6.25 9092 0 9092 9092 9092 7.99670337126 7.99670337126 1 -h sparse-match expert sparse-match h:sparse-match 1048576 1 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 0.16 0 0.16 0.16 0.16 0.15 0 6.25 6.25 14220 0 14220 14220 14220 8.00150959613 8.00150959613 1 -compress mamba expert mamba compress:mamba 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.14 0 0.14 0.14 0.14 0.13 0 0.0279017857143 0.0279017857143 7568 0 7568 7568 7568 2928 2928 0.71484375 0.71484375 1 -compress mamba expert mamba compress:mamba 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.54 0 0.54 0.54 0.54 0.53 0 0.0289351851852 0.0289351851852 7460 0 7460 7460 7460 11608 11608 0.70849609375 0.70849609375 1 -compress mamba expert mamba compress:mamba 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 2.16 0 2.16 2.16 2.16 2.16 0 0.0289351851852 0.0289351851852 7788 0 7788 7788 7788 33307 33307 0.508224487305 0.508224487305 1 -compress mamba expert mamba compress:mamba 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 8.62 0 8.62 8.62 8.62 8.61 0 0.0290023201856 0.0290023201856 7588 0 7588 7588 7588 132067 132067 0.503795623779 0.503795623779 1 -compress mamba expert mamba compress:mamba 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 35.13 0 35.13 35.13 35.13 35.11 0 0.0284656988329 0.0284656988329 9316 0 9316 9316 9316 510342 510342 0.486700057983 0.486700057983 1 -compress neural_mixture mixture neural-mixture compress:neural_mixture 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.23 0 0.23 0.23 0.23 0.23 0 0.0169836956522 0.0169836956522 7896 0 7896 7896 7896 2789 2789 0.680908203125 0.680908203125 1 -compress neural_mixture mixture neural-mixture compress:neural_mixture 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.94 0 0.94 0.94 0.94 0.94 0 0.0166223404255 0.0166223404255 8184 0 8184 8184 8184 10186 10186 0.621704101562 0.621704101562 1 -compress neural_mixture mixture neural-mixture compress:neural_mixture 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 3.74 0 3.74 3.74 3.74 3.72 0.01 0.0167112299465 0.0167112299465 10128 0 10128 10128 10128 32883 32883 0.501754760742 0.501754760742 1 -compress neural_mixture mixture neural-mixture compress:neural_mixture 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 14.85 0 14.85 14.85 14.85 14.8 0.03 0.016835016835 0.016835016835 12356 0 12356 12356 12356 122761 122761 0.468296051025 0.468296051025 1 -compress neural_mixture mixture neural-mixture compress:neural_mixture 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 59.74 0 59.74 59.74 59.74 59.68 0.01 0.0167392032139 0.0167392032139 19572 0 19572 19572 19572 469008 469008 0.447280883789 0.447280883789 1 -compress particle expert particle compress:particle 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.09 0 0.09 0.09 0.09 0.08 0 0.0434027777778 0.0434027777778 5624 0 5624 5624 5624 2939 2939 0.717529296875 0.717529296875 1 -compress particle expert particle compress:particle 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.36 0 0.36 0.36 0.36 0.36 0 0.0434027777778 0.0434027777778 5628 0 5628 5628 5628 10706 10706 0.653442382812 0.653442382812 1 -compress particle expert particle compress:particle 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 1.42 0 1.42 1.42 1.42 1.42 0 0.044014084507 0.044014084507 5680 0 5680 5680 5680 40867 40867 0.623580932617 0.623580932617 1 -compress particle expert particle compress:particle 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 5.64 0 5.64 5.64 5.64 5.63 0 0.0443262411348 0.0443262411348 5992 0 5992 5992 5992 160857 160857 0.613620758057 0.613620758057 1 -compress particle expert particle compress:particle 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 22.54 0 22.54 22.54 22.54 22.53 0 0.0443655723159 0.0443655723159 7896 0 7896 7896 7896 642442 642442 0.612680435181 0.612680435181 1 -compress sparse-match expert sparse-match compress:sparse-match 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0 0 0 0 0 0 0 5576 0 5576 5576 5576 3996 3996 0.9755859375 0.9755859375 1 -compress sparse-match expert sparse-match compress:sparse-match 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.01 0 0.01 0.01 0.01 0.01 0 1.5625 1.5625 5548 0 5548 5548 5548 16292 16292 0.994384765625 0.994384765625 1 -compress sparse-match expert sparse-match compress:sparse-match 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.03 0 0.03 0.03 0.03 0.03 0 2.08333333333 2.08333333333 6908 0 6908 6908 6908 65361 65361 0.997329711914 0.997329711914 1 -compress sparse-match expert sparse-match compress:sparse-match 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 0.14 0 0.14 0.14 0.14 0.14 0 1.78571428571 1.78571428571 9284 0 9284 9284 9284 262055 262055 0.999660491943 0.999660491943 1 -compress sparse-match expert sparse-match compress:sparse-match 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 0.58 0 0.58 0.58 0.58 0.57 0 1.72413793103 1.72413793103 14724 0 14724 14724 14724 1048792 1048792 1.00020599365 1.00020599365 1 -decompress mamba expert mamba decompress:mamba 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.13 0 0.13 0.13 0.13 0.13 0 0.0300480769231 0.0300480769231 7564 0 7564 7564 7564 2928 2928 0.71484375 0.71484375 1 -decompress mamba expert mamba decompress:mamba 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.55 0 0.55 0.55 0.55 0.54 0 0.0284090909091 0.0284090909091 7492 0 7492 7492 7492 11608 11608 0.70849609375 0.70849609375 1 -decompress mamba expert mamba decompress:mamba 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 2.17 0 2.17 2.17 2.17 2.17 0 0.028801843318 0.028801843318 7696 0 7696 7696 7696 33307 33307 0.508224487305 0.508224487305 1 -decompress mamba expert mamba decompress:mamba 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 8.61 0 8.61 8.61 8.61 8.6 0 0.0290360046458 0.0290360046458 7740 0 7740 7740 7740 132067 132067 0.503795623779 0.503795623779 1 -decompress mamba expert mamba decompress:mamba 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 35.1 0 35.1 35.1 35.1 35.07 0 0.02849002849 0.02849002849 8904 0 8904 8904 8904 510342 510342 0.486700057983 0.486700057983 1 -decompress neural_mixture mixture neural-mixture decompress:neural_mixture 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.23 0 0.23 0.23 0.23 0.23 0 0.0169836956522 0.0169836956522 8056 0 8056 8056 8056 2789 2789 0.680908203125 0.680908203125 1 -decompress neural_mixture mixture neural-mixture decompress:neural_mixture 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.95 0 0.95 0.95 0.95 0.95 0 0.0164473684211 0.0164473684211 8200 0 8200 8200 8200 10186 10186 0.621704101562 0.621704101562 1 -decompress neural_mixture mixture neural-mixture decompress:neural_mixture 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 3.72 0 3.72 3.72 3.72 3.71 0 0.0168010752688 0.0168010752688 10372 0 10372 10372 10372 32883 32883 0.501754760742 0.501754760742 1 -decompress neural_mixture mixture neural-mixture decompress:neural_mixture 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 15.01 0 15.01 15.01 15.01 14.99 0 0.016655562958 0.016655562958 12300 0 12300 12300 12300 122761 122761 0.468296051025 0.468296051025 1 -decompress neural_mixture mixture neural-mixture decompress:neural_mixture 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 59.88 0 59.88 59.88 59.88 59.82 0.01 0.0167000668003 0.0167000668003 19660 0 19660 19660 19660 469008 469008 0.447280883789 0.447280883789 1 -decompress particle expert particle decompress:particle 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.09 0 0.09 0.09 0.09 0.09 0 0.0434027777778 0.0434027777778 5484 0 5484 5484 5484 2939 2939 0.717529296875 0.717529296875 1 -decompress particle expert particle decompress:particle 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.35 0 0.35 0.35 0.35 0.35 0 0.0446428571429 0.0446428571429 5536 0 5536 5536 5536 10706 10706 0.653442382812 0.653442382812 1 -decompress particle expert particle decompress:particle 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 1.41 0 1.41 1.41 1.41 1.41 0 0.0443262411348 0.0443262411348 5756 0 5756 5756 5756 40867 40867 0.623580932617 0.623580932617 1 -decompress particle expert particle decompress:particle 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 5.66 0 5.66 5.66 5.66 5.65 0 0.0441696113074 0.0441696113074 5804 0 5804 5804 5804 160857 160857 0.613620758057 0.613620758057 1 -decompress particle expert particle decompress:particle 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 23 0 23 23 23 22.98 0 0.0434782608696 0.0434782608696 6836 0 6836 6836 6836 642442 642442 0.612680435181 0.612680435181 1 -decompress sparse-match expert sparse-match decompress:sparse-match 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0 0 0 0 0 0 0 5380 0 5380 5380 5380 3996 3996 0.9755859375 0.9755859375 1 -decompress sparse-match expert sparse-match decompress:sparse-match 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.01 0 0.01 0.01 0.01 0.01 0 1.5625 1.5625 5656 0 5656 5656 5656 16292 16292 0.994384765625 0.994384765625 1 -decompress sparse-match expert sparse-match decompress:sparse-match 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.04 0 0.04 0.04 0.04 0.04 0 1.5625 1.5625 6832 0 6832 6832 6832 65361 65361 0.997329711914 0.997329711914 1 -decompress sparse-match expert sparse-match decompress:sparse-match 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 0.15 0 0.15 0.15 0.15 0.15 0 1.66666666667 1.66666666667 8832 0 8832 8832 8832 262055 262055 0.999660491943 0.999660491943 1 -decompress sparse-match expert sparse-match decompress:sparse-match 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 0.6 0 0.6 0.6 0.6 0.59 0 1.66666666667 1.66666666667 14868 0 14868 14868 14868 1048792 1048792 1.00020599365 1.00020599365 1 +operation subject subject_kind expert_kind series size_bytes repeats cpu compression_backend input_sha256 real_seconds_mean real_seconds_stdev real_seconds_median real_seconds_min real_seconds_max user_seconds_mean sys_seconds_mean throughput_mib_s_mean throughput_mib_s_median rss_kib_mean rss_kib_stdev rss_kib_median rss_kib_min rss_kib_max archive_bytes_mean archive_bytes_median archive_ratio_mean archive_ratio_median entropy_bpb_mean entropy_bpb_median verified_all +h mamba expert mamba h:mamba 4096 1 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.13 0 0.13 0.13 0.13 0.13 0 0.0300480769231 0.0300480769231 8072 0 8072 8072 8072 5.68299885572 5.68299885572 1 +h mamba expert mamba h:mamba 16384 1 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.54 0 0.54 0.54 0.54 0.54 0 0.0289351851852 0.0289351851852 8264 0 8264 8264 8264 5.6590588313 5.6590588313 1 +h mamba expert mamba h:mamba 65536 1 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 2.13 0 2.13 2.13 2.13 2.13 0 0.0293427230047 0.0293427230047 8184 0 8184 8184 8184 4.0634890812 4.0634890812 1 +h mamba expert mamba h:mamba 262144 1 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 8.49 0 8.49 8.49 8.49 8.48 0 0.0294464075383 0.0294464075383 8328 0 8328 8328 8328 4.02980188788 4.02980188788 1 +h mamba expert mamba h:mamba 1048576 1 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 34.52 0 34.52 34.52 34.52 34.49 0 0.0289687137891 0.0289687137891 8896 0 8896 8896 8896 3.89348085266 3.89348085266 1 +h neural_mixture mixture neural-mixture h:neural_mixture 4096 1 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.17 0 0.17 0.17 0.17 0.17 0 0.0229779411765 0.0229779411765 7264 0 7264 7264 7264 5.80859336517 5.80859336517 1 +h neural_mixture mixture neural-mixture h:neural_mixture 16384 1 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.69 0 0.69 0.69 0.69 0.69 0 0.0226449275362 0.0226449275362 7252 0 7252 7252 7252 5.29204448749 5.29204448749 1 +h neural_mixture mixture neural-mixture h:neural_mixture 65536 1 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 2.71 0 2.71 2.71 2.71 2.7 0 0.0230627306273 0.0230627306273 8992 0 8992 8992 8992 4.99650919052 4.99650919052 1 +h neural_mixture mixture neural-mixture h:neural_mixture 262144 1 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 10.76 0 10.76 10.76 10.76 10.74 0 0.0232342007435 0.0232342007435 11708 0 11708 11708 11708 4.88773353096 4.88773353096 1 +h neural_mixture mixture neural-mixture h:neural_mixture 1048576 1 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 43.19 0 43.19 43.19 43.19 43.16 0 0.0231535077564 0.0231535077564 17956 0 17956 17956 17956 4.86654569805 4.86654569805 1 +h particle expert particle h:particle 4096 1 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.08 0 0.08 0.08 0.08 0.08 0 0.048828125 0.048828125 5480 0 5480 5480 5480 5.70387590178 5.70387590178 1 +h particle expert particle h:particle 16384 1 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.35 0 0.35 0.35 0.35 0.35 0 0.0446428571429 0.0446428571429 5548 0 5548 5548 5548 5.21845840387 5.21845840387 1 +h particle expert particle h:particle 65536 1 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 1.39 0 1.39 1.39 1.39 1.39 0 0.044964028777 0.044964028777 5552 0 5552 5552 5552 4.98633070488 4.98633070488 1 +h particle expert particle h:particle 262144 1 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 5.83 0 5.83 5.83 5.83 5.82 0 0.0428816466552 0.0428816466552 5456 0 5456 5456 5456 4.90839121162 4.90839121162 1 +h particle expert particle h:particle 1048576 1 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 23.29 0 23.29 23.29 23.29 23.28 0 0.0429368827823 0.0429368827823 6168 0 6168 6168 6168 4.90130162409 4.90130162409 1 +h sparse-match expert sparse-match h:sparse-match 4096 1 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0 0 0 0 0 0 0 5620 0 5620 5620 5620 7.7675225889 7.7675225889 1 +h sparse-match expert sparse-match h:sparse-match 16384 1 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0 0 0 0 0 0 0 5888 0 5888 5888 5888 7.9458951059 7.9458951059 1 +h sparse-match expert sparse-match h:sparse-match 65536 1 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.01 0 0.01 0.01 0.01 0.01 0 6.25 6.25 7144 0 7144 7144 7144 7.97635753927 7.97635753927 1 +h sparse-match expert sparse-match h:sparse-match 262144 1 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 0.04 0 0.04 0.04 0.04 0.03 0 6.25 6.25 9092 0 9092 9092 9092 7.99670337126 7.99670337126 1 +h sparse-match expert sparse-match h:sparse-match 1048576 1 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 0.16 0 0.16 0.16 0.16 0.15 0 6.25 6.25 14220 0 14220 14220 14220 8.00150959613 8.00150959613 1 +compress mamba expert mamba compress:mamba 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.14 0 0.14 0.14 0.14 0.13 0 0.0279017857143 0.0279017857143 7568 0 7568 7568 7568 2928 2928 0.71484375 0.71484375 1 +compress mamba expert mamba compress:mamba 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.54 0 0.54 0.54 0.54 0.53 0 0.0289351851852 0.0289351851852 7460 0 7460 7460 7460 11608 11608 0.70849609375 0.70849609375 1 +compress mamba expert mamba compress:mamba 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 2.16 0 2.16 2.16 2.16 2.16 0 0.0289351851852 0.0289351851852 7788 0 7788 7788 7788 33307 33307 0.508224487305 0.508224487305 1 +compress mamba expert mamba compress:mamba 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 8.62 0 8.62 8.62 8.62 8.61 0 0.0290023201856 0.0290023201856 7588 0 7588 7588 7588 132067 132067 0.503795623779 0.503795623779 1 +compress mamba expert mamba compress:mamba 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 35.13 0 35.13 35.13 35.13 35.11 0 0.0284656988329 0.0284656988329 9316 0 9316 9316 9316 510342 510342 0.486700057983 0.486700057983 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.23 0 0.23 0.23 0.23 0.23 0 0.0169836956522 0.0169836956522 7896 0 7896 7896 7896 2789 2789 0.680908203125 0.680908203125 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.94 0 0.94 0.94 0.94 0.94 0 0.0166223404255 0.0166223404255 8184 0 8184 8184 8184 10186 10186 0.621704101562 0.621704101562 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 3.74 0 3.74 3.74 3.74 3.72 0.01 0.0167112299465 0.0167112299465 10128 0 10128 10128 10128 32883 32883 0.501754760742 0.501754760742 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 14.85 0 14.85 14.85 14.85 14.8 0.03 0.016835016835 0.016835016835 12356 0 12356 12356 12356 122761 122761 0.468296051025 0.468296051025 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 59.74 0 59.74 59.74 59.74 59.68 0.01 0.0167392032139 0.0167392032139 19572 0 19572 19572 19572 469008 469008 0.447280883789 0.447280883789 1 +compress particle expert particle compress:particle 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.09 0 0.09 0.09 0.09 0.08 0 0.0434027777778 0.0434027777778 5624 0 5624 5624 5624 2939 2939 0.717529296875 0.717529296875 1 +compress particle expert particle compress:particle 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.36 0 0.36 0.36 0.36 0.36 0 0.0434027777778 0.0434027777778 5628 0 5628 5628 5628 10706 10706 0.653442382812 0.653442382812 1 +compress particle expert particle compress:particle 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 1.42 0 1.42 1.42 1.42 1.42 0 0.044014084507 0.044014084507 5680 0 5680 5680 5680 40867 40867 0.623580932617 0.623580932617 1 +compress particle expert particle compress:particle 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 5.64 0 5.64 5.64 5.64 5.63 0 0.0443262411348 0.0443262411348 5992 0 5992 5992 5992 160857 160857 0.613620758057 0.613620758057 1 +compress particle expert particle compress:particle 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 22.54 0 22.54 22.54 22.54 22.53 0 0.0443655723159 0.0443655723159 7896 0 7896 7896 7896 642442 642442 0.612680435181 0.612680435181 1 +compress sparse-match expert sparse-match compress:sparse-match 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0 0 0 0 0 0 0 5576 0 5576 5576 5576 3996 3996 0.9755859375 0.9755859375 1 +compress sparse-match expert sparse-match compress:sparse-match 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.01 0 0.01 0.01 0.01 0.01 0 1.5625 1.5625 5548 0 5548 5548 5548 16292 16292 0.994384765625 0.994384765625 1 +compress sparse-match expert sparse-match compress:sparse-match 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.03 0 0.03 0.03 0.03 0.03 0 2.08333333333 2.08333333333 6908 0 6908 6908 6908 65361 65361 0.997329711914 0.997329711914 1 +compress sparse-match expert sparse-match compress:sparse-match 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 0.14 0 0.14 0.14 0.14 0.14 0 1.78571428571 1.78571428571 9284 0 9284 9284 9284 262055 262055 0.999660491943 0.999660491943 1 +compress sparse-match expert sparse-match compress:sparse-match 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 0.58 0 0.58 0.58 0.58 0.57 0 1.72413793103 1.72413793103 14724 0 14724 14724 14724 1048792 1048792 1.00020599365 1.00020599365 1 +decompress mamba expert mamba decompress:mamba 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.13 0 0.13 0.13 0.13 0.13 0 0.0300480769231 0.0300480769231 7564 0 7564 7564 7564 2928 2928 0.71484375 0.71484375 1 +decompress mamba expert mamba decompress:mamba 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.55 0 0.55 0.55 0.55 0.54 0 0.0284090909091 0.0284090909091 7492 0 7492 7492 7492 11608 11608 0.70849609375 0.70849609375 1 +decompress mamba expert mamba decompress:mamba 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 2.17 0 2.17 2.17 2.17 2.17 0 0.028801843318 0.028801843318 7696 0 7696 7696 7696 33307 33307 0.508224487305 0.508224487305 1 +decompress mamba expert mamba decompress:mamba 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 8.61 0 8.61 8.61 8.61 8.6 0 0.0290360046458 0.0290360046458 7740 0 7740 7740 7740 132067 132067 0.503795623779 0.503795623779 1 +decompress mamba expert mamba decompress:mamba 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 35.1 0 35.1 35.1 35.1 35.07 0 0.02849002849 0.02849002849 8904 0 8904 8904 8904 510342 510342 0.486700057983 0.486700057983 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.23 0 0.23 0.23 0.23 0.23 0 0.0169836956522 0.0169836956522 8056 0 8056 8056 8056 2789 2789 0.680908203125 0.680908203125 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.95 0 0.95 0.95 0.95 0.95 0 0.0164473684211 0.0164473684211 8200 0 8200 8200 8200 10186 10186 0.621704101562 0.621704101562 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 3.72 0 3.72 3.72 3.72 3.71 0 0.0168010752688 0.0168010752688 10372 0 10372 10372 10372 32883 32883 0.501754760742 0.501754760742 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 15.01 0 15.01 15.01 15.01 14.99 0 0.016655562958 0.016655562958 12300 0 12300 12300 12300 122761 122761 0.468296051025 0.468296051025 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 59.88 0 59.88 59.88 59.88 59.82 0.01 0.0167000668003 0.0167000668003 19660 0 19660 19660 19660 469008 469008 0.447280883789 0.447280883789 1 +decompress particle expert particle decompress:particle 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.09 0 0.09 0.09 0.09 0.09 0 0.0434027777778 0.0434027777778 5484 0 5484 5484 5484 2939 2939 0.717529296875 0.717529296875 1 +decompress particle expert particle decompress:particle 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.35 0 0.35 0.35 0.35 0.35 0 0.0446428571429 0.0446428571429 5536 0 5536 5536 5536 10706 10706 0.653442382812 0.653442382812 1 +decompress particle expert particle decompress:particle 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 1.41 0 1.41 1.41 1.41 1.41 0 0.0443262411348 0.0443262411348 5756 0 5756 5756 5756 40867 40867 0.623580932617 0.623580932617 1 +decompress particle expert particle decompress:particle 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 5.66 0 5.66 5.66 5.66 5.65 0 0.0441696113074 0.0441696113074 5804 0 5804 5804 5804 160857 160857 0.613620758057 0.613620758057 1 +decompress particle expert particle decompress:particle 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 23 0 23 23 23 22.98 0 0.0434782608696 0.0434782608696 6836 0 6836 6836 6836 642442 642442 0.612680435181 0.612680435181 1 +decompress sparse-match expert sparse-match decompress:sparse-match 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0 0 0 0 0 0 0 5380 0 5380 5380 5380 3996 3996 0.9755859375 0.9755859375 1 +decompress sparse-match expert sparse-match decompress:sparse-match 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.01 0 0.01 0.01 0.01 0.01 0 1.5625 1.5625 5656 0 5656 5656 5656 16292 16292 0.994384765625 0.994384765625 1 +decompress sparse-match expert sparse-match decompress:sparse-match 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.04 0 0.04 0.04 0.04 0.04 0 1.5625 1.5625 6832 0 6832 6832 6832 65361 65361 0.997329711914 0.997329711914 1 +decompress sparse-match expert sparse-match decompress:sparse-match 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 0.15 0 0.15 0.15 0.15 0.15 0 1.66666666667 1.66666666667 8832 0 8832 8832 8832 262055 262055 0.999660491943 0.999660491943 1 +decompress sparse-match expert sparse-match decompress:sparse-match 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 0.6 0 0.6 0.6 0.6 0.59 0 1.66666666667 1.66666666667 14868 0 14868 14868 14868 1048792 1048792 1.00020599365 1.00020599365 1 diff --git a/benchmarks/8d3701d/infotheory-two-json-summary-full.tsv b/benchmarks/8d3701d/infotheory-two-json-summary-full.tsv index 5408b993..35b82e9e 100644 --- a/benchmarks/8d3701d/infotheory-two-json-summary-full.tsv +++ b/benchmarks/8d3701d/infotheory-two-json-summary-full.tsv @@ -1,145 +1,145 @@ -operation subject subject_kind expert_kind series size_bytes repeats cpu compression_backend input_sha256 real_seconds_mean real_seconds_stdev real_seconds_median real_seconds_min real_seconds_max user_seconds_mean sys_seconds_mean throughput_mib_s_mean throughput_mib_s_median rss_kib_mean rss_kib_stdev rss_kib_median rss_kib_min rss_kib_max archive_bytes_mean archive_bytes_median archive_ratio_mean archive_ratio_median entropy_bpb_mean entropy_bpb_median verified_all -h ctw expert ctw h:ctw 4096 3 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.0366666666667 0.0057735026919 0.04 0.03 0.04 0.03 0 0.108506944444 0.09765625 11901.3333333 151.437555888 11968 11728 12008 2.56549624542 2.56549624542 1 -h ctw expert ctw h:ctw 16384 3 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.17 0 0.17 0.17 0.17 0.163333333333 0 0.0919117647059 0.0919117647059 34488 62.4819974073 34456 34448 34560 3.20340951209 3.20340951209 1 -h ctw expert ctw h:ctw 65536 3 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.75 0 0.75 0.75 0.75 0.72 0.0166666666667 0.0833333333333 0.0833333333333 76324 176.13631085 76392 76124 76456 2.77213895634 2.77213895634 1 -h ctw expert ctw h:ctw 262144 3 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 3.17333333333 0.0057735026919 3.17 3.17 3.18 3.09 0.0766666666667 0.0787816862753 0.0788643533123 185469.333333 133.16656237 185536 185316 185556 2.44096714152 2.44096714152 1 -h ctw expert ctw h:ctw 1048576 3 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 14.0166666667 0.0115470053838 14.01 14.01 14.03 13.84 0.156666666667 0.0713436707887 0.0713775874375 470858.666667 90.1849950565 470852 470772 470952 2.30174274755 2.30174274755 1 -h ctw expert ctw h:ctw 2097152 3 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 29.6 0.01 29.6 29.59 29.61 29.2633333333 0.303333333333 0.0675675727088 0.0675675675676 775094.666667 132.020200475 775092 774964 775228 2.27731545661 2.27731545661 1 -h ctw expert ctw h:ctw 4194304 3 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 62.8266666667 0.140118997047 62.87 62.67 62.94 62.2833333333 0.486666666667 0.0636674439169 0.0636233497694 1291712 26.2297540972 1291716 1291684 1291736 2.24674930722 2.24674930722 1 -h ctw expert ctw h:ctw 10000000 3 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 160.896666667 0.0472581562625 160.88 160.86 160.95 159.776666667 0.97 0.0592724753723 0.0592786124072 2487692 77.1492060879 2487660 2487636 2487780 2.19747398481 2.19747398481 1 -h match expert match h:match 4096 3 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0 0 0 0 0 0 0 5770.66666667 46.8757222167 5752 5736 5824 5.57734993252 5.57734993252 1 -h match expert match h:match 16384 3 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0 0 0 0 0 0 0 5732 157.124154731 5696 5596 5904 6.58586286407 6.58586286407 1 -h match expert match h:match 65536 3 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0 0 0 0 0 0 0 6746.66666667 31.0698138606 6756 6712 6772 6.5588255305 6.5588255305 1 -h match expert match h:match 262144 3 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 0.03 0 0.03 0.03 0.03 0.0233333333333 0 8.33333333333 8.33333333333 7885.33333333 169.81558625 7844 7740 8072 6.29225944669 6.29225944669 1 -h match expert match h:match 1048576 3 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 0.126666666667 0.0057735026919 0.13 0.12 0.13 0.126666666667 0 7.90598290598 7.69230769231 11401.3333333 98.1699207157 11408 11300 11496 6.23692635511 6.23692635511 1 -h match expert match h:match 2097152 3 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 0.263333333333 0.0152752523165 0.26 0.25 0.28 0.26 0 7.61172161172 7.69230769231 19294.6666667 37.1662929727 19284 19264 19336 6.2741772649 6.2741772649 1 -h match expert match h:match 4194304 3 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 0.523333333333 0.0057735026919 0.52 0.52 0.53 0.513333333333 0.01 7.64392839865 7.69230769231 21257.3333333 134.600643882 21272 21116 21384 6.28038800899 6.28038800899 1 -h match expert match h:match 10000000 3 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 1.29333333333 0.0152752523165 1.29 1.28 1.31 1.27333333333 0.01 7.37445378963 7.39282415819 40373.3333333 87.300248186 40356 40296 40468 6.28744436593 6.28744436593 1 -h neural_mixture mixture neural-mixture h:neural_mixture 4096 3 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.13 0 0.13 0.13 0.13 0.123333333333 0.00333333333333 0.0300480769231 0.0300480769231 17454.6666667 134.004975032 17528 17300 17536 1.91857366907 1.91857366907 1 -h neural_mixture mixture neural-mixture h:neural_mixture 16384 3 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.566666666667 0.0057735026919 0.57 0.56 0.57 0.543333333333 0.0166666666667 0.0275754490393 0.0274122807018 52590.6666667 16.6533279957 52596 52572 52604 2.69245779825 2.69245779825 1 -h neural_mixture mixture neural-mixture h:neural_mixture 65536 3 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 2.45666666667 0.0057735026919 2.46 2.45 2.46 2.41 0.04 0.0254410707372 0.025406504065 139388 122.572427568 139336 139300 139528 2.35902117116 2.35902117116 1 -h neural_mixture mixture neural-mixture h:neural_mixture 262144 3 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 10.66 0.0173205080757 10.65 10.65 10.68 10.5066666667 0.136666666667 0.023452198836 0.0234741784038 427940 40 427940 427900 427980 2.05795611601 2.05795611601 1 -h neural_mixture mixture neural-mixture h:neural_mixture 1048576 3 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 46.9333333333 0.0321455025366 46.92 46.91 46.97 46.37 0.513333333333 0.021306824843 0.0213128729753 1107021.33333 43.8786204584 1107036 1106972 1107056 1.93963669332 1.93963669332 1 -h neural_mixture mixture neural-mixture h:neural_mixture 2097152 3 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 98.5133333333 0.165630109984 98.53 98.34 98.67 97.64 0.776666666667 0.0203018586652 0.0202983862783 1717540 80.8949936646 1717572 1717448 1717600 1.92069854644 1.92069854644 1 -h neural_mixture mixture neural-mixture h:neural_mixture 4194304 3 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 206.973333333 0.222785397487 206.86 206.83 207.23 205.693333333 1.08 0.019326176099 0.0193367494924 2643548 100.955435713 2643520 2643464 2643660 1.88509670528 1.88509670528 1 -h neural_mixture mixture neural-mixture h:neural_mixture 10000000 3 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 528.096666667 0.439355588713 527.9 527.79 528.6 525.41 2.17666666667 0.0180587156948 0.0180654350522 4790344 124.771791684 4790412 4790200 4790420 1.81617547371 1.81617547371 1 -h ppmd expert ppmd h:ppmd 4096 3 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.01 0 0.01 0.01 0.01 0.00333333333333 0 0.390625 0.390625 7466.66666667 70.691819423 7432 7420 7548 2.02916276221 2.02916276221 1 -h ppmd expert ppmd h:ppmd 16384 3 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.0333333333333 0.0057735026919 0.03 0.03 0.04 0.0333333333333 0 0.477430555556 0.520833333333 16036 72.7736215946 15996 15992 16120 3.05111741489 3.05111741489 1 -h ppmd expert ppmd h:ppmd 65536 3 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.186666666667 0.0057735026919 0.19 0.18 0.19 0.163333333333 0.0166666666667 0.335038986355 0.328947368421 45244 235.796522451 45304 44984 45444 2.82800309907 2.82800309907 1 -h ppmd expert ppmd h:ppmd 262144 3 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 0.906666666667 0.0057735026919 0.91 0.9 0.91 0.833333333333 0.0666666666667 0.275742775743 0.274725274725 144282.666667 138.814024267 144300 144136 144412 2.54059561824 2.54059561824 1 -h ppmd expert ppmd h:ppmd 1048576 3 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 3.80666666667 0.0057735026919 3.81 3.8 3.81 3.60666666667 0.19 0.26269742598 0.262467191601 398937.333333 97.7616148257 398916 398852 399044 2.48956475398 2.48956475398 1 -h ppmd expert ppmd h:ppmd 2097152 3 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 7.45 0.0916515138991 7.43 7.37 7.55 7.21333333333 0.22 0.268483362304 0.269179004038 457117.333333 122.659420076 457180 456976 457196 2.52508057302 2.52508057302 1 -h ppmd expert ppmd h:ppmd 4194304 3 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 14.77 0.105356537529 14.76 14.67 14.88 14.53 0.22 0.270828405889 0.271002710027 459222.666667 180.945664036 459280 459020 459368 2.5353209835 2.5353209835 1 -h ppmd expert ppmd h:ppmd 10000000 3 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 35.5666666667 0.0723417813807 35.53 35.52 35.65 35.2733333333 0.253333333333 0.268137847329 0.268413823925 557416 187.488666324 557484 557204 557560 2.52355814767 2.52355814767 1 -h rosa expert rosaplus h:rosa 4096 3 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.02 0 0.02 0.02 0.02 0.0133333333333 0 0.1953125 0.1953125 5938.66666667 181.563579314 5980 5740 6096 2.11172124286 2.11172124286 1 -h rosa expert rosaplus h:rosa 16384 3 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.12 0 0.12 0.12 0.12 0.106666666667 0.00666666666667 0.130208333333 0.130208333333 8038.66666667 49.6923870762 8012 8008 8096 3.48630622986 3.48630622986 1 -h rosa expert rosaplus h:rosa 65536 3 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.596666666667 0.0057735026919 0.6 0.59 0.6 0.573333333333 0.0133333333333 0.104755178908 0.104166666667 16633.3333333 123.309907685 16592 16536 16772 3.19056696201 3.19056696201 1 -h rosa expert rosaplus h:rosa 262144 3 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 3.42 0 3.42 3.42 3.42 3.31 0.0966666666667 0.0730994152047 0.0730994152047 47050.6666667 138.120720145 47012 46936 47204 2.90567254487 2.90567254487 1 -h rosa expert rosaplus h:rosa 1048576 3 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 20.0533333333 0.0115470053838 20.06 20.04 20.06 19.2933333333 0.726666666667 0.049867032303 0.049850448654 203772 34.6410161514 203752 203752 203812 2.80123027825 2.80123027825 1 -h rosa expert rosaplus h:rosa 2097152 3 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 47.6166666667 0.0057735026919 47.62 47.61 47.62 45.67 1.89 0.0420021005167 0.0419991600168 337785.333333 76.1402215214 337780 337712 337864 2.70437138702 2.70437138702 1 -h rosa expert rosaplus h:rosa 4194304 3 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 113.323333333 0.0838649708361 113.28 113.27 113.42 108.21 4.98333333333 0.0352972449909 0.0353107344633 643504 68.3520299625 643496 643440 643576 2.62099230176 2.62099230176 1 -h rosa expert rosaplus h:rosa 10000000 3 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 330.516666667 0.598024525695 330.66 329.86 331.03 317.14 13.0266666667 0.028854109196 0.028841538632 1551453.33333 88.1211287566 1551496 1551352 1551512 2.48481897791 2.48481897791 1 -h rwkv expert rwkv h:rwkv 4096 3 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.0633333333333 0.0057735026919 0.06 0.06 0.07 0.06 0 0.062003968254 0.0651041666667 8172 76 8152 8108 8256 7.21701437947 7.21701437947 1 -h rwkv expert rwkv h:rwkv 16384 3 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.23 0 0.23 0.23 0.23 0.226666666667 0 0.0679347826087 0.0679347826087 8221.33333333 161.855902992 8136 8120 8408 5.85021072042 5.85021072042 1 -h rwkv expert rwkv h:rwkv 65536 3 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.903333333333 0.0057735026919 0.9 0.9 0.91 0.9 0 0.0691900691901 0.0694444444444 8308 58.9236794506 8276 8272 8376 4.31173851819 4.31173851819 1 -h rwkv expert rwkv h:rwkv 262144 3 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 3.65333333333 0.0057735026919 3.65 3.65 3.66 3.64666666667 0 0.0684307707663 0.0684931506849 8248 86.5332306111 8224 8176 8344 4.18065126521 4.18065126521 1 -h rwkv expert rwkv h:rwkv 1048576 3 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 15.0733333333 0.119303534454 15.11 14.94 15.17 15.06 0 0.0663451064205 0.066181336863 8833.33333333 91.2432645916 8816 8752 8932 3.61829048151 3.61829048151 1 -h rwkv expert rwkv h:rwkv 2097152 3 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 30.61 0.283548937575 30.51 30.39 30.93 30.5833333333 0 0.0653418467794 0.0655522779417 9874.66666667 127.080027279 9944 9728 9952 3.49833649966 3.49833649966 1 -h rwkv expert rwkv h:rwkv 4194304 3 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 61.61 0.308058436015 61.73 61.26 61.84 61.5666666667 0 0.0649256100831 0.0647983152438 11898.6666667 22.0302821891 11888 11884 11924 3.33570098711 3.33570098711 1 -h rwkv expert rwkv h:rwkv 10000000 3 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 149.02 0.46184412955 148.9 148.63 149.53 148.913333333 0 0.0639968068945 0.064047972895 17758.6666667 116.091917606 17784 17632 17860 3.17490278117 3.17490278117 1 -compress ctw expert ctw compress:ctw 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.05 0 0.05 0.05 0.05 0.04 0 0.078125 0.078125 11792 78.7908624144 11768 11728 11880 1332 1332 0.3251953125 0.3251953125 1 -compress ctw expert ctw compress:ctw 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.22 0 0.22 0.22 0.22 0.2 0.01 0.0710227272727 0.0710227272727 34390.6666667 98.33276836 34400 34288 34484 6579 6579 0.401550292969 0.401550292969 1 -compress ctw expert ctw compress:ctw 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.953333333333 0.0057735026919 0.95 0.95 0.96 0.92 0.0233333333333 0.0655610380117 0.0657894736842 77308 68.3520299625 77300 77244 77380 22728 22728 0.346801757812 0.346801757812 1 -compress ctw expert ctw compress:ctw 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 3.98 0 3.98 3.98 3.98 3.89333333333 0.08 0.0628140703518 0.0628140703518 185717.333333 66.2520439936 185724 185648 185780 80004 80004 0.305191040039 0.305191040039 1 -compress ctw expert ctw compress:ctw 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 17.0666666667 0.0057735026919 17.07 17.06 17.07 16.89 0.16 0.0585937544712 0.0585823081429 471393.333333 95.4428275636 471436 471284 471460 301713 301713 0.287735939026 0.287735939026 1 -compress ctw expert ctw compress:ctw 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 35.4266666667 0.0251661147842 35.43 35.4 35.45 35.1 0.293333333333 0.0564546670944 0.0564493367203 776241.333333 108.541850608 776304 776116 776304 597003 597003 0.284673213959 0.284673213959 1 -compress ctw expert ctw compress:ctw 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 73.8833333333 0.0862167810425 73.9 73.79 73.96 73.3033333333 0.51 0.054139458143 0.0541271989175 1293981.33333 9.23760430703 1293976 1293976 1293992 1177962 1177962 0.280848026276 0.280848026276 1 -compress ctw expert ctw compress:ctw 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 187.286666667 0.605750223552 187.02 186.86 187.98 186.09 1.02666666667 0.0509209210098 0.0509931727305 2492988 100.637965003 2492936 2492924 2493104 2746861 2746861 0.2746861 0.2746861 1 -compress match expert match compress:match 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0 0 0 0 0 0 0 5580 30.1993377411 5576 5552 5612 2874 2874 0.70166015625 0.70166015625 1 -compress match expert match compress:match 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.01 0 0.01 0.01 0.01 0.00333333333333 0 1.5625 1.5625 5444 38.1575680567 5448 5404 5480 13506 13506 0.824340820312 0.824340820312 1 -compress match expert match compress:match 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.03 0 0.03 0.03 0.03 0.03 0 2.08333333333 2.08333333333 6345.33333333 135.371094896 6380 6196 6460 53748 53748 0.820129394531 0.820129394531 1 -compress match expert match compress:match 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 0.13 0 0.13 0.13 0.13 0.13 0 1.92307692308 1.92307692308 8185.33333333 129.26458654 8152 8076 8328 206203 206203 0.786602020264 0.786602020264 1 -compress match expert match compress:match 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 0.523333333333 0.0057735026919 0.52 0.52 0.53 0.516666666667 0 1.91098209966 1.92307692308 12101.3333333 55.4737175006 12116 12040 12148 817505 817505 0.779633522034 0.779633522034 1 -compress match expert match compress:match 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 1.05333333333 0.0057735026919 1.05 1.05 1.06 1.04333333333 0 1.89877208745 1.90476190476 22105.3333333 80.133222407 22100 22028 22188 1644756 1644756 0.784280776978 0.784280776978 1 -compress match expert match compress:match 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 2.12 0.01 2.12 2.11 2.13 2.10666666667 0.00666666666667 1.88682044076 1.88679245283 28874.6666667 71.1430483838 28844 28824 28956 3292751 3292751 0.785053014755 0.785053014755 1 -compress match expert match compress:match 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 5.1 0.0529150262213 5.08 5.06 5.16 5.07666666667 0.0133333333333 1.87008317887 1.87731164647 55541.3333333 159.214739686 55564 55372 55688 7859324 7859324 0.7859324 0.7859324 1 -compress neural_mixture mixture neural-mixture compress:neural_mixture 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.14 0 0.14 0.14 0.14 0.14 0 0.0279017857143 0.0279017857143 17324 97.0772887961 17296 17244 17432 1001 1001 0.244384765625 0.244384765625 1 -compress neural_mixture mixture neural-mixture compress:neural_mixture 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.616666666667 0.0057735026919 0.62 0.61 0.62 0.596666666667 0.0133333333333 0.0253393266349 0.0252016129032 52718.6666667 78.621455935 52688 52660 52808 5533 5533 0.337707519531 0.337707519531 1 -compress neural_mixture mixture neural-mixture compress:neural_mixture 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 2.68333333333 0.0152752523165 2.68 2.67 2.7 2.64 0.04 0.0232924277903 0.0233208955224 139078.666667 72.0370275159 139112 138996 139128 19344 19344 0.295166015625 0.295166015625 1 -compress neural_mixture mixture neural-mixture compress:neural_mixture 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 11.75 0.0360555127546 11.74 11.72 11.79 11.6 0.13 0.0212767291492 0.0212947189097 428732 126.427845034 428720 428612 428864 67454 67454 0.257316589355 0.257316589355 1 -compress neural_mixture mixture neural-mixture compress:neural_mixture 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 52.2133333333 0.0929157324318 52.17 52.15 52.32 51.6833333333 0.473333333333 0.0191522365129 0.0191681042745 1107253.33333 68.0392043849 1107280 1107176 1107304 254251 254251 0.242472648621 0.242472648621 1 -compress neural_mixture mixture neural-mixture compress:neural_mixture 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 109.75 0.0754983443527 109.76 109.67 109.82 108.873333333 0.763333333333 0.018223240374 0.018221574344 1711381.33333 3690.44243057 1713500 1707120 1713524 503519 503519 0.240096569061 0.240096569061 1 -compress neural_mixture mixture neural-mixture compress:neural_mixture 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 232.046666667 0.343850742813 232.23 231.65 232.26 230.756666667 1.06333333333 0.0172379371695 0.0172243034922 2635456 86.8101376568 2635500 2635356 2635512 988355 988355 0.235642194748 0.235642194748 1 -compress neural_mixture mixture neural-mixture compress:neural_mixture 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 597.026666667 0.767875858022 596.9 596.33 597.85 594.173333333 2.25 0.0159737482604 0.0159771203955 4779589.33333 126.258993079 4779652 4779444 4779672 2270248 2270248 0.2270248 0.2270248 1 -compress ppmd expert ppmd compress:ppmd 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.01 0 0.01 0.01 0.01 0.00666666666667 0 0.390625 0.390625 7286.66666667 30.2875111776 7300 7252 7308 1058 1058 0.25830078125 0.25830078125 1 -compress ppmd expert ppmd compress:ppmd 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.04 0 0.04 0.04 0.04 0.04 0 0.390625 0.390625 16149.3333333 71.8145760506 16176 16068 16204 6267 6267 0.382507324219 0.382507324219 1 -compress ppmd expert ppmd compress:ppmd 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.21 0 0.21 0.21 0.21 0.19 0.01 0.297619047619 0.297619047619 44996 31.2409987036 44980 44976 45032 23186 23186 0.353790283203 0.353790283203 1 -compress ppmd expert ppmd compress:ppmd 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 1.00333333333 0.0057735026919 1 1 1.01 0.953333333333 0.0433333333333 0.249174917492 0.25 144040 81.1911325207 144024 143968 144128 83269 83269 0.317646026611 0.317646026611 1 -compress ppmd expert ppmd compress:ppmd 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 4.18333333333 0.0115470053838 4.19 4.17 4.19 3.97333333333 0.196666666667 0.239045040817 0.238663484487 399304 106.056588669 399280 399212 399420 326331 326331 0.311213493347 0.311213493347 1 -compress ppmd expert ppmd compress:ppmd 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 8.16 0.02 8.16 8.14 8.18 7.93666666667 0.206666666667 0.245099020807 0.245098039216 457593.333333 57.1780843797 457580 457544 457656 661953 661953 0.315643787384 0.315643787384 1 -compress ppmd expert ppmd compress:ppmd 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 16.1566666667 0.0404145188433 16.15 16.12 16.2 15.9333333333 0.206666666667 0.247576852213 0.247678018576 459790.666667 68.1566822354 459796 459720 459856 1329257 1329257 0.316919565201 0.316919565201 1 -compress ppmd expert ppmd compress:ppmd 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 39.45 0.503884907494 39.2 39.12 40.03 39.15 0.256666666667 0.241768647263 0.243284264389 569009.333333 68.8573404463 569032 568932 569064 3154465 3154465 0.3154465 0.3154465 1 -compress rosa expert rosaplus compress:rosa 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0 0 0 0 0 0 0 5778.66666667 128.332900432 5768 5656 5912 1127 1127 0.275146484375 0.275146484375 1 -compress rosa expert rosaplus compress:rosa 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.02 0 0.02 0.02 0.02 0.02 0 0.78125 0.78125 8544 131.696621065 8472 8464 8696 6359 6359 0.388122558594 0.388122558594 1 -compress rosa expert rosaplus compress:rosa 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.13 0 0.13 0.13 0.13 0.12 0.00333333333333 0.480769230769 0.480769230769 19288 97.3242004848 19336 19176 19352 22843 22843 0.348556518555 0.348556518555 1 -compress rosa expert rosaplus compress:rosa 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 0.77 0 0.77 0.77 0.77 0.743333333333 0.02 0.324675324675 0.324675324675 61938.6666667 54.6015872785 61964 61876 61976 80590 80590 0.307426452637 0.307426452637 1 -compress rosa expert rosaplus compress:rosa 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 4.42 0.01 4.42 4.41 4.43 4.36 0.0533333333333 0.226245115939 0.226244343891 172262.666667 104.025637866 172260 172160 172368 306522 306522 0.292322158813 0.292322158813 1 -compress rosa expert rosaplus compress:rosa 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 10.7166666667 0.0351188458428 10.72 10.68 10.75 10.5833333333 0.12 0.186626531137 0.186567164179 337446.666667 77.1837634048 337472 337360 337508 608681 608681 0.290241718292 0.290241718292 1 -compress rosa expert rosaplus compress:rosa 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 25.8666666667 0.037859388972 25.85 25.84 25.91 25.57 0.266666666667 0.154639395935 0.154738878143 669257.333333 84.5064100133 669268 669168 669336 1199345 1199345 0.285946130753 0.285946130753 1 -compress rosa expert rosaplus compress:rosa 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 77.03 0.209523268398 76.98 76.85 77.26 76.2933333333 0.666666666667 0.123806181484 0.123885985504 1556880 64 1556880 1556816 1556944 2752778 2752778 0.2752778 0.2752778 1 -compress rwkv expert rwkv compress:rwkv 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.06 0 0.06 0.06 0.06 0.06 0 0.0651041666667 0.0651041666667 7686.66666667 71.5914333795 7704 7608 7748 3714 3714 0.90673828125 0.90673828125 1 -compress rwkv expert rwkv compress:rwkv 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.236666666667 0.0057735026919 0.24 0.23 0.24 0.236666666667 0 0.066047705314 0.0651041666667 7550.66666667 115.677713209 7528 7448 7676 12000 12000 0.732421875 0.732421875 1 -compress rwkv expert rwkv compress:rwkv 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.946666666667 0.0057735026919 0.95 0.94 0.95 0.943333333333 0 0.0660227696902 0.0657894736842 7664 93.7229961109 7712 7556 7724 35340 35340 0.539245605469 0.539245605469 1 -compress rwkv expert rwkv compress:rwkv 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 3.78666666667 0.030550504633 3.78 3.76 3.82 3.78 0 0.0660239846726 0.0661375661376 7850.66666667 49.6923870762 7824 7820 7908 137010 137010 0.522651672363 0.522651672363 1 -compress rwkv expert rwkv compress:rwkv 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 15.8833333333 0.145716619963 15.93 15.72 16 15.7833333333 0.0766666666667 0.0629626235327 0.0627746390458 9188 115.723809132 9140 9104 9320 474275 474275 0.452303886414 0.452303886414 1 -compress rwkv expert rwkv compress:rwkv 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 32.5 0.223383079037 32.43 32.32 32.75 31.9066666667 0.56 0.0615403941408 0.0616712920136 11309.3333333 94.0070919311 11308 11216 11404 917086 917086 0.437300682068 0.437300682068 1 -compress rwkv expert rwkv compress:rwkv 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 66.5466666667 0.297713508819 66.39 66.36 66.89 64.73 1.75333333333 0.0601089947417 0.0602500376563 14800 34.1760149813 14796 14768 14836 1748887 1748887 0.416967153549 0.416967153549 1 -compress rwkv expert rwkv compress:rwkv 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 157.09 0.0556776436283 157.08 157.04 157.15 155.013333333 1.94666666667 0.0607087909012 0.0607126506497 24866.6666667 202.596479074 24904 24648 25048 3968645 3968645 0.3968645 0.3968645 1 -decompress ctw expert ctw decompress:ctw 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.05 0 0.05 0.05 0.05 0.04 0 0.078125 0.078125 12066.6666667 37.1662929727 12084 12024 12092 1332 1332 0.3251953125 0.3251953125 1 -decompress ctw expert ctw decompress:ctw 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.22 0 0.22 0.22 0.22 0.206666666667 0.00333333333333 0.0710227272727 0.0710227272727 34340 101.429778665 34384 34224 34412 6579 6579 0.401550292969 0.401550292969 1 -decompress ctw expert ctw decompress:ctw 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.953333333333 0.0057735026919 0.95 0.95 0.96 0.93 0.02 0.0655610380117 0.0657894736842 76301.3333333 81.7149517122 76336 76208 76360 22728 22728 0.346801757812 0.346801757812 1 -decompress ctw expert ctw decompress:ctw 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 3.99 0 3.99 3.99 3.99 3.92 0.0633333333333 0.062656641604 0.062656641604 185817.333333 27.2274371422 185808 185796 185848 80004 80004 0.305191040039 0.305191040039 1 -decompress ctw expert ctw decompress:ctw 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 17.13 0.01 17.13 17.12 17.14 16.94 0.173333333333 0.0583771294333 0.0583771161705 471121.333333 39.4630628985 471140 471076 471148 301713 301713 0.287735939026 0.287735939026 1 -decompress ctw expert ctw decompress:ctw 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 35.62 0.0458257569496 35.63 35.57 35.66 35.28 0.303333333333 0.0561482933107 0.0561324726354 775714.666667 133.226623966 775672 775608 775864 597003 597003 0.284673213959 0.284673213959 1 -decompress ctw expert ctw decompress:ctw 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 74.2766666667 0.0901849950565 74.27 74.19 74.37 73.68 0.526666666667 0.0538527657507 0.0538575467887 1292921.33333 95.1910359925 1292904 1292836 1293024 1177962 1177962 0.280848026276 0.280848026276 1 -decompress ctw expert ctw decompress:ctw 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 187.813333333 0.110151410946 187.82 187.7 187.92 186.63 1.01333333333 0.0507777865493 0.0507759725485 2490286.66667 11.5470053838 2490280 2490280 2490300 2746861 2746861 0.2746861 0.2746861 1 -decompress match expert match decompress:match 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0 0 0 0 0 0 0 5437.33333333 42.3949682549 5444 5392 5476 2874 2874 0.70166015625 0.70166015625 1 -decompress match expert match decompress:match 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.01 0 0.01 0.01 0.01 0.00666666666667 0 1.5625 1.5625 5546.66666667 89.1141589947 5592 5444 5604 13506 13506 0.824340820312 0.824340820312 1 -decompress match expert match decompress:match 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.03 0 0.03 0.03 0.03 0.03 0 2.08333333333 2.08333333333 6501.33333333 72.590173807 6472 6448 6584 53748 53748 0.820129394531 0.820129394531 1 -decompress match expert match decompress:match 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 0.14 0 0.14 0.14 0.14 0.14 0 1.78571428571 1.78571428571 7830.66666667 125.049323602 7812 7716 7964 206203 206203 0.786602020264 0.786602020264 1 -decompress match expert match decompress:match 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 0.566666666667 0.0057735026919 0.57 0.56 0.57 0.556666666667 0 1.76482873851 1.75438596491 11585.3333333 157.09020763 11564 11440 11752 817505 817505 0.779633522034 0.779633522034 1 -decompress match expert match decompress:match 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 1.13666666667 0.0115470053838 1.13 1.13 1.15 1.12 0.01 1.75965114788 1.76991150442 20094.6666667 70.2376916857 20088 20028 20168 1644756 1644756 0.784280776978 0.784280776978 1 -decompress match expert match decompress:match 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 2.27 0.0173205080757 2.26 2.26 2.29 2.25333333333 0.00666666666667 1.76218263323 1.76991150442 23809.3333333 211.105029152 23708 23668 24052 3292751 3292751 0.785053014755 0.785053014755 1 -decompress match expert match decompress:match 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 5.47666666667 0.0404145188433 5.47 5.44 5.52 5.44666666667 0.0233333333333 1.74140391729 1.74346310129 45964 131.635861375 45888 45888 46116 7859324 7859324 0.7859324 0.7859324 1 -decompress neural_mixture mixture neural-mixture decompress:neural_mixture 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.14 0 0.14 0.14 0.14 0.136666666667 0 0.0279017857143 0.0279017857143 17712 92.2605007574 17744 17608 17784 1001 1001 0.244384765625 0.244384765625 1 -decompress neural_mixture mixture neural-mixture decompress:neural_mixture 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.62 0.01 0.62 0.61 0.63 0.603333333333 0.01 0.0252059847677 0.0252016129032 52370.6666667 102.787807318 52356 52276 52480 5533 5533 0.337707519531 0.337707519531 1 -decompress neural_mixture mixture neural-mixture decompress:neural_mixture 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 2.67333333333 0.0057735026919 2.67 2.67 2.68 2.61666666667 0.05 0.0233791249744 0.0234082397004 140181.333333 92.7218061371 140136 140120 140288 19344 19344 0.295166015625 0.295166015625 1 -decompress neural_mixture mixture neural-mixture decompress:neural_mixture 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 11.6933333333 0.030550504633 11.7 11.66 11.72 11.5333333333 0.143333333333 0.0213798009052 0.0213675213675 423696 69.7423830967 423664 423648 423776 67454 67454 0.257316589355 0.257316589355 1 -decompress neural_mixture mixture neural-mixture decompress:neural_mixture 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 52.1033333333 0.0416333199893 52.09 52.07 52.15 51.54 0.506666666667 0.0191926381967 0.0191975427145 1107317.33333 124.085991689 1107312 1107196 1107444 254251 254251 0.242472648621 0.242472648621 1 -decompress neural_mixture mixture neural-mixture decompress:neural_mixture 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 109.723333333 0.528803681278 109.48 109.36 110.33 108.856666667 0.753333333333 0.0182279450316 0.018268176836 1715134.66667 40.0666112035 1715120 1715104 1715180 503519 503519 0.240096569061 0.240096569061 1 -decompress neural_mixture mixture neural-mixture decompress:neural_mixture 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 231.536666667 0.375410886008 231.33 231.31 231.97 230.143333333 1.16333333333 0.0172759116794 0.0172913154368 2634072 98.0612053771 2634068 2633976 2634172 988355 988355 0.235642194748 0.235642194748 1 -decompress neural_mixture mixture neural-mixture decompress:neural_mixture 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 595.826666667 0.380043857118 595.67 595.55 596.26 593.1 2.14333333333 0.0160059062197 0.0160101115787 4789958.66667 65.0333247907 4789972 4789888 4790016 2270248 2270248 0.2270248 0.2270248 1 -decompress ppmd expert ppmd decompress:ppmd 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.01 0 0.01 0.01 0.01 0.00666666666667 0 0.390625 0.390625 7206.66666667 140.986997036 7240 7052 7328 1058 1058 0.25830078125 0.25830078125 1 -decompress ppmd expert ppmd decompress:ppmd 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.04 0 0.04 0.04 0.04 0.04 0 0.390625 0.390625 16002.6666667 68.973424834 16016 15928 16064 6267 6267 0.382507324219 0.382507324219 1 -decompress ppmd expert ppmd decompress:ppmd 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.21 0 0.21 0.21 0.21 0.193333333333 0.0166666666667 0.297619047619 0.297619047619 45142.6666667 140.076169755 45148 45000 45280 23186 23186 0.353790283203 0.353790283203 1 -decompress ppmd expert ppmd decompress:ppmd 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 1.01333333333 0.0152752523165 1.01 1 1.03 0.95 0.06 0.246747733026 0.247524752475 144078.666667 99.9466524369 144056 143992 144188 83269 83269 0.317646026611 0.317646026611 1 -decompress ppmd expert ppmd decompress:ppmd 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 4.23333333333 0.0152752523165 4.23 4.22 4.25 4.05333333333 0.176666666667 0.236222520559 0.236406619385 399106.666667 30.550504633 399100 399080 399140 326331 326331 0.311213493347 0.311213493347 1 -decompress ppmd expert ppmd decompress:ppmd 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 8.25666666667 0.0351188458428 8.26 8.22 8.29 8.03666666667 0.203333333333 0.24223142552 0.242130750605 457565.333333 212.577828885 457476 457412 457808 661953 661953 0.315643787384 0.315643787384 1 -decompress ppmd expert ppmd decompress:ppmd 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 16.29 0.03 16.29 16.26 16.32 16.0666666667 0.206666666667 0.24554997202 0.24554941682 458100 96.0832971957 458068 458024 458208 1329257 1329257 0.316919565201 0.316919565201 1 -decompress ppmd expert ppmd decompress:ppmd 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 39.41 0.0781024967591 39.45 39.32 39.46 39.0966666667 0.273333333333 0.241988534956 0.241742539013 559100 82.6559132791 559112 559012 559176 3154465 3154465 0.3154465 0.3154465 1 -decompress rosa expert rosaplus decompress:rosa 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0 0 0 0 0 0 0 5856 48.4974226119 5864 5804 5900 1127 1127 0.275146484375 0.275146484375 1 -decompress rosa expert rosaplus decompress:rosa 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.02 0 0.02 0.02 0.02 0.02 0 0.78125 0.78125 8648 10.5830052443 8652 8636 8656 6359 6359 0.388122558594 0.388122558594 1 -decompress rosa expert rosaplus decompress:rosa 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.13 0 0.13 0.13 0.13 0.13 0 0.480769230769 0.480769230769 19366.6666667 40.0666112035 19364 19328 19408 22843 22843 0.348556518555 0.348556518555 1 -decompress rosa expert rosaplus decompress:rosa 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 0.78 0 0.78 0.78 0.78 0.77 0.00666666666667 0.320512820513 0.320512820513 61964 114.332847424 61900 61896 62096 80590 80590 0.307426452637 0.307426452637 1 -decompress rosa expert rosaplus decompress:rosa 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 4.47666666667 0.0057735026919 4.48 4.47 4.48 4.42333333333 0.05 0.22338073932 0.223214285714 172705.333333 138.814024267 172688 172576 172852 306522 306522 0.292322158813 0.292322158813 1 -decompress rosa expert rosaplus decompress:rosa 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 10.83 0.07 10.8 10.78 10.91 10.6866666667 0.13 0.18467733299 0.185185185185 338074.666667 98.7387124351 338048 337992 338184 608681 608681 0.290241718292 0.290241718292 1 -decompress rosa expert rosaplus decompress:rosa 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 26.2533333333 0.159478316185 26.21 26.12 26.43 25.95 0.273333333333 0.152365344583 0.152613506295 658405.333333 70.4651213959 658444 658324 658448 1199345 1199345 0.285946130753 0.285946130753 1 -decompress rosa expert rosaplus decompress:rosa 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 77.3966666667 0.0650640709865 77.4 77.33 77.46 76.62 0.713333333333 0.123219100618 0.12321373597 1555084 45.4312667664 1555104 1555032 1555116 2752778 2752778 0.2752778 0.2752778 1 -decompress rwkv expert rwkv decompress:rwkv 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.06 0 0.06 0.06 0.06 0.06 0 0.0651041666667 0.0651041666667 7670.66666667 28.3783955384 7676 7640 7696 3714 3714 0.90673828125 0.90673828125 1 -decompress rwkv expert rwkv decompress:rwkv 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.233333333333 0.0057735026919 0.23 0.23 0.24 0.233333333333 0 0.0669912439614 0.0679347826087 7644 48.6621002424 7668 7588 7676 12000 12000 0.732421875 0.732421875 1 -decompress rwkv expert rwkv decompress:rwkv 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.936666666667 0.0057735026919 0.94 0.93 0.94 0.936666666667 0 0.0667276748265 0.0664893617021 7785.33333333 136.489315821 7772 7656 7928 35340 35340 0.539245605469 0.539245605469 1 -decompress rwkv expert rwkv decompress:rwkv 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 3.79333333333 0.0450924975282 3.79 3.75 3.84 3.78666666667 0 0.0659112980064 0.065963060686 7810.66666667 68.8573404463 7788 7756 7888 137010 137010 0.522651672363 0.522651672363 1 -decompress rwkv expert rwkv decompress:rwkv 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 15.6733333333 0.136503968196 15.65 15.55 15.82 15.6566666667 0 0.0638058568028 0.0638977635783 8841.33333333 52.8141395209 8852 8784 8888 474275 474275 0.452303886414 0.452303886414 1 -decompress rwkv expert rwkv decompress:rwkv 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 31.74 0.173493515729 31.83 31.54 31.85 31.72 0 0.063013231332 0.0628338045869 10320 113.41957503 10280 10232 10448 917086 917086 0.437300682068 0.437300682068 1 -decompress rwkv expert rwkv decompress:rwkv 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 63.79 0.347706773014 63.96 63.39 64.02 63.7433333333 0 0.0627069991129 0.0625390869293 13062.6666667 36.2950869035 13048 13036 13104 1748887 1748887 0.416967153549 0.416967153549 1 -decompress rwkv expert rwkv decompress:rwkv 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 153.38 0.75146523539 153.71 152.52 153.91 153.26 0.00333333333333 0.0621782251757 0.0620437392757 20960 32.7414110875 20968 20924 20988 3968645 3968645 0.3968645 0.3968645 1 +operation subject subject_kind expert_kind series size_bytes repeats cpu compression_backend input_sha256 real_seconds_mean real_seconds_stdev real_seconds_median real_seconds_min real_seconds_max user_seconds_mean sys_seconds_mean throughput_mib_s_mean throughput_mib_s_median rss_kib_mean rss_kib_stdev rss_kib_median rss_kib_min rss_kib_max archive_bytes_mean archive_bytes_median archive_ratio_mean archive_ratio_median entropy_bpb_mean entropy_bpb_median verified_all +h ctw expert ctw h:ctw 4096 3 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.0366666666667 0.0057735026919 0.04 0.03 0.04 0.03 0 0.108506944444 0.09765625 11901.3333333 151.437555888 11968 11728 12008 2.56549624542 2.56549624542 1 +h ctw expert ctw h:ctw 16384 3 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.17 0 0.17 0.17 0.17 0.163333333333 0 0.0919117647059 0.0919117647059 34488 62.4819974073 34456 34448 34560 3.20340951209 3.20340951209 1 +h ctw expert ctw h:ctw 65536 3 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.75 0 0.75 0.75 0.75 0.72 0.0166666666667 0.0833333333333 0.0833333333333 76324 176.13631085 76392 76124 76456 2.77213895634 2.77213895634 1 +h ctw expert ctw h:ctw 262144 3 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 3.17333333333 0.0057735026919 3.17 3.17 3.18 3.09 0.0766666666667 0.0787816862753 0.0788643533123 185469.333333 133.16656237 185536 185316 185556 2.44096714152 2.44096714152 1 +h ctw expert ctw h:ctw 1048576 3 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 14.0166666667 0.0115470053838 14.01 14.01 14.03 13.84 0.156666666667 0.0713436707887 0.0713775874375 470858.666667 90.1849950565 470852 470772 470952 2.30174274755 2.30174274755 1 +h ctw expert ctw h:ctw 2097152 3 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 29.6 0.01 29.6 29.59 29.61 29.2633333333 0.303333333333 0.0675675727088 0.0675675675676 775094.666667 132.020200475 775092 774964 775228 2.27731545661 2.27731545661 1 +h ctw expert ctw h:ctw 4194304 3 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 62.8266666667 0.140118997047 62.87 62.67 62.94 62.2833333333 0.486666666667 0.0636674439169 0.0636233497694 1291712 26.2297540972 1291716 1291684 1291736 2.24674930722 2.24674930722 1 +h ctw expert ctw h:ctw 10000000 3 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 160.896666667 0.0472581562625 160.88 160.86 160.95 159.776666667 0.97 0.0592724753723 0.0592786124072 2487692 77.1492060879 2487660 2487636 2487780 2.19747398481 2.19747398481 1 +h match expert match h:match 4096 3 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0 0 0 0 0 0 0 5770.66666667 46.8757222167 5752 5736 5824 5.57734993252 5.57734993252 1 +h match expert match h:match 16384 3 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0 0 0 0 0 0 0 5732 157.124154731 5696 5596 5904 6.58586286407 6.58586286407 1 +h match expert match h:match 65536 3 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0 0 0 0 0 0 0 6746.66666667 31.0698138606 6756 6712 6772 6.5588255305 6.5588255305 1 +h match expert match h:match 262144 3 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 0.03 0 0.03 0.03 0.03 0.0233333333333 0 8.33333333333 8.33333333333 7885.33333333 169.81558625 7844 7740 8072 6.29225944669 6.29225944669 1 +h match expert match h:match 1048576 3 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 0.126666666667 0.0057735026919 0.13 0.12 0.13 0.126666666667 0 7.90598290598 7.69230769231 11401.3333333 98.1699207157 11408 11300 11496 6.23692635511 6.23692635511 1 +h match expert match h:match 2097152 3 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 0.263333333333 0.0152752523165 0.26 0.25 0.28 0.26 0 7.61172161172 7.69230769231 19294.6666667 37.1662929727 19284 19264 19336 6.2741772649 6.2741772649 1 +h match expert match h:match 4194304 3 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 0.523333333333 0.0057735026919 0.52 0.52 0.53 0.513333333333 0.01 7.64392839865 7.69230769231 21257.3333333 134.600643882 21272 21116 21384 6.28038800899 6.28038800899 1 +h match expert match h:match 10000000 3 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 1.29333333333 0.0152752523165 1.29 1.28 1.31 1.27333333333 0.01 7.37445378963 7.39282415819 40373.3333333 87.300248186 40356 40296 40468 6.28744436593 6.28744436593 1 +h neural_mixture mixture neural-mixture h:neural_mixture 4096 3 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.13 0 0.13 0.13 0.13 0.123333333333 0.00333333333333 0.0300480769231 0.0300480769231 17454.6666667 134.004975032 17528 17300 17536 1.91857366907 1.91857366907 1 +h neural_mixture mixture neural-mixture h:neural_mixture 16384 3 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.566666666667 0.0057735026919 0.57 0.56 0.57 0.543333333333 0.0166666666667 0.0275754490393 0.0274122807018 52590.6666667 16.6533279957 52596 52572 52604 2.69245779825 2.69245779825 1 +h neural_mixture mixture neural-mixture h:neural_mixture 65536 3 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 2.45666666667 0.0057735026919 2.46 2.45 2.46 2.41 0.04 0.0254410707372 0.025406504065 139388 122.572427568 139336 139300 139528 2.35902117116 2.35902117116 1 +h neural_mixture mixture neural-mixture h:neural_mixture 262144 3 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 10.66 0.0173205080757 10.65 10.65 10.68 10.5066666667 0.136666666667 0.023452198836 0.0234741784038 427940 40 427940 427900 427980 2.05795611601 2.05795611601 1 +h neural_mixture mixture neural-mixture h:neural_mixture 1048576 3 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 46.9333333333 0.0321455025366 46.92 46.91 46.97 46.37 0.513333333333 0.021306824843 0.0213128729753 1107021.33333 43.8786204584 1107036 1106972 1107056 1.93963669332 1.93963669332 1 +h neural_mixture mixture neural-mixture h:neural_mixture 2097152 3 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 98.5133333333 0.165630109984 98.53 98.34 98.67 97.64 0.776666666667 0.0203018586652 0.0202983862783 1717540 80.8949936646 1717572 1717448 1717600 1.92069854644 1.92069854644 1 +h neural_mixture mixture neural-mixture h:neural_mixture 4194304 3 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 206.973333333 0.222785397487 206.86 206.83 207.23 205.693333333 1.08 0.019326176099 0.0193367494924 2643548 100.955435713 2643520 2643464 2643660 1.88509670528 1.88509670528 1 +h neural_mixture mixture neural-mixture h:neural_mixture 10000000 3 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 528.096666667 0.439355588713 527.9 527.79 528.6 525.41 2.17666666667 0.0180587156948 0.0180654350522 4790344 124.771791684 4790412 4790200 4790420 1.81617547371 1.81617547371 1 +h ppmd expert ppmd h:ppmd 4096 3 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.01 0 0.01 0.01 0.01 0.00333333333333 0 0.390625 0.390625 7466.66666667 70.691819423 7432 7420 7548 2.02916276221 2.02916276221 1 +h ppmd expert ppmd h:ppmd 16384 3 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.0333333333333 0.0057735026919 0.03 0.03 0.04 0.0333333333333 0 0.477430555556 0.520833333333 16036 72.7736215946 15996 15992 16120 3.05111741489 3.05111741489 1 +h ppmd expert ppmd h:ppmd 65536 3 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.186666666667 0.0057735026919 0.19 0.18 0.19 0.163333333333 0.0166666666667 0.335038986355 0.328947368421 45244 235.796522451 45304 44984 45444 2.82800309907 2.82800309907 1 +h ppmd expert ppmd h:ppmd 262144 3 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 0.906666666667 0.0057735026919 0.91 0.9 0.91 0.833333333333 0.0666666666667 0.275742775743 0.274725274725 144282.666667 138.814024267 144300 144136 144412 2.54059561824 2.54059561824 1 +h ppmd expert ppmd h:ppmd 1048576 3 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 3.80666666667 0.0057735026919 3.81 3.8 3.81 3.60666666667 0.19 0.26269742598 0.262467191601 398937.333333 97.7616148257 398916 398852 399044 2.48956475398 2.48956475398 1 +h ppmd expert ppmd h:ppmd 2097152 3 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 7.45 0.0916515138991 7.43 7.37 7.55 7.21333333333 0.22 0.268483362304 0.269179004038 457117.333333 122.659420076 457180 456976 457196 2.52508057302 2.52508057302 1 +h ppmd expert ppmd h:ppmd 4194304 3 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 14.77 0.105356537529 14.76 14.67 14.88 14.53 0.22 0.270828405889 0.271002710027 459222.666667 180.945664036 459280 459020 459368 2.5353209835 2.5353209835 1 +h ppmd expert ppmd h:ppmd 10000000 3 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 35.5666666667 0.0723417813807 35.53 35.52 35.65 35.2733333333 0.253333333333 0.268137847329 0.268413823925 557416 187.488666324 557484 557204 557560 2.52355814767 2.52355814767 1 +h rosa expert rosaplus h:rosa 4096 3 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.02 0 0.02 0.02 0.02 0.0133333333333 0 0.1953125 0.1953125 5938.66666667 181.563579314 5980 5740 6096 2.11172124286 2.11172124286 1 +h rosa expert rosaplus h:rosa 16384 3 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.12 0 0.12 0.12 0.12 0.106666666667 0.00666666666667 0.130208333333 0.130208333333 8038.66666667 49.6923870762 8012 8008 8096 3.48630622986 3.48630622986 1 +h rosa expert rosaplus h:rosa 65536 3 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.596666666667 0.0057735026919 0.6 0.59 0.6 0.573333333333 0.0133333333333 0.104755178908 0.104166666667 16633.3333333 123.309907685 16592 16536 16772 3.19056696201 3.19056696201 1 +h rosa expert rosaplus h:rosa 262144 3 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 3.42 0 3.42 3.42 3.42 3.31 0.0966666666667 0.0730994152047 0.0730994152047 47050.6666667 138.120720145 47012 46936 47204 2.90567254487 2.90567254487 1 +h rosa expert rosaplus h:rosa 1048576 3 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 20.0533333333 0.0115470053838 20.06 20.04 20.06 19.2933333333 0.726666666667 0.049867032303 0.049850448654 203772 34.6410161514 203752 203752 203812 2.80123027825 2.80123027825 1 +h rosa expert rosaplus h:rosa 2097152 3 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 47.6166666667 0.0057735026919 47.62 47.61 47.62 45.67 1.89 0.0420021005167 0.0419991600168 337785.333333 76.1402215214 337780 337712 337864 2.70437138702 2.70437138702 1 +h rosa expert rosaplus h:rosa 4194304 3 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 113.323333333 0.0838649708361 113.28 113.27 113.42 108.21 4.98333333333 0.0352972449909 0.0353107344633 643504 68.3520299625 643496 643440 643576 2.62099230176 2.62099230176 1 +h rosa expert rosaplus h:rosa 10000000 3 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 330.516666667 0.598024525695 330.66 329.86 331.03 317.14 13.0266666667 0.028854109196 0.028841538632 1551453.33333 88.1211287566 1551496 1551352 1551512 2.48481897791 2.48481897791 1 +h rwkv expert rwkv h:rwkv 4096 3 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.0633333333333 0.0057735026919 0.06 0.06 0.07 0.06 0 0.062003968254 0.0651041666667 8172 76 8152 8108 8256 7.21701437947 7.21701437947 1 +h rwkv expert rwkv h:rwkv 16384 3 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.23 0 0.23 0.23 0.23 0.226666666667 0 0.0679347826087 0.0679347826087 8221.33333333 161.855902992 8136 8120 8408 5.85021072042 5.85021072042 1 +h rwkv expert rwkv h:rwkv 65536 3 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.903333333333 0.0057735026919 0.9 0.9 0.91 0.9 0 0.0691900691901 0.0694444444444 8308 58.9236794506 8276 8272 8376 4.31173851819 4.31173851819 1 +h rwkv expert rwkv h:rwkv 262144 3 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 3.65333333333 0.0057735026919 3.65 3.65 3.66 3.64666666667 0 0.0684307707663 0.0684931506849 8248 86.5332306111 8224 8176 8344 4.18065126521 4.18065126521 1 +h rwkv expert rwkv h:rwkv 1048576 3 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 15.0733333333 0.119303534454 15.11 14.94 15.17 15.06 0 0.0663451064205 0.066181336863 8833.33333333 91.2432645916 8816 8752 8932 3.61829048151 3.61829048151 1 +h rwkv expert rwkv h:rwkv 2097152 3 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 30.61 0.283548937575 30.51 30.39 30.93 30.5833333333 0 0.0653418467794 0.0655522779417 9874.66666667 127.080027279 9944 9728 9952 3.49833649966 3.49833649966 1 +h rwkv expert rwkv h:rwkv 4194304 3 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 61.61 0.308058436015 61.73 61.26 61.84 61.5666666667 0 0.0649256100831 0.0647983152438 11898.6666667 22.0302821891 11888 11884 11924 3.33570098711 3.33570098711 1 +h rwkv expert rwkv h:rwkv 10000000 3 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 149.02 0.46184412955 148.9 148.63 149.53 148.913333333 0 0.0639968068945 0.064047972895 17758.6666667 116.091917606 17784 17632 17860 3.17490278117 3.17490278117 1 +compress ctw expert ctw compress:ctw 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.05 0 0.05 0.05 0.05 0.04 0 0.078125 0.078125 11792 78.7908624144 11768 11728 11880 1332 1332 0.3251953125 0.3251953125 1 +compress ctw expert ctw compress:ctw 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.22 0 0.22 0.22 0.22 0.2 0.01 0.0710227272727 0.0710227272727 34390.6666667 98.33276836 34400 34288 34484 6579 6579 0.401550292969 0.401550292969 1 +compress ctw expert ctw compress:ctw 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.953333333333 0.0057735026919 0.95 0.95 0.96 0.92 0.0233333333333 0.0655610380117 0.0657894736842 77308 68.3520299625 77300 77244 77380 22728 22728 0.346801757812 0.346801757812 1 +compress ctw expert ctw compress:ctw 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 3.98 0 3.98 3.98 3.98 3.89333333333 0.08 0.0628140703518 0.0628140703518 185717.333333 66.2520439936 185724 185648 185780 80004 80004 0.305191040039 0.305191040039 1 +compress ctw expert ctw compress:ctw 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 17.0666666667 0.0057735026919 17.07 17.06 17.07 16.89 0.16 0.0585937544712 0.0585823081429 471393.333333 95.4428275636 471436 471284 471460 301713 301713 0.287735939026 0.287735939026 1 +compress ctw expert ctw compress:ctw 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 35.4266666667 0.0251661147842 35.43 35.4 35.45 35.1 0.293333333333 0.0564546670944 0.0564493367203 776241.333333 108.541850608 776304 776116 776304 597003 597003 0.284673213959 0.284673213959 1 +compress ctw expert ctw compress:ctw 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 73.8833333333 0.0862167810425 73.9 73.79 73.96 73.3033333333 0.51 0.054139458143 0.0541271989175 1293981.33333 9.23760430703 1293976 1293976 1293992 1177962 1177962 0.280848026276 0.280848026276 1 +compress ctw expert ctw compress:ctw 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 187.286666667 0.605750223552 187.02 186.86 187.98 186.09 1.02666666667 0.0509209210098 0.0509931727305 2492988 100.637965003 2492936 2492924 2493104 2746861 2746861 0.2746861 0.2746861 1 +compress match expert match compress:match 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0 0 0 0 0 0 0 5580 30.1993377411 5576 5552 5612 2874 2874 0.70166015625 0.70166015625 1 +compress match expert match compress:match 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.01 0 0.01 0.01 0.01 0.00333333333333 0 1.5625 1.5625 5444 38.1575680567 5448 5404 5480 13506 13506 0.824340820312 0.824340820312 1 +compress match expert match compress:match 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.03 0 0.03 0.03 0.03 0.03 0 2.08333333333 2.08333333333 6345.33333333 135.371094896 6380 6196 6460 53748 53748 0.820129394531 0.820129394531 1 +compress match expert match compress:match 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 0.13 0 0.13 0.13 0.13 0.13 0 1.92307692308 1.92307692308 8185.33333333 129.26458654 8152 8076 8328 206203 206203 0.786602020264 0.786602020264 1 +compress match expert match compress:match 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 0.523333333333 0.0057735026919 0.52 0.52 0.53 0.516666666667 0 1.91098209966 1.92307692308 12101.3333333 55.4737175006 12116 12040 12148 817505 817505 0.779633522034 0.779633522034 1 +compress match expert match compress:match 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 1.05333333333 0.0057735026919 1.05 1.05 1.06 1.04333333333 0 1.89877208745 1.90476190476 22105.3333333 80.133222407 22100 22028 22188 1644756 1644756 0.784280776978 0.784280776978 1 +compress match expert match compress:match 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 2.12 0.01 2.12 2.11 2.13 2.10666666667 0.00666666666667 1.88682044076 1.88679245283 28874.6666667 71.1430483838 28844 28824 28956 3292751 3292751 0.785053014755 0.785053014755 1 +compress match expert match compress:match 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 5.1 0.0529150262213 5.08 5.06 5.16 5.07666666667 0.0133333333333 1.87008317887 1.87731164647 55541.3333333 159.214739686 55564 55372 55688 7859324 7859324 0.7859324 0.7859324 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.14 0 0.14 0.14 0.14 0.14 0 0.0279017857143 0.0279017857143 17324 97.0772887961 17296 17244 17432 1001 1001 0.244384765625 0.244384765625 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.616666666667 0.0057735026919 0.62 0.61 0.62 0.596666666667 0.0133333333333 0.0253393266349 0.0252016129032 52718.6666667 78.621455935 52688 52660 52808 5533 5533 0.337707519531 0.337707519531 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 2.68333333333 0.0152752523165 2.68 2.67 2.7 2.64 0.04 0.0232924277903 0.0233208955224 139078.666667 72.0370275159 139112 138996 139128 19344 19344 0.295166015625 0.295166015625 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 11.75 0.0360555127546 11.74 11.72 11.79 11.6 0.13 0.0212767291492 0.0212947189097 428732 126.427845034 428720 428612 428864 67454 67454 0.257316589355 0.257316589355 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 52.2133333333 0.0929157324318 52.17 52.15 52.32 51.6833333333 0.473333333333 0.0191522365129 0.0191681042745 1107253.33333 68.0392043849 1107280 1107176 1107304 254251 254251 0.242472648621 0.242472648621 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 109.75 0.0754983443527 109.76 109.67 109.82 108.873333333 0.763333333333 0.018223240374 0.018221574344 1711381.33333 3690.44243057 1713500 1707120 1713524 503519 503519 0.240096569061 0.240096569061 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 232.046666667 0.343850742813 232.23 231.65 232.26 230.756666667 1.06333333333 0.0172379371695 0.0172243034922 2635456 86.8101376568 2635500 2635356 2635512 988355 988355 0.235642194748 0.235642194748 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 597.026666667 0.767875858022 596.9 596.33 597.85 594.173333333 2.25 0.0159737482604 0.0159771203955 4779589.33333 126.258993079 4779652 4779444 4779672 2270248 2270248 0.2270248 0.2270248 1 +compress ppmd expert ppmd compress:ppmd 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.01 0 0.01 0.01 0.01 0.00666666666667 0 0.390625 0.390625 7286.66666667 30.2875111776 7300 7252 7308 1058 1058 0.25830078125 0.25830078125 1 +compress ppmd expert ppmd compress:ppmd 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.04 0 0.04 0.04 0.04 0.04 0 0.390625 0.390625 16149.3333333 71.8145760506 16176 16068 16204 6267 6267 0.382507324219 0.382507324219 1 +compress ppmd expert ppmd compress:ppmd 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.21 0 0.21 0.21 0.21 0.19 0.01 0.297619047619 0.297619047619 44996 31.2409987036 44980 44976 45032 23186 23186 0.353790283203 0.353790283203 1 +compress ppmd expert ppmd compress:ppmd 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 1.00333333333 0.0057735026919 1 1 1.01 0.953333333333 0.0433333333333 0.249174917492 0.25 144040 81.1911325207 144024 143968 144128 83269 83269 0.317646026611 0.317646026611 1 +compress ppmd expert ppmd compress:ppmd 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 4.18333333333 0.0115470053838 4.19 4.17 4.19 3.97333333333 0.196666666667 0.239045040817 0.238663484487 399304 106.056588669 399280 399212 399420 326331 326331 0.311213493347 0.311213493347 1 +compress ppmd expert ppmd compress:ppmd 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 8.16 0.02 8.16 8.14 8.18 7.93666666667 0.206666666667 0.245099020807 0.245098039216 457593.333333 57.1780843797 457580 457544 457656 661953 661953 0.315643787384 0.315643787384 1 +compress ppmd expert ppmd compress:ppmd 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 16.1566666667 0.0404145188433 16.15 16.12 16.2 15.9333333333 0.206666666667 0.247576852213 0.247678018576 459790.666667 68.1566822354 459796 459720 459856 1329257 1329257 0.316919565201 0.316919565201 1 +compress ppmd expert ppmd compress:ppmd 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 39.45 0.503884907494 39.2 39.12 40.03 39.15 0.256666666667 0.241768647263 0.243284264389 569009.333333 68.8573404463 569032 568932 569064 3154465 3154465 0.3154465 0.3154465 1 +compress rosa expert rosaplus compress:rosa 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0 0 0 0 0 0 0 5778.66666667 128.332900432 5768 5656 5912 1127 1127 0.275146484375 0.275146484375 1 +compress rosa expert rosaplus compress:rosa 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.02 0 0.02 0.02 0.02 0.02 0 0.78125 0.78125 8544 131.696621065 8472 8464 8696 6359 6359 0.388122558594 0.388122558594 1 +compress rosa expert rosaplus compress:rosa 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.13 0 0.13 0.13 0.13 0.12 0.00333333333333 0.480769230769 0.480769230769 19288 97.3242004848 19336 19176 19352 22843 22843 0.348556518555 0.348556518555 1 +compress rosa expert rosaplus compress:rosa 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 0.77 0 0.77 0.77 0.77 0.743333333333 0.02 0.324675324675 0.324675324675 61938.6666667 54.6015872785 61964 61876 61976 80590 80590 0.307426452637 0.307426452637 1 +compress rosa expert rosaplus compress:rosa 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 4.42 0.01 4.42 4.41 4.43 4.36 0.0533333333333 0.226245115939 0.226244343891 172262.666667 104.025637866 172260 172160 172368 306522 306522 0.292322158813 0.292322158813 1 +compress rosa expert rosaplus compress:rosa 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 10.7166666667 0.0351188458428 10.72 10.68 10.75 10.5833333333 0.12 0.186626531137 0.186567164179 337446.666667 77.1837634048 337472 337360 337508 608681 608681 0.290241718292 0.290241718292 1 +compress rosa expert rosaplus compress:rosa 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 25.8666666667 0.037859388972 25.85 25.84 25.91 25.57 0.266666666667 0.154639395935 0.154738878143 669257.333333 84.5064100133 669268 669168 669336 1199345 1199345 0.285946130753 0.285946130753 1 +compress rosa expert rosaplus compress:rosa 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 77.03 0.209523268398 76.98 76.85 77.26 76.2933333333 0.666666666667 0.123806181484 0.123885985504 1556880 64 1556880 1556816 1556944 2752778 2752778 0.2752778 0.2752778 1 +compress rwkv expert rwkv compress:rwkv 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.06 0 0.06 0.06 0.06 0.06 0 0.0651041666667 0.0651041666667 7686.66666667 71.5914333795 7704 7608 7748 3714 3714 0.90673828125 0.90673828125 1 +compress rwkv expert rwkv compress:rwkv 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.236666666667 0.0057735026919 0.24 0.23 0.24 0.236666666667 0 0.066047705314 0.0651041666667 7550.66666667 115.677713209 7528 7448 7676 12000 12000 0.732421875 0.732421875 1 +compress rwkv expert rwkv compress:rwkv 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.946666666667 0.0057735026919 0.95 0.94 0.95 0.943333333333 0 0.0660227696902 0.0657894736842 7664 93.7229961109 7712 7556 7724 35340 35340 0.539245605469 0.539245605469 1 +compress rwkv expert rwkv compress:rwkv 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 3.78666666667 0.030550504633 3.78 3.76 3.82 3.78 0 0.0660239846726 0.0661375661376 7850.66666667 49.6923870762 7824 7820 7908 137010 137010 0.522651672363 0.522651672363 1 +compress rwkv expert rwkv compress:rwkv 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 15.8833333333 0.145716619963 15.93 15.72 16 15.7833333333 0.0766666666667 0.0629626235327 0.0627746390458 9188 115.723809132 9140 9104 9320 474275 474275 0.452303886414 0.452303886414 1 +compress rwkv expert rwkv compress:rwkv 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 32.5 0.223383079037 32.43 32.32 32.75 31.9066666667 0.56 0.0615403941408 0.0616712920136 11309.3333333 94.0070919311 11308 11216 11404 917086 917086 0.437300682068 0.437300682068 1 +compress rwkv expert rwkv compress:rwkv 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 66.5466666667 0.297713508819 66.39 66.36 66.89 64.73 1.75333333333 0.0601089947417 0.0602500376563 14800 34.1760149813 14796 14768 14836 1748887 1748887 0.416967153549 0.416967153549 1 +compress rwkv expert rwkv compress:rwkv 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 157.09 0.0556776436283 157.08 157.04 157.15 155.013333333 1.94666666667 0.0607087909012 0.0607126506497 24866.6666667 202.596479074 24904 24648 25048 3968645 3968645 0.3968645 0.3968645 1 +decompress ctw expert ctw decompress:ctw 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.05 0 0.05 0.05 0.05 0.04 0 0.078125 0.078125 12066.6666667 37.1662929727 12084 12024 12092 1332 1332 0.3251953125 0.3251953125 1 +decompress ctw expert ctw decompress:ctw 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.22 0 0.22 0.22 0.22 0.206666666667 0.00333333333333 0.0710227272727 0.0710227272727 34340 101.429778665 34384 34224 34412 6579 6579 0.401550292969 0.401550292969 1 +decompress ctw expert ctw decompress:ctw 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.953333333333 0.0057735026919 0.95 0.95 0.96 0.93 0.02 0.0655610380117 0.0657894736842 76301.3333333 81.7149517122 76336 76208 76360 22728 22728 0.346801757812 0.346801757812 1 +decompress ctw expert ctw decompress:ctw 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 3.99 0 3.99 3.99 3.99 3.92 0.0633333333333 0.062656641604 0.062656641604 185817.333333 27.2274371422 185808 185796 185848 80004 80004 0.305191040039 0.305191040039 1 +decompress ctw expert ctw decompress:ctw 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 17.13 0.01 17.13 17.12 17.14 16.94 0.173333333333 0.0583771294333 0.0583771161705 471121.333333 39.4630628985 471140 471076 471148 301713 301713 0.287735939026 0.287735939026 1 +decompress ctw expert ctw decompress:ctw 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 35.62 0.0458257569496 35.63 35.57 35.66 35.28 0.303333333333 0.0561482933107 0.0561324726354 775714.666667 133.226623966 775672 775608 775864 597003 597003 0.284673213959 0.284673213959 1 +decompress ctw expert ctw decompress:ctw 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 74.2766666667 0.0901849950565 74.27 74.19 74.37 73.68 0.526666666667 0.0538527657507 0.0538575467887 1292921.33333 95.1910359925 1292904 1292836 1293024 1177962 1177962 0.280848026276 0.280848026276 1 +decompress ctw expert ctw decompress:ctw 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 187.813333333 0.110151410946 187.82 187.7 187.92 186.63 1.01333333333 0.0507777865493 0.0507759725485 2490286.66667 11.5470053838 2490280 2490280 2490300 2746861 2746861 0.2746861 0.2746861 1 +decompress match expert match decompress:match 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0 0 0 0 0 0 0 5437.33333333 42.3949682549 5444 5392 5476 2874 2874 0.70166015625 0.70166015625 1 +decompress match expert match decompress:match 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.01 0 0.01 0.01 0.01 0.00666666666667 0 1.5625 1.5625 5546.66666667 89.1141589947 5592 5444 5604 13506 13506 0.824340820312 0.824340820312 1 +decompress match expert match decompress:match 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.03 0 0.03 0.03 0.03 0.03 0 2.08333333333 2.08333333333 6501.33333333 72.590173807 6472 6448 6584 53748 53748 0.820129394531 0.820129394531 1 +decompress match expert match decompress:match 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 0.14 0 0.14 0.14 0.14 0.14 0 1.78571428571 1.78571428571 7830.66666667 125.049323602 7812 7716 7964 206203 206203 0.786602020264 0.786602020264 1 +decompress match expert match decompress:match 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 0.566666666667 0.0057735026919 0.57 0.56 0.57 0.556666666667 0 1.76482873851 1.75438596491 11585.3333333 157.09020763 11564 11440 11752 817505 817505 0.779633522034 0.779633522034 1 +decompress match expert match decompress:match 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 1.13666666667 0.0115470053838 1.13 1.13 1.15 1.12 0.01 1.75965114788 1.76991150442 20094.6666667 70.2376916857 20088 20028 20168 1644756 1644756 0.784280776978 0.784280776978 1 +decompress match expert match decompress:match 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 2.27 0.0173205080757 2.26 2.26 2.29 2.25333333333 0.00666666666667 1.76218263323 1.76991150442 23809.3333333 211.105029152 23708 23668 24052 3292751 3292751 0.785053014755 0.785053014755 1 +decompress match expert match decompress:match 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 5.47666666667 0.0404145188433 5.47 5.44 5.52 5.44666666667 0.0233333333333 1.74140391729 1.74346310129 45964 131.635861375 45888 45888 46116 7859324 7859324 0.7859324 0.7859324 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.14 0 0.14 0.14 0.14 0.136666666667 0 0.0279017857143 0.0279017857143 17712 92.2605007574 17744 17608 17784 1001 1001 0.244384765625 0.244384765625 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.62 0.01 0.62 0.61 0.63 0.603333333333 0.01 0.0252059847677 0.0252016129032 52370.6666667 102.787807318 52356 52276 52480 5533 5533 0.337707519531 0.337707519531 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 2.67333333333 0.0057735026919 2.67 2.67 2.68 2.61666666667 0.05 0.0233791249744 0.0234082397004 140181.333333 92.7218061371 140136 140120 140288 19344 19344 0.295166015625 0.295166015625 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 11.6933333333 0.030550504633 11.7 11.66 11.72 11.5333333333 0.143333333333 0.0213798009052 0.0213675213675 423696 69.7423830967 423664 423648 423776 67454 67454 0.257316589355 0.257316589355 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 52.1033333333 0.0416333199893 52.09 52.07 52.15 51.54 0.506666666667 0.0191926381967 0.0191975427145 1107317.33333 124.085991689 1107312 1107196 1107444 254251 254251 0.242472648621 0.242472648621 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 109.723333333 0.528803681278 109.48 109.36 110.33 108.856666667 0.753333333333 0.0182279450316 0.018268176836 1715134.66667 40.0666112035 1715120 1715104 1715180 503519 503519 0.240096569061 0.240096569061 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 231.536666667 0.375410886008 231.33 231.31 231.97 230.143333333 1.16333333333 0.0172759116794 0.0172913154368 2634072 98.0612053771 2634068 2633976 2634172 988355 988355 0.235642194748 0.235642194748 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 595.826666667 0.380043857118 595.67 595.55 596.26 593.1 2.14333333333 0.0160059062197 0.0160101115787 4789958.66667 65.0333247907 4789972 4789888 4790016 2270248 2270248 0.2270248 0.2270248 1 +decompress ppmd expert ppmd decompress:ppmd 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.01 0 0.01 0.01 0.01 0.00666666666667 0 0.390625 0.390625 7206.66666667 140.986997036 7240 7052 7328 1058 1058 0.25830078125 0.25830078125 1 +decompress ppmd expert ppmd decompress:ppmd 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.04 0 0.04 0.04 0.04 0.04 0 0.390625 0.390625 16002.6666667 68.973424834 16016 15928 16064 6267 6267 0.382507324219 0.382507324219 1 +decompress ppmd expert ppmd decompress:ppmd 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.21 0 0.21 0.21 0.21 0.193333333333 0.0166666666667 0.297619047619 0.297619047619 45142.6666667 140.076169755 45148 45000 45280 23186 23186 0.353790283203 0.353790283203 1 +decompress ppmd expert ppmd decompress:ppmd 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 1.01333333333 0.0152752523165 1.01 1 1.03 0.95 0.06 0.246747733026 0.247524752475 144078.666667 99.9466524369 144056 143992 144188 83269 83269 0.317646026611 0.317646026611 1 +decompress ppmd expert ppmd decompress:ppmd 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 4.23333333333 0.0152752523165 4.23 4.22 4.25 4.05333333333 0.176666666667 0.236222520559 0.236406619385 399106.666667 30.550504633 399100 399080 399140 326331 326331 0.311213493347 0.311213493347 1 +decompress ppmd expert ppmd decompress:ppmd 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 8.25666666667 0.0351188458428 8.26 8.22 8.29 8.03666666667 0.203333333333 0.24223142552 0.242130750605 457565.333333 212.577828885 457476 457412 457808 661953 661953 0.315643787384 0.315643787384 1 +decompress ppmd expert ppmd decompress:ppmd 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 16.29 0.03 16.29 16.26 16.32 16.0666666667 0.206666666667 0.24554997202 0.24554941682 458100 96.0832971957 458068 458024 458208 1329257 1329257 0.316919565201 0.316919565201 1 +decompress ppmd expert ppmd decompress:ppmd 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 39.41 0.0781024967591 39.45 39.32 39.46 39.0966666667 0.273333333333 0.241988534956 0.241742539013 559100 82.6559132791 559112 559012 559176 3154465 3154465 0.3154465 0.3154465 1 +decompress rosa expert rosaplus decompress:rosa 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0 0 0 0 0 0 0 5856 48.4974226119 5864 5804 5900 1127 1127 0.275146484375 0.275146484375 1 +decompress rosa expert rosaplus decompress:rosa 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.02 0 0.02 0.02 0.02 0.02 0 0.78125 0.78125 8648 10.5830052443 8652 8636 8656 6359 6359 0.388122558594 0.388122558594 1 +decompress rosa expert rosaplus decompress:rosa 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.13 0 0.13 0.13 0.13 0.13 0 0.480769230769 0.480769230769 19366.6666667 40.0666112035 19364 19328 19408 22843 22843 0.348556518555 0.348556518555 1 +decompress rosa expert rosaplus decompress:rosa 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 0.78 0 0.78 0.78 0.78 0.77 0.00666666666667 0.320512820513 0.320512820513 61964 114.332847424 61900 61896 62096 80590 80590 0.307426452637 0.307426452637 1 +decompress rosa expert rosaplus decompress:rosa 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 4.47666666667 0.0057735026919 4.48 4.47 4.48 4.42333333333 0.05 0.22338073932 0.223214285714 172705.333333 138.814024267 172688 172576 172852 306522 306522 0.292322158813 0.292322158813 1 +decompress rosa expert rosaplus decompress:rosa 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 10.83 0.07 10.8 10.78 10.91 10.6866666667 0.13 0.18467733299 0.185185185185 338074.666667 98.7387124351 338048 337992 338184 608681 608681 0.290241718292 0.290241718292 1 +decompress rosa expert rosaplus decompress:rosa 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 26.2533333333 0.159478316185 26.21 26.12 26.43 25.95 0.273333333333 0.152365344583 0.152613506295 658405.333333 70.4651213959 658444 658324 658448 1199345 1199345 0.285946130753 0.285946130753 1 +decompress rosa expert rosaplus decompress:rosa 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 77.3966666667 0.0650640709865 77.4 77.33 77.46 76.62 0.713333333333 0.123219100618 0.12321373597 1555084 45.4312667664 1555104 1555032 1555116 2752778 2752778 0.2752778 0.2752778 1 +decompress rwkv expert rwkv decompress:rwkv 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.06 0 0.06 0.06 0.06 0.06 0 0.0651041666667 0.0651041666667 7670.66666667 28.3783955384 7676 7640 7696 3714 3714 0.90673828125 0.90673828125 1 +decompress rwkv expert rwkv decompress:rwkv 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.233333333333 0.0057735026919 0.23 0.23 0.24 0.233333333333 0 0.0669912439614 0.0679347826087 7644 48.6621002424 7668 7588 7676 12000 12000 0.732421875 0.732421875 1 +decompress rwkv expert rwkv decompress:rwkv 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.936666666667 0.0057735026919 0.94 0.93 0.94 0.936666666667 0 0.0667276748265 0.0664893617021 7785.33333333 136.489315821 7772 7656 7928 35340 35340 0.539245605469 0.539245605469 1 +decompress rwkv expert rwkv decompress:rwkv 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 3.79333333333 0.0450924975282 3.79 3.75 3.84 3.78666666667 0 0.0659112980064 0.065963060686 7810.66666667 68.8573404463 7788 7756 7888 137010 137010 0.522651672363 0.522651672363 1 +decompress rwkv expert rwkv decompress:rwkv 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 15.6733333333 0.136503968196 15.65 15.55 15.82 15.6566666667 0 0.0638058568028 0.0638977635783 8841.33333333 52.8141395209 8852 8784 8888 474275 474275 0.452303886414 0.452303886414 1 +decompress rwkv expert rwkv decompress:rwkv 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 31.74 0.173493515729 31.83 31.54 31.85 31.72 0 0.063013231332 0.0628338045869 10320 113.41957503 10280 10232 10448 917086 917086 0.437300682068 0.437300682068 1 +decompress rwkv expert rwkv decompress:rwkv 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 63.79 0.347706773014 63.96 63.39 64.02 63.7433333333 0 0.0627069991129 0.0625390869293 13062.6666667 36.2950869035 13048 13036 13104 1748887 1748887 0.416967153549 0.416967153549 1 +decompress rwkv expert rwkv decompress:rwkv 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 153.38 0.75146523539 153.71 152.52 153.91 153.26 0.00333333333333 0.0621782251757 0.0620437392757 20960 32.7414110875 20968 20924 20988 3968645 3968645 0.3968645 0.3968645 1 diff --git a/benchmarks/baseline/infotheory-two-json-summary-20260310-212017.tsv b/benchmarks/baseline/infotheory-two-json-summary-20260310-212017.tsv index 9adc03ee..de4b01fb 100644 --- a/benchmarks/baseline/infotheory-two-json-summary-20260310-212017.tsv +++ b/benchmarks/baseline/infotheory-two-json-summary-20260310-212017.tsv @@ -1,145 +1,145 @@ -operation subject subject_kind expert_kind series size_bytes repeats cpu compression_backend input_sha256 real_seconds_mean real_seconds_stdev real_seconds_median real_seconds_min real_seconds_max user_seconds_mean sys_seconds_mean throughput_mib_s_mean throughput_mib_s_median rss_kib_mean rss_kib_stdev rss_kib_median rss_kib_min rss_kib_max archive_bytes_mean archive_bytes_median archive_ratio_mean archive_ratio_median entropy_bpb_mean entropy_bpb_median verified_all -h ctw expert ctw h:ctw 4096 3 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.04 0 0.04 0.04 0.04 0.0333333333333 0 0.09765625 0.09765625 13352 89.3532316148 13380 13252 13424 2.56549624542 2.56549624542 1 -h ctw expert ctw h:ctw 16384 3 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.18 0 0.18 0.18 0.18 0.166666666667 0.01 0.0868055555556 0.0868055555556 40962.6666667 72.7002980278 40940 40904 41044 3.20340951209 3.20340951209 1 -h ctw expert ctw h:ctw 65536 3 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.783333333333 0.0057735026919 0.78 0.78 0.79 0.756666666667 0.0266666666667 0.0797901114357 0.0801282051282 92328 135.233132035 92256 92244 92484 2.77213895634 2.77213895634 1 -h ctw expert ctw h:ctw 262144 3 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 3.32 0.01 3.32 3.31 3.33 3.23 0.0766666666667 0.0753016602669 0.0753012048193 221872 14.4222051019 221876 221856 221884 2.44096714152 2.44096714152 1 -h ctw expert ctw h:ctw 1048576 3 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 14.6266666667 0.0057735026919 14.63 14.62 14.63 14.3766666667 0.233333333333 0.0683682842226 0.0683526999316 552148 107.554637278 552204 552024 552216 2.30174274755 2.30174274755 1 -h ctw expert ctw h:ctw 2097152 3 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 30.88 0.0435889894354 30.9 30.83 30.91 30.48 0.366666666667 0.0647669254765 0.0647249190939 897320 83.809307359 897340 897228 897392 2.27731545661 2.27731545661 1 -h ctw expert ctw h:ctw 4194304 3 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 65.37 0.02 65.37 65.35 65.39 64.6866666667 0.62 0.0611901522046 0.0611901483861 1472909.33333 42.7707064863 1472932 1472860 1472936 2.24674930722 2.24674930722 1 -h ctw expert ctw h:ctw 10000000 3 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 167.303333333 0.198578280115 167.37 167.08 167.46 165.95 1.19666666667 0.0570027622034 0.0569800033702 2773780 58.9236794506 2773768 2773728 2773844 2.19747398481 2.19747398481 1 -h match expert match h:match 4096 3 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0 0 0 0 0 0 0 5657.33333333 113.513582154 5636 5556 5780 5.57734993252 5.57734993252 1 -h match expert match h:match 16384 3 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0 0 0 0 0 0 0 5670.66666667 109.276407945 5640 5580 5792 6.58586286407 6.58586286407 1 -h match expert match h:match 65536 3 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0 0 0 0 0 0 0 6469.33333333 48.8808074129 6480 6416 6512 6.5588255305 6.5588255305 1 -h match expert match h:match 262144 3 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 0.03 0 0.03 0.03 0.03 0.03 0 8.33333333333 8.33333333333 7980 110.054531938 7984 7868 8088 6.29225944669 6.29225944669 1 -h match expert match h:match 1048576 3 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 0.13 0 0.13 0.13 0.13 0.13 0 7.69230769231 7.69230769231 11332 86.8101376568 11288 11276 11432 6.23692635511 6.23692635511 1 -h match expert match h:match 2097152 3 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 0.266666666667 0.0057735026919 0.27 0.26 0.27 0.256666666667 0 7.50237416904 7.40740740741 19205.3333333 122.659420076 19244 19068 19304 6.2741772649 6.2741772649 1 -h match expert match h:match 4194304 3 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 0.553333333333 0.0208166599947 0.56 0.53 0.57 0.54 0.00333333333333 7.23585693794 7.14285714286 21197.3333333 22.0302821891 21208 21172 21212 6.28038800899 6.28038800899 1 -h match expert match h:match 10000000 3 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 1.31333333333 0.0115470053838 1.32 1.3 1.32 1.29666666667 0.01 7.26185571156 7.22480542732 40428 127.812362469 40384 40328 40572 6.28744436593 6.28744436593 1 -h neural_mixture mixture neural-mixture h:neural_mixture 4096 3 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.2 0 0.2 0.2 0.2 0.196666666667 0 0.01953125 0.01953125 18869.3333333 50.6491197686 18860 18824 18924 1.8841161677 1.8841161677 1 -h neural_mixture mixture neural-mixture h:neural_mixture 16384 3 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.876666666667 0.0115470053838 0.87 0.87 0.89 0.856666666667 0.0166666666667 0.0178252400017 0.0179597701149 59341.3333333 38.0175398117 59340 59304 59380 2.6715395598 2.6715395598 1 -h neural_mixture mixture neural-mixture h:neural_mixture 65536 3 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 3.56333333333 0.0057735026919 3.56 3.56 3.57 3.50333333333 0.05 0.0175397874506 0.0175561797753 164522.666667 112.023806994 164460 164456 164652 2.35267958216 2.35267958216 1 -h neural_mixture mixture neural-mixture h:neural_mixture 262144 3 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 15.1866666667 0.0550757054729 15.16 15.15 15.25 14.9933333333 0.17 0.0164619526532 0.0164907651715 478552 17.4355957742 478544 478540 478572 2.05460030125 2.05460030125 1 -h neural_mixture mixture neural-mixture h:neural_mixture 1048576 3 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 65.1766666667 0.0404145188433 65.17 65.14 65.22 64.58 0.536666666667 0.0153429180631 0.0153444836581 1188004 152.630272227 1187924 1187908 1188180 1.93872008703 1.93872008703 1 -h neural_mixture mixture neural-mixture h:neural_mixture 2097152 3 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 135.373333333 0.350761077278 135.4 135.01 135.71 134.416666667 0.83 0.0147740245803 0.0147710487445 1841026.66667 108.836268465 1840972 1840956 1841152 1.91993009037 1.91993009037 1 -h neural_mixture mixture neural-mixture h:neural_mixture 4194304 3 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 286.243333333 0.467368519836 286.12 285.85 286.76 284.773333333 1.21666666667 0.0139741494004 0.0139801481896 2780197.33333 76.3500709452 2780168 2780140 2780284 1.88464660438 1.88464660438 1 -h neural_mixture mixture neural-mixture h:neural_mixture 10000000 3 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 923.666666667 0.341516227042 923.59 923.37 924.04 920.38 2.46 0.0103248762538 0.0103257323748 5084989.33333 168.586278603 5084928 5084860 5085180 1.81642239361 1.81642239361 1 -h ppmd expert ppmd h:ppmd 4096 3 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0 0 0 0 0 0 0 7304 34.1760149813 7300 7272 7340 2.02915198126 2.02915198126 1 -h ppmd expert ppmd h:ppmd 16384 3 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.03 0 0.03 0.03 0.03 0.0233333333333 0 0.520833333333 0.520833333333 15921.3333333 22.0302821891 15932 15896 15936 3.05111195443 3.05111195443 1 -h ppmd expert ppmd h:ppmd 65536 3 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.17 0 0.17 0.17 0.17 0.15 0.0133333333333 0.367647058824 0.367647058824 45038.6666667 102.943350117 45092 44920 45104 2.82799347281 2.82799347281 1 -h ppmd expert ppmd h:ppmd 262144 3 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 0.843333333333 0.0152752523165 0.84 0.83 0.86 0.783333333333 0.0566666666667 0.296507180438 0.297619047619 143953.333333 78.9261257971 143916 143900 144044 2.54058234497 2.54058234497 1 -h ppmd expert ppmd h:ppmd 1048576 3 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 3.57 0.0264575131106 3.56 3.55 3.6 3.38333333333 0.18 0.280122265009 0.280898876404 398808 43.2666153056 398820 398760 398844 2.489549756 2.489549756 1 -h ppmd expert ppmd h:ppmd 2097152 3 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 6.88 0.02 6.88 6.86 6.9 6.64 0.226666666667 0.290699312129 0.290697674419 456989.333333 93.8367376529 456968 456908 457092 2.52506560801 2.52506560801 1 -h ppmd expert ppmd h:ppmd 4194304 3 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 13.72 0 13.72 13.72 13.72 13.46 0.243333333333 0.291545189504 0.291545189504 459152 24.3310501212 459140 459136 459180 2.53530592864 2.53530592864 1 -h ppmd expert ppmd h:ppmd 10000000 3 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 33.4333333333 0.0404145188433 33.41 33.41 33.48 33.1133333333 0.286666666667 0.285246832934 0.285445769652 557269.333333 80.8290376865 557316 557176 557316 2.5235430657 2.5235430657 1 -h rosa expert rosaplus h:rosa 4096 3 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.02 0 0.02 0.02 0.02 0.02 0 0.1953125 0.1953125 5926.66666667 151.490373732 5880 5804 6096 2.11172124286 2.11172124286 1 -h rosa expert rosaplus h:rosa 16384 3 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.12 0 0.12 0.12 0.12 0.106666666667 0.00333333333333 0.130208333333 0.130208333333 8074.66666667 157.293780339 8020 7952 8252 3.48630622986 3.48630622986 1 -h rosa expert rosaplus h:rosa 65536 3 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.6 0 0.6 0.6 0.6 0.576666666667 0.02 0.104166666667 0.104166666667 16733.3333333 78.621455935 16764 16644 16792 3.19056696201 3.19056696201 1 -h rosa expert rosaplus h:rosa 262144 3 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 3.51333333333 0.0057735026919 3.51 3.51 3.52 3.41333333333 0.0933333333333 0.071157623241 0.0712250712251 46994.6666667 96.0277737602 46972 46912 47100 2.90567254487 2.90567254487 1 -h rosa expert rosaplus h:rosa 1048576 3 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 20.9033333333 0.284487843911 20.77 20.71 21.23 20.1766666667 0.696666666667 0.0478451243687 0.0481463649494 203784 156.614175604 203800 203620 203932 2.80123027825 2.80123027825 1 -h rosa expert rosaplus h:rosa 2097152 3 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 49.6933333333 0.256969518296 49.55 49.54 49.99 47.67 1.96333333333 0.0402475626873 0.0403632694248 337494.666667 129.573659875 337428 337412 337644 2.70437138702 2.70437138702 1 -h rosa expert rosaplus h:rosa 4194304 3 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 119.193333333 0.0642910050733 119.22 119.12 119.24 113.823333333 5.23333333333 0.033558930388 0.0335514175474 643501.333333 130.986004341 643460 643396 643648 2.62099230176 2.62099230176 1 -h rosa expert rosaplus h:rosa 10000000 3 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 350.966666667 0.260256283946 350.98 350.7 351.22 337.113333333 13.4966666667 0.0271727989182 0.0271717566929 1551558.66667 145.894939368 1551520 1551436 1551720 2.48481897791 2.48481897791 1 -h rwkv expert rwkv h:rwkv 4096 3 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.08 0 0.08 0.08 0.08 0.08 0 0.048828125 0.048828125 7630.66666667 88.7543426168 7656 7532 7704 4.58837588813 4.58837588813 1 -h rwkv expert rwkv h:rwkv 16384 3 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.31 0 0.31 0.31 0.31 0.31 0 0.0504032258065 0.0504032258065 7622.66666667 64.6632301492 7660 7548 7660 4.92164750489 4.92164750489 1 -h rwkv expert rwkv h:rwkv 65536 3 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 1.25333333333 0.0057735026919 1.25 1.25 1.26 1.25 0 0.0498677248677 0.05 7781.33333333 123.698558332 7816 7644 7884 3.94457012734 3.94457012734 1 -h rwkv expert rwkv h:rwkv 262144 3 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 5.03 0.02 5.03 5.01 5.05 5.02 0 0.0497023131201 0.0497017892644 7650.66666667 109.276407945 7620 7560 7772 3.68331047311 3.68331047311 1 -h rwkv expert rwkv h:rwkv 1048576 3 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 20.1866666667 0.0907377172588 20.15 20.12 20.29 20.1733333333 0 0.0495383143584 0.0496277915633 8317.33333333 99.1429943734 8360 8204 8388 3.62635365568 3.62635365568 1 -h rwkv expert rwkv h:rwkv 2097152 3 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 40.34 0.718957578721 39.94 39.91 41.17 40.3133333333 0 0.049588976263 0.050075112669 9345.33333333 40.8574758561 9328 9316 9392 3.64949816345 3.64949816345 1 -h rwkv expert rwkv h:rwkv 4194304 3 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 86.2666666667 0.235867194271 86.29 86.02 86.49 86.2133333333 0 0.046368082804 0.0463553134778 11318.6666667 77.3907832583 11276 11272 11408 3.70207627028 3.70207627028 1 -h rwkv expert rwkv h:rwkv 10000000 3 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 410.166666667 0.767094083756 410.01 409.49 411 409.903333333 0 0.0232509517824 0.0232597818689 17169.3333333 83.1705075933 17136 17108 17264 3.76255426699 3.76255426699 1 -compress ctw expert ctw compress:ctw 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.05 0 0.05 0.05 0.05 0.0433333333333 0 0.078125 0.078125 13493.3333333 16.6533279957 13488 13480 13512 1332 1332 0.3251953125 0.3251953125 1 -compress ctw expert ctw compress:ctw 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.24 0 0.24 0.24 0.24 0.223333333333 0.01 0.0651041666667 0.0651041666667 41070.6666667 114.844822841 41100 40944 41168 6579 6579 0.401550292969 0.401550292969 1 -compress ctw expert ctw compress:ctw 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 1.02333333333 0.0057735026919 1.02 1.02 1.03 0.996666666667 0.02 0.0610762104194 0.0612745098039 92374.6666667 68.1566822354 92412 92296 92416 22728 22728 0.346801757812 0.346801757812 1 -compress ctw expert ctw compress:ctw 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 4.27666666667 0.0208166599947 4.27 4.26 4.3 4.19666666667 0.07 0.0584576634203 0.0585480093677 221977.333333 224.582575756 221996 221744 222192 80004 80004 0.305191040039 0.305191040039 1 -compress ctw expert ctw compress:ctw 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 18.29 0.05 18.29 18.24 18.34 18.0433333333 0.226666666667 0.0546749580229 0.0546746856206 552529.333333 193.052669843 552580 552316 552692 301713 301713 0.287735939026 0.287735939026 1 -compress ctw expert ctw compress:ctw 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 38.0366666667 0.142243921956 37.97 37.94 38.2 37.6433333333 0.356666666667 0.0525813322761 0.0526731630234 898530.666667 33.5459883344 898548 898492 898552 597003 597003 0.284673213959 0.284673213959 1 -compress ctw expert ctw compress:ctw 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 78.5666666667 0.111504857891 78.61 78.44 78.65 77.91 0.586666666667 0.0509122449099 0.0508841114362 1475328 35.5527776693 1475316 1475300 1475368 1177962 1177962 0.280848026276 0.280848026276 1 -compress ctw expert ctw compress:ctw 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 199.09 0.170587221092 199.04 198.95 199.28 197.626666667 1.28333333333 0.0479016918487 0.0479137015879 2779064 83.1384387633 2779016 2779016 2779160 2746861 2746861 0.2746861 0.2746861 1 -compress match expert match compress:match 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0 0 0 0 0 0 0 5346.66666667 211.521472511 5444 5104 5492 2874 2874 0.70166015625 0.70166015625 1 -compress match expert match compress:match 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.01 0 0.01 0.01 0.01 0.01 0 1.5625 1.5625 5422.66666667 105.096780794 5460 5304 5504 13506 13506 0.824340820312 0.824340820312 1 -compress match expert match compress:match 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.05 0 0.05 0.05 0.05 0.05 0 1.25 1.25 6277.33333333 104.332800851 6252 6188 6392 53748 53748 0.820129394531 0.820129394531 1 -compress match expert match compress:match 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 0.2 0 0.2 0.2 0.2 0.196666666667 0 1.25 1.25 8057.33333333 10.0664459137 8056 8048 8068 206203 206203 0.786602020264 0.786602020264 1 -compress match expert match compress:match 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 0.82 0.01 0.82 0.81 0.83 0.82 0 1.21963312449 1.21951219512 12077.3333333 111.880889044 12036 11992 12204 817505 817505 0.779633522034 0.779633522034 1 -compress match expert match compress:match 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 1.61 0.01 1.61 1.6 1.62 1.6 0 1.24226797536 1.24223602484 22096 86.6256313108 22048 22044 22196 1644756 1644756 0.784280776978 0.784280776978 1 -compress match expert match compress:match 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 3.24333333333 0.0115470053838 3.25 3.23 3.25 3.23 0.00666666666667 1.23330951814 1.23076923077 28921.3333333 67.448745973 28952 28844 28968 3292751 3292751 0.785053014755 0.785053014755 1 -compress match expert match compress:match 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 7.75666666667 0.0404145188433 7.75 7.72 7.8 7.73666666667 0.01 1.22951210348 1.23054750504 55398.6666667 22.0302821891 55388 55384 55424 7859324 7859324 0.7859324 0.7859324 1 -compress neural_mixture mixture neural-mixture compress:neural_mixture 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.163333333333 0.0057735026919 0.16 0.16 0.17 0.16 0 0.0239353553922 0.0244140625 18694.6666667 138.120720145 18760 18536 18788 983 983 0.239990234375 0.239990234375 1 -compress neural_mixture mixture neural-mixture compress:neural_mixture 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.736666666667 0.0152752523165 0.74 0.72 0.75 0.713333333333 0.0133333333333 0.021216529029 0.0211148648649 59294.6666667 197.274766084 59336 59080 59468 5490 5490 0.335083007812 0.335083007812 1 -compress neural_mixture mixture neural-mixture compress:neural_mixture 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 2.98 0 2.98 2.98 2.98 2.91666666667 0.0533333333333 0.0209731543624 0.0209731543624 157646.666667 40.8574758561 157664 157600 157676 19292 19292 0.294372558594 0.294372558594 1 -compress neural_mixture mixture neural-mixture compress:neural_mixture 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 12.8933333333 0.0057735026919 12.89 12.89 12.9 12.7 0.173333333333 0.0193898681549 0.0193948797517 479310.666667 141.213785918 479332 479160 479440 67344 67344 0.256896972656 0.256896972656 1 -compress neural_mixture mixture neural-mixture compress:neural_mixture 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 56.4 0.03 56.4 56.37 56.43 55.81 0.53 0.0177304997983 0.0177304964539 1190862.66667 79.4313120459 1190896 1190772 1190920 254132 254132 0.242359161377 0.242359161377 1 -compress neural_mixture mixture neural-mixture compress:neural_mixture 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 117.83 0.31224989992 117.93 117.48 118.08 116.87 0.846666666667 0.0169736855989 0.0169592130925 1837317.33333 56.1901533485 1837312 1837264 1837376 503320 503320 0.240001678467 0.240001678467 1 -compress neural_mixture mixture neural-mixture compress:neural_mixture 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 254.493333333 0.185831464864 254.58 254.28 254.62 253.04 1.21666666667 0.0157175096496 0.0157121533506 2799441.33333 10.0664459137 2799440 2799432 2799452 991873 991873 0.236480951309 0.236480951309 1 -compress neural_mixture mixture neural-mixture compress:neural_mixture 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 835.173333333 1.87324139751 834.33 833.87 837.32 832.083333333 2.33666666667 0.0114189171633 0.0114304210133 5061216 93.5521245082 5061268 5061108 5061272 2272630 2272630 0.227263 0.227263 1 -compress ppmd expert ppmd compress:ppmd 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.01 0 0.01 0.01 0.01 0.00666666666667 0 0.390625 0.390625 7044 274.021896935 6900 6872 7360 1058 1058 0.25830078125 0.25830078125 1 -compress ppmd expert ppmd compress:ppmd 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.04 0 0.04 0.04 0.04 0.0266666666667 0.00333333333333 0.390625 0.390625 16032 162.923294835 16012 15880 16204 6267 6267 0.382507324219 0.382507324219 1 -compress ppmd expert ppmd compress:ppmd 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.21 0 0.21 0.21 0.21 0.193333333333 0.0133333333333 0.297619047619 0.297619047619 44961.3333333 263.949490118 45052 44664 45168 23186 23186 0.353790283203 0.353790283203 1 -compress ppmd expert ppmd compress:ppmd 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 1.00333333333 0.0057735026919 1 1 1.01 0.923333333333 0.0733333333333 0.249174917492 0.25 144137.333333 88.4835201229 144148 144044 144220 83269 83269 0.317646026611 0.317646026611 1 -compress ppmd expert ppmd compress:ppmd 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 4.21 0 4.21 4.21 4.21 4.01 0.19 0.237529691211 0.237529691211 399285.333333 90.0074071026 399248 399220 399388 326331 326331 0.311213493347 0.311213493347 1 -compress ppmd expert ppmd compress:ppmd 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 8.28333333333 0.0750555349947 8.28 8.21 8.36 8.04333333333 0.223333333333 0.241461900933 0.24154589372 457633.333333 182.486529183 457532 457524 457844 661953 661953 0.315643787384 0.315643787384 1 -compress ppmd expert ppmd compress:ppmd 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 16.41 0.0346410161514 16.39 16.39 16.45 16.1666666667 0.226666666667 0.243754531917 0.244051250763 459718.666667 60.5750223552 459692 459676 459788 1329257 1329257 0.316919565201 0.316919565201 1 -compress ppmd expert ppmd compress:ppmd 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 39.86 0.0435889894354 39.88 39.81 39.89 39.5433333333 0.273333333333 0.239256165872 0.239135987063 568756 154.970965022 568816 568580 568872 3154465 3154465 0.3154465 0.3154465 1 -compress rosa expert rosaplus compress:rosa 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0 0 0 0 0 0 0 5684 246.933189345 5804 5400 5848 1127 1127 0.275146484375 0.275146484375 1 -compress rosa expert rosaplus compress:rosa 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.03 0 0.03 0.03 0.03 0.02 0 0.520833333333 0.520833333333 8556 81.6823114266 8528 8492 8648 6359 6359 0.388122558594 0.388122558594 1 -compress rosa expert rosaplus compress:rosa 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.15 0 0.15 0.15 0.15 0.143333333333 0.00333333333333 0.416666666667 0.416666666667 19252 65.482822175 19268 19180 19308 22843 22843 0.348556518555 0.348556518555 1 -compress rosa expert rosaplus compress:rosa 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 0.85 0 0.85 0.85 0.85 0.833333333333 0.01 0.294117647059 0.294117647059 61973.3333333 48.0555234425 61976 61924 62020 80590 80590 0.307426452637 0.307426452637 1 -compress rosa expert rosaplus compress:rosa 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 4.76666666667 0.0152752523165 4.77 4.75 4.78 4.69666666667 0.0633333333333 0.209791647527 0.20964360587 172204 49.1528229098 172224 172148 172240 306522 306522 0.292322158813 0.292322158813 1 -compress rosa expert rosaplus compress:rosa 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 11.5466666667 0.204287379281 11.46 11.4 11.78 11.3866666667 0.146666666667 0.173245984409 0.174520069808 337397.333333 32.083225108 337400 337364 337428 608681 608681 0.290241718292 0.290241718292 1 -compress rosa expert rosaplus compress:rosa 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 27.2766666667 0.119303534454 27.33 27.14 27.36 26.9466666667 0.3 0.146647359224 0.146359312111 669366.666667 62.0107517559 669392 669296 669412 1199345 1199345 0.285946130753 0.285946130753 1 -compress rosa expert rosaplus compress:rosa 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 80.5633333333 0.310214979221 80.55 80.26 80.88 79.76 0.733333333333 0.118376896964 0.118395321714 1556642.66667 52.2047252012 1556648 1556588 1556692 2752778 2752778 0.2752778 0.2752778 1 -compress rwkv expert rwkv compress:rwkv 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.0633333333333 0.0057735026919 0.06 0.06 0.07 0.06 0 0.062003968254 0.0651041666667 6570.66666667 81.0267445559 6596 6480 6636 2368 2368 0.578125 0.578125 1 -compress rwkv expert rwkv compress:rwkv 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.253333333333 0.0057735026919 0.25 0.25 0.26 0.253333333333 0 0.0616987179487 0.0625 6613.33333333 127.582652948 6648 6472 6720 10098 10098 0.616333007812 0.616333007812 1 -compress rwkv expert rwkv compress:rwkv 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 1.02 0 1.02 1.02 1.02 1.02 0 0.0612745098039 0.0612745098039 6748 112.853887837 6764 6628 6852 32333 32333 0.493362426758 0.493362426758 1 -compress rwkv expert rwkv compress:rwkv 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 4.08333333333 0.0251661147842 4.08 4.06 4.11 4.07666666667 0 0.061226038364 0.0612745098039 7014.66666667 20.5264057578 7020 6992 7032 120713 120713 0.460483551025 0.460483551025 1 -compress rwkv expert rwkv compress:rwkv 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 16.45 0.0346410161514 16.47 16.41 16.47 16.4333333333 0 0.0607904534938 0.0607164541591 8654.66666667 54.3077649451 8684 8592 8688 475332 475332 0.453311920166 0.453311920166 1 -compress rwkv expert rwkv compress:rwkv 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 37.1566666667 0.255016339346 37.16 36.9 37.41 37.13 0 0.0538278321726 0.05382131324 10502.6666667 117.189305542 10456 10416 10636 956112 956112 0.455909729004 0.455909729004 1 -compress rwkv expert rwkv compress:rwkv 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 72.72 0.286879765756 72.86 72.39 72.91 72.67 0.00333333333333 0.0550060725133 0.0548998078507 17928 110.489818535 17892 17840 18052 3599584 3599584 0.858207702637 0.858207702637 1 -compress rwkv expert rwkv compress:rwkv 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 360.813333333 0.453908948285 361.05 360.29 361.1 360.573333333 0.0133333333333 0.0264312661207 0.0264139126549 25844 86.0697391654 25840 25760 25932 4788130 4788130 0.478813 0.478813 1 -decompress ctw expert ctw decompress:ctw 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.05 0 0.05 0.05 0.05 0.0466666666667 0 0.078125 0.078125 13437.3333333 64.0416531121 13440 13372 13500 1332 1332 0.3251953125 0.3251953125 1 -decompress ctw expert ctw decompress:ctw 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.233333333333 0.0057735026919 0.23 0.23 0.24 0.22 0.0133333333333 0.0669912439614 0.0679347826087 40985.3333333 188.226813534 40996 40792 41168 6579 6579 0.401550292969 0.401550292969 1 -decompress ctw expert ctw decompress:ctw 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 1.02333333333 0.0057735026919 1.02 1.02 1.03 0.99 0.0233333333333 0.0610762104194 0.0612745098039 92397.3333333 80.133222407 92392 92320 92480 22728 22728 0.346801757812 0.346801757812 1 -decompress ctw expert ctw decompress:ctw 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 4.28666666667 0.0057735026919 4.29 4.28 4.29 4.21 0.0633333333333 0.0583204438345 0.0582750582751 222077.333333 59.3745175419 222092 222012 222128 80004 80004 0.305191040039 0.305191040039 1 -decompress ctw expert ctw decompress:ctw 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 18.3433333333 0.0057735026919 18.34 18.34 18.35 18.0866666667 0.236666666667 0.0545157222987 0.0545256270447 552432 94.5727233403 552392 552364 552540 301713 301713 0.287735939026 0.287735939026 1 -decompress ctw expert ctw decompress:ctw 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 38.08 0.0264575131106 38.09 38.05 38.1 37.6533333333 0.386666666667 0.0525210253114 0.0525072197427 897984 52 897956 897952 898044 597003 597003 0.284673213959 0.284673213959 1 -decompress ctw expert ctw decompress:ctw 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 79.1066666667 0.254820198048 78.98 78.94 79.4 78.42 0.613333333333 0.0505649876157 0.050645733097 1474340 84.2852300228 1474348 1474252 1474420 1177962 1177962 0.280848026276 0.280848026276 1 -decompress ctw expert ctw decompress:ctw 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 199.673333333 0.132790561914 199.75 199.52 199.75 198.316666667 1.18333333333 0.0477617407285 0.0477433950641 2776528 49.9599839872 2776512 2776488 2776584 2746861 2746861 0.2746861 0.2746861 1 -decompress match expert match decompress:match 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0 0 0 0 0 0 0 5361.33333333 88.7543426168 5348 5280 5456 2874 2874 0.70166015625 0.70166015625 1 -decompress match expert match decompress:match 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.01 0 0.01 0.01 0.01 0.01 0 1.5625 1.5625 5436 98.0612053771 5440 5336 5532 13506 13506 0.824340820312 0.824340820312 1 -decompress match expert match decompress:match 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.05 0 0.05 0.05 0.05 0.05 0 1.25 1.25 6380 115.723809132 6428 6248 6464 53748 53748 0.820129394531 0.820129394531 1 -decompress match expert match decompress:match 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 0.2 0 0.2 0.2 0.2 0.2 0 1.25 1.25 7877.33333333 75.0821772016 7892 7796 7944 206203 206203 0.786602020264 0.786602020264 1 -decompress match expert match decompress:match 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 0.853333333333 0.0115470053838 0.86 0.84 0.86 0.846666666667 0 1.17201919528 1.16279069767 11396 66.813172354 11384 11336 11468 817505 817505 0.779633522034 0.779633522034 1 -decompress match expert match decompress:match 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 1.67666666667 0.0152752523165 1.68 1.66 1.69 1.67 0 1.19290914008 1.19047619048 20069.3333333 88.9344327768 20020 20016 20172 1644756 1644756 0.784280776978 0.784280776978 1 -decompress match expert match decompress:match 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 3.34666666667 0.0251661147842 3.35 3.32 3.37 3.33666666667 0.00333333333333 1.19526424934 1.19402985075 23890.6666667 54.3077649451 23920 23828 23924 3292751 3292751 0.785053014755 0.785053014755 1 -decompress match expert match decompress:match 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 8.01666666667 0.0550757054729 7.99 7.98 8.08 7.99 0.0166666666667 1.18965182305 1.1935848766 45941.3333333 161.013456995 45880 45820 46124 7859324 7859324 0.7859324 0.7859324 1 -decompress neural_mixture mixture neural-mixture decompress:neural_mixture 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.16 0 0.16 0.16 0.16 0.156666666667 0 0.0244140625 0.0244140625 18765.3333333 92.2893998969 18804 18660 18832 983 983 0.239990234375 0.239990234375 1 -decompress neural_mixture mixture neural-mixture decompress:neural_mixture 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.75 0.01 0.75 0.74 0.76 0.723333333333 0.02 0.0208358029082 0.0208333333333 58865.3333333 86.0077515886 58844 58792 58960 5490 5490 0.335083007812 0.335083007812 1 -decompress neural_mixture mixture neural-mixture decompress:neural_mixture 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 2.97 0 2.97 2.97 2.97 2.90333333333 0.06 0.0210437710438 0.0210437710438 156965.333333 98.7387124351 156992 156856 157048 19292 19292 0.294372558594 0.294372558594 1 -decompress neural_mixture mixture neural-mixture decompress:neural_mixture 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 12.9166666667 0.0115470053838 12.91 12.91 12.93 12.7166666667 0.183333333333 0.0193548490162 0.0193648334624 479588 58.9236794506 479600 479524 479640 67344 67344 0.256896972656 0.256896972656 1 -decompress neural_mixture mixture neural-mixture decompress:neural_mixture 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 56.3733333333 0.0750555349947 56.37 56.3 56.45 55.7866666667 0.53 0.0177389045941 0.0177399325883 1182824 90.0666419936 1182828 1182732 1182912 254132 254132 0.242359161377 0.242359161377 1 -decompress neural_mixture mixture neural-mixture decompress:neural_mixture 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 118.15 0.276224546339 118.06 117.93 118.46 117.25 0.783333333333 0.016927695983 0.0169405387091 1835526.66667 93.7514444333 1835564 1835420 1835596 503320 503320 0.240001678467 0.240001678467 1 -decompress neural_mixture mixture neural-mixture decompress:neural_mixture 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 254.33 0.298161030318 254.41 254 254.58 252.84 1.26 0.0157276124194 0.0157226524115 2777461.33333 42.7707064863 2777452 2777424 2777508 991873 991873 0.236480951309 0.236480951309 1 -decompress neural_mixture mixture neural-mixture decompress:neural_mixture 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 837.243333333 0.50013331556 837.23 836.75 837.75 834.096666667 2.38666666667 0.0113906495914 0.0113908282838 5041721.33333 92.1158690636 5041744 5041620 5041800 2272630 2272630 0.227263 0.227263 1 -decompress ppmd expert ppmd decompress:ppmd 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.01 0 0.01 0.01 0.01 0.00333333333333 0 0.390625 0.390625 7198.66666667 163.23398339 7236 7020 7340 1058 1058 0.25830078125 0.25830078125 1 -decompress ppmd expert ppmd decompress:ppmd 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.04 0 0.04 0.04 0.04 0.04 0 0.390625 0.390625 16117.3333333 88.4835201229 16128 16024 16200 6267 6267 0.382507324219 0.382507324219 1 -decompress ppmd expert ppmd decompress:ppmd 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.213333333333 0.0057735026919 0.21 0.21 0.22 0.196666666667 0.0133333333333 0.29310966811 0.297619047619 45034.6666667 98.1699207157 44980 44976 45148 23186 23186 0.353790283203 0.353790283203 1 -decompress ppmd expert ppmd decompress:ppmd 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 1.01333333333 0.0057735026919 1.01 1.01 1.02 0.946666666667 0.06 0.246715848055 0.247524752475 143966.666667 91.2432645916 143916 143912 144072 83269 83269 0.317646026611 0.317646026611 1 -decompress ppmd expert ppmd decompress:ppmd 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 4.23666666667 0.0115470053838 4.23 4.23 4.25 4.02333333333 0.203333333333 0.236035785473 0.236406619385 399052 121.786698781 399076 398920 399160 326331 326331 0.311213493347 0.311213493347 1 -decompress ppmd expert ppmd decompress:ppmd 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 8.29333333333 0.057735026919 8.26 8.26 8.36 8.08333333333 0.196666666667 0.24116531699 0.242130750605 457612 58.9236794506 457600 457560 457676 661953 661953 0.315643787384 0.315643787384 1 -decompress ppmd expert ppmd decompress:ppmd 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 16.5566666667 0.0635085296109 16.52 16.52 16.63 16.3133333333 0.22 0.241596888457 0.242130750605 458101.333333 224.582575756 458180 457848 458276 1329257 1329257 0.316919565201 0.316919565201 1 -decompress ppmd expert ppmd decompress:ppmd 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 40.2 0.0754983443527 40.21 40.12 40.27 39.89 0.263333333333 0.237232975058 0.237173418654 559082.666667 354.679197774 559244 558676 559328 3154465 3154465 0.3154465 0.3154465 1 -decompress rosa expert rosaplus decompress:rosa 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0 0 0 0 0 0 0 5866.66666667 65.0333247907 5844 5816 5940 1127 1127 0.275146484375 0.275146484375 1 -decompress rosa expert rosaplus decompress:rosa 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.03 0 0.03 0.03 0.03 0.0233333333333 0 0.520833333333 0.520833333333 8504 58.9236794506 8472 8468 8572 6359 6359 0.388122558594 0.388122558594 1 -decompress rosa expert rosaplus decompress:rosa 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.15 0 0.15 0.15 0.15 0.143333333333 0.00333333333333 0.416666666667 0.416666666667 19317.3333333 73.3575717519 19292 19260 19400 22843 22843 0.348556518555 0.348556518555 1 -decompress rosa expert rosaplus decompress:rosa 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 0.846666666667 0.0115470053838 0.84 0.84 0.86 0.83 0.01 0.295311923219 0.297619047619 61872 66.0908465674 61904 61796 61916 80590 80590 0.307426452637 0.307426452637 1 -decompress rosa expert rosaplus decompress:rosa 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 4.78333333333 0.0057735026919 4.78 4.78 4.79 4.71 0.0633333333333 0.209059436355 0.209205020921 172616 115.377640815 172584 172520 172744 306522 306522 0.292322158813 0.292322158813 1 -decompress rosa expert rosaplus decompress:rosa 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 11.4333333333 0.0115470053838 11.44 11.42 11.44 11.2966666667 0.123333333333 0.174927232721 0.174825174825 338036 106.056588669 338060 337920 338128 608681 608681 0.290241718292 0.290241718292 1 -decompress rosa expert rosaplus decompress:rosa 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 27.31 0.0556776436283 27.32 27.25 27.36 27.0033333333 0.276666666667 0.146466901856 0.146412884334 658334.666667 71.7030915187 658372 658252 658380 1199345 1199345 0.285946130753 0.285946130753 1 -decompress rosa expert rosaplus decompress:rosa 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 80.5866666667 0.0503322295685 80.58 80.54 80.64 79.81 0.706666666667 0.118341483007 0.118351243039 1554957.33333 138.236512302 1554948 1554824 1555100 2752778 2752778 0.2752778 0.2752778 1 -decompress rwkv expert rwkv decompress:rwkv 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.0666666666667 0.0057735026919 0.07 0.06 0.07 0.0633333333333 0 0.0589037698413 0.0558035714286 6648 61.5792172734 6632 6596 6716 2368 2368 0.578125 0.578125 1 -decompress rwkv expert rwkv decompress:rwkv 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.256666666667 0.0057735026919 0.26 0.25 0.26 0.256666666667 0 0.0608974358974 0.0600961538462 6530.66666667 44.9592408002 6520 6492 6580 10098 10098 0.616333007812 0.616333007812 1 -decompress rwkv expert rwkv decompress:rwkv 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 1.03 0 1.03 1.03 1.03 1.03 0 0.0606796116505 0.0606796116505 6670.66666667 34.9475797922 6680 6632 6700 32333 32333 0.493362426758 0.493362426758 1 -decompress rwkv expert rwkv decompress:rwkv 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 4.11 0.0346410161514 4.09 4.09 4.15 4.10333333333 0 0.0608301175362 0.0611246943765 7001.33333333 114.705419808 7016 6880 7108 120713 120713 0.460483551025 0.460483551025 1 -decompress rwkv expert rwkv decompress:rwkv 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 16.29 0.0264575131106 16.28 16.27 16.32 16.2766666667 0 0.0613874620753 0.0614250614251 8069.33333333 28.9367125523 8084 8036 8088 475332 475332 0.453311920166 0.453311920166 1 -decompress rwkv expert rwkv decompress:rwkv 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 37.2566666667 0.136503968196 37.28 37.11 37.38 37.23 0 0.0536821485709 0.0536480686695 9566.66666667 105.248911317 9512 9500 9688 956112 956112 0.455909729004 0.455909729004 1 -decompress rwkv expert rwkv decompress:rwkv 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 72.8666666667 0.366924152017 72.91 72.48 73.21 72.8133333333 0 0.0548957138067 0.0548621588259 14153.3333333 34.0196021925 14140 14128 14192 3599584 3599584 0.858207702637 0.858207702637 1 -decompress rwkv expert rwkv decompress:rwkv 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 359.19 0.676683086829 358.84 358.76 359.97 358.96 0.00333333333333 0.0265507550454 0.0265765889089 21162.6666667 56.1901533485 21168 21104 21216 4788130 4788130 0.478813 0.478813 1 +operation subject subject_kind expert_kind series size_bytes repeats cpu compression_backend input_sha256 real_seconds_mean real_seconds_stdev real_seconds_median real_seconds_min real_seconds_max user_seconds_mean sys_seconds_mean throughput_mib_s_mean throughput_mib_s_median rss_kib_mean rss_kib_stdev rss_kib_median rss_kib_min rss_kib_max archive_bytes_mean archive_bytes_median archive_ratio_mean archive_ratio_median entropy_bpb_mean entropy_bpb_median verified_all +h ctw expert ctw h:ctw 4096 3 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.04 0 0.04 0.04 0.04 0.0333333333333 0 0.09765625 0.09765625 13352 89.3532316148 13380 13252 13424 2.56549624542 2.56549624542 1 +h ctw expert ctw h:ctw 16384 3 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.18 0 0.18 0.18 0.18 0.166666666667 0.01 0.0868055555556 0.0868055555556 40962.6666667 72.7002980278 40940 40904 41044 3.20340951209 3.20340951209 1 +h ctw expert ctw h:ctw 65536 3 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.783333333333 0.0057735026919 0.78 0.78 0.79 0.756666666667 0.0266666666667 0.0797901114357 0.0801282051282 92328 135.233132035 92256 92244 92484 2.77213895634 2.77213895634 1 +h ctw expert ctw h:ctw 262144 3 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 3.32 0.01 3.32 3.31 3.33 3.23 0.0766666666667 0.0753016602669 0.0753012048193 221872 14.4222051019 221876 221856 221884 2.44096714152 2.44096714152 1 +h ctw expert ctw h:ctw 1048576 3 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 14.6266666667 0.0057735026919 14.63 14.62 14.63 14.3766666667 0.233333333333 0.0683682842226 0.0683526999316 552148 107.554637278 552204 552024 552216 2.30174274755 2.30174274755 1 +h ctw expert ctw h:ctw 2097152 3 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 30.88 0.0435889894354 30.9 30.83 30.91 30.48 0.366666666667 0.0647669254765 0.0647249190939 897320 83.809307359 897340 897228 897392 2.27731545661 2.27731545661 1 +h ctw expert ctw h:ctw 4194304 3 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 65.37 0.02 65.37 65.35 65.39 64.6866666667 0.62 0.0611901522046 0.0611901483861 1472909.33333 42.7707064863 1472932 1472860 1472936 2.24674930722 2.24674930722 1 +h ctw expert ctw h:ctw 10000000 3 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 167.303333333 0.198578280115 167.37 167.08 167.46 165.95 1.19666666667 0.0570027622034 0.0569800033702 2773780 58.9236794506 2773768 2773728 2773844 2.19747398481 2.19747398481 1 +h match expert match h:match 4096 3 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0 0 0 0 0 0 0 5657.33333333 113.513582154 5636 5556 5780 5.57734993252 5.57734993252 1 +h match expert match h:match 16384 3 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0 0 0 0 0 0 0 5670.66666667 109.276407945 5640 5580 5792 6.58586286407 6.58586286407 1 +h match expert match h:match 65536 3 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0 0 0 0 0 0 0 6469.33333333 48.8808074129 6480 6416 6512 6.5588255305 6.5588255305 1 +h match expert match h:match 262144 3 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 0.03 0 0.03 0.03 0.03 0.03 0 8.33333333333 8.33333333333 7980 110.054531938 7984 7868 8088 6.29225944669 6.29225944669 1 +h match expert match h:match 1048576 3 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 0.13 0 0.13 0.13 0.13 0.13 0 7.69230769231 7.69230769231 11332 86.8101376568 11288 11276 11432 6.23692635511 6.23692635511 1 +h match expert match h:match 2097152 3 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 0.266666666667 0.0057735026919 0.27 0.26 0.27 0.256666666667 0 7.50237416904 7.40740740741 19205.3333333 122.659420076 19244 19068 19304 6.2741772649 6.2741772649 1 +h match expert match h:match 4194304 3 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 0.553333333333 0.0208166599947 0.56 0.53 0.57 0.54 0.00333333333333 7.23585693794 7.14285714286 21197.3333333 22.0302821891 21208 21172 21212 6.28038800899 6.28038800899 1 +h match expert match h:match 10000000 3 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 1.31333333333 0.0115470053838 1.32 1.3 1.32 1.29666666667 0.01 7.26185571156 7.22480542732 40428 127.812362469 40384 40328 40572 6.28744436593 6.28744436593 1 +h neural_mixture mixture neural-mixture h:neural_mixture 4096 3 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.2 0 0.2 0.2 0.2 0.196666666667 0 0.01953125 0.01953125 18869.3333333 50.6491197686 18860 18824 18924 1.8841161677 1.8841161677 1 +h neural_mixture mixture neural-mixture h:neural_mixture 16384 3 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.876666666667 0.0115470053838 0.87 0.87 0.89 0.856666666667 0.0166666666667 0.0178252400017 0.0179597701149 59341.3333333 38.0175398117 59340 59304 59380 2.6715395598 2.6715395598 1 +h neural_mixture mixture neural-mixture h:neural_mixture 65536 3 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 3.56333333333 0.0057735026919 3.56 3.56 3.57 3.50333333333 0.05 0.0175397874506 0.0175561797753 164522.666667 112.023806994 164460 164456 164652 2.35267958216 2.35267958216 1 +h neural_mixture mixture neural-mixture h:neural_mixture 262144 3 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 15.1866666667 0.0550757054729 15.16 15.15 15.25 14.9933333333 0.17 0.0164619526532 0.0164907651715 478552 17.4355957742 478544 478540 478572 2.05460030125 2.05460030125 1 +h neural_mixture mixture neural-mixture h:neural_mixture 1048576 3 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 65.1766666667 0.0404145188433 65.17 65.14 65.22 64.58 0.536666666667 0.0153429180631 0.0153444836581 1188004 152.630272227 1187924 1187908 1188180 1.93872008703 1.93872008703 1 +h neural_mixture mixture neural-mixture h:neural_mixture 2097152 3 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 135.373333333 0.350761077278 135.4 135.01 135.71 134.416666667 0.83 0.0147740245803 0.0147710487445 1841026.66667 108.836268465 1840972 1840956 1841152 1.91993009037 1.91993009037 1 +h neural_mixture mixture neural-mixture h:neural_mixture 4194304 3 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 286.243333333 0.467368519836 286.12 285.85 286.76 284.773333333 1.21666666667 0.0139741494004 0.0139801481896 2780197.33333 76.3500709452 2780168 2780140 2780284 1.88464660438 1.88464660438 1 +h neural_mixture mixture neural-mixture h:neural_mixture 10000000 3 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 923.666666667 0.341516227042 923.59 923.37 924.04 920.38 2.46 0.0103248762538 0.0103257323748 5084989.33333 168.586278603 5084928 5084860 5085180 1.81642239361 1.81642239361 1 +h ppmd expert ppmd h:ppmd 4096 3 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0 0 0 0 0 0 0 7304 34.1760149813 7300 7272 7340 2.02915198126 2.02915198126 1 +h ppmd expert ppmd h:ppmd 16384 3 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.03 0 0.03 0.03 0.03 0.0233333333333 0 0.520833333333 0.520833333333 15921.3333333 22.0302821891 15932 15896 15936 3.05111195443 3.05111195443 1 +h ppmd expert ppmd h:ppmd 65536 3 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.17 0 0.17 0.17 0.17 0.15 0.0133333333333 0.367647058824 0.367647058824 45038.6666667 102.943350117 45092 44920 45104 2.82799347281 2.82799347281 1 +h ppmd expert ppmd h:ppmd 262144 3 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 0.843333333333 0.0152752523165 0.84 0.83 0.86 0.783333333333 0.0566666666667 0.296507180438 0.297619047619 143953.333333 78.9261257971 143916 143900 144044 2.54058234497 2.54058234497 1 +h ppmd expert ppmd h:ppmd 1048576 3 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 3.57 0.0264575131106 3.56 3.55 3.6 3.38333333333 0.18 0.280122265009 0.280898876404 398808 43.2666153056 398820 398760 398844 2.489549756 2.489549756 1 +h ppmd expert ppmd h:ppmd 2097152 3 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 6.88 0.02 6.88 6.86 6.9 6.64 0.226666666667 0.290699312129 0.290697674419 456989.333333 93.8367376529 456968 456908 457092 2.52506560801 2.52506560801 1 +h ppmd expert ppmd h:ppmd 4194304 3 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 13.72 0 13.72 13.72 13.72 13.46 0.243333333333 0.291545189504 0.291545189504 459152 24.3310501212 459140 459136 459180 2.53530592864 2.53530592864 1 +h ppmd expert ppmd h:ppmd 10000000 3 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 33.4333333333 0.0404145188433 33.41 33.41 33.48 33.1133333333 0.286666666667 0.285246832934 0.285445769652 557269.333333 80.8290376865 557316 557176 557316 2.5235430657 2.5235430657 1 +h rosa expert rosaplus h:rosa 4096 3 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.02 0 0.02 0.02 0.02 0.02 0 0.1953125 0.1953125 5926.66666667 151.490373732 5880 5804 6096 2.11172124286 2.11172124286 1 +h rosa expert rosaplus h:rosa 16384 3 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.12 0 0.12 0.12 0.12 0.106666666667 0.00333333333333 0.130208333333 0.130208333333 8074.66666667 157.293780339 8020 7952 8252 3.48630622986 3.48630622986 1 +h rosa expert rosaplus h:rosa 65536 3 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.6 0 0.6 0.6 0.6 0.576666666667 0.02 0.104166666667 0.104166666667 16733.3333333 78.621455935 16764 16644 16792 3.19056696201 3.19056696201 1 +h rosa expert rosaplus h:rosa 262144 3 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 3.51333333333 0.0057735026919 3.51 3.51 3.52 3.41333333333 0.0933333333333 0.071157623241 0.0712250712251 46994.6666667 96.0277737602 46972 46912 47100 2.90567254487 2.90567254487 1 +h rosa expert rosaplus h:rosa 1048576 3 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 20.9033333333 0.284487843911 20.77 20.71 21.23 20.1766666667 0.696666666667 0.0478451243687 0.0481463649494 203784 156.614175604 203800 203620 203932 2.80123027825 2.80123027825 1 +h rosa expert rosaplus h:rosa 2097152 3 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 49.6933333333 0.256969518296 49.55 49.54 49.99 47.67 1.96333333333 0.0402475626873 0.0403632694248 337494.666667 129.573659875 337428 337412 337644 2.70437138702 2.70437138702 1 +h rosa expert rosaplus h:rosa 4194304 3 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 119.193333333 0.0642910050733 119.22 119.12 119.24 113.823333333 5.23333333333 0.033558930388 0.0335514175474 643501.333333 130.986004341 643460 643396 643648 2.62099230176 2.62099230176 1 +h rosa expert rosaplus h:rosa 10000000 3 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 350.966666667 0.260256283946 350.98 350.7 351.22 337.113333333 13.4966666667 0.0271727989182 0.0271717566929 1551558.66667 145.894939368 1551520 1551436 1551720 2.48481897791 2.48481897791 1 +h rwkv expert rwkv h:rwkv 4096 3 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.08 0 0.08 0.08 0.08 0.08 0 0.048828125 0.048828125 7630.66666667 88.7543426168 7656 7532 7704 4.58837588813 4.58837588813 1 +h rwkv expert rwkv h:rwkv 16384 3 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.31 0 0.31 0.31 0.31 0.31 0 0.0504032258065 0.0504032258065 7622.66666667 64.6632301492 7660 7548 7660 4.92164750489 4.92164750489 1 +h rwkv expert rwkv h:rwkv 65536 3 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 1.25333333333 0.0057735026919 1.25 1.25 1.26 1.25 0 0.0498677248677 0.05 7781.33333333 123.698558332 7816 7644 7884 3.94457012734 3.94457012734 1 +h rwkv expert rwkv h:rwkv 262144 3 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 5.03 0.02 5.03 5.01 5.05 5.02 0 0.0497023131201 0.0497017892644 7650.66666667 109.276407945 7620 7560 7772 3.68331047311 3.68331047311 1 +h rwkv expert rwkv h:rwkv 1048576 3 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 20.1866666667 0.0907377172588 20.15 20.12 20.29 20.1733333333 0 0.0495383143584 0.0496277915633 8317.33333333 99.1429943734 8360 8204 8388 3.62635365568 3.62635365568 1 +h rwkv expert rwkv h:rwkv 2097152 3 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 40.34 0.718957578721 39.94 39.91 41.17 40.3133333333 0 0.049588976263 0.050075112669 9345.33333333 40.8574758561 9328 9316 9392 3.64949816345 3.64949816345 1 +h rwkv expert rwkv h:rwkv 4194304 3 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 86.2666666667 0.235867194271 86.29 86.02 86.49 86.2133333333 0 0.046368082804 0.0463553134778 11318.6666667 77.3907832583 11276 11272 11408 3.70207627028 3.70207627028 1 +h rwkv expert rwkv h:rwkv 10000000 3 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 410.166666667 0.767094083756 410.01 409.49 411 409.903333333 0 0.0232509517824 0.0232597818689 17169.3333333 83.1705075933 17136 17108 17264 3.76255426699 3.76255426699 1 +compress ctw expert ctw compress:ctw 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.05 0 0.05 0.05 0.05 0.0433333333333 0 0.078125 0.078125 13493.3333333 16.6533279957 13488 13480 13512 1332 1332 0.3251953125 0.3251953125 1 +compress ctw expert ctw compress:ctw 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.24 0 0.24 0.24 0.24 0.223333333333 0.01 0.0651041666667 0.0651041666667 41070.6666667 114.844822841 41100 40944 41168 6579 6579 0.401550292969 0.401550292969 1 +compress ctw expert ctw compress:ctw 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 1.02333333333 0.0057735026919 1.02 1.02 1.03 0.996666666667 0.02 0.0610762104194 0.0612745098039 92374.6666667 68.1566822354 92412 92296 92416 22728 22728 0.346801757812 0.346801757812 1 +compress ctw expert ctw compress:ctw 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 4.27666666667 0.0208166599947 4.27 4.26 4.3 4.19666666667 0.07 0.0584576634203 0.0585480093677 221977.333333 224.582575756 221996 221744 222192 80004 80004 0.305191040039 0.305191040039 1 +compress ctw expert ctw compress:ctw 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 18.29 0.05 18.29 18.24 18.34 18.0433333333 0.226666666667 0.0546749580229 0.0546746856206 552529.333333 193.052669843 552580 552316 552692 301713 301713 0.287735939026 0.287735939026 1 +compress ctw expert ctw compress:ctw 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 38.0366666667 0.142243921956 37.97 37.94 38.2 37.6433333333 0.356666666667 0.0525813322761 0.0526731630234 898530.666667 33.5459883344 898548 898492 898552 597003 597003 0.284673213959 0.284673213959 1 +compress ctw expert ctw compress:ctw 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 78.5666666667 0.111504857891 78.61 78.44 78.65 77.91 0.586666666667 0.0509122449099 0.0508841114362 1475328 35.5527776693 1475316 1475300 1475368 1177962 1177962 0.280848026276 0.280848026276 1 +compress ctw expert ctw compress:ctw 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 199.09 0.170587221092 199.04 198.95 199.28 197.626666667 1.28333333333 0.0479016918487 0.0479137015879 2779064 83.1384387633 2779016 2779016 2779160 2746861 2746861 0.2746861 0.2746861 1 +compress match expert match compress:match 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0 0 0 0 0 0 0 5346.66666667 211.521472511 5444 5104 5492 2874 2874 0.70166015625 0.70166015625 1 +compress match expert match compress:match 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.01 0 0.01 0.01 0.01 0.01 0 1.5625 1.5625 5422.66666667 105.096780794 5460 5304 5504 13506 13506 0.824340820312 0.824340820312 1 +compress match expert match compress:match 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.05 0 0.05 0.05 0.05 0.05 0 1.25 1.25 6277.33333333 104.332800851 6252 6188 6392 53748 53748 0.820129394531 0.820129394531 1 +compress match expert match compress:match 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 0.2 0 0.2 0.2 0.2 0.196666666667 0 1.25 1.25 8057.33333333 10.0664459137 8056 8048 8068 206203 206203 0.786602020264 0.786602020264 1 +compress match expert match compress:match 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 0.82 0.01 0.82 0.81 0.83 0.82 0 1.21963312449 1.21951219512 12077.3333333 111.880889044 12036 11992 12204 817505 817505 0.779633522034 0.779633522034 1 +compress match expert match compress:match 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 1.61 0.01 1.61 1.6 1.62 1.6 0 1.24226797536 1.24223602484 22096 86.6256313108 22048 22044 22196 1644756 1644756 0.784280776978 0.784280776978 1 +compress match expert match compress:match 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 3.24333333333 0.0115470053838 3.25 3.23 3.25 3.23 0.00666666666667 1.23330951814 1.23076923077 28921.3333333 67.448745973 28952 28844 28968 3292751 3292751 0.785053014755 0.785053014755 1 +compress match expert match compress:match 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 7.75666666667 0.0404145188433 7.75 7.72 7.8 7.73666666667 0.01 1.22951210348 1.23054750504 55398.6666667 22.0302821891 55388 55384 55424 7859324 7859324 0.7859324 0.7859324 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.163333333333 0.0057735026919 0.16 0.16 0.17 0.16 0 0.0239353553922 0.0244140625 18694.6666667 138.120720145 18760 18536 18788 983 983 0.239990234375 0.239990234375 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.736666666667 0.0152752523165 0.74 0.72 0.75 0.713333333333 0.0133333333333 0.021216529029 0.0211148648649 59294.6666667 197.274766084 59336 59080 59468 5490 5490 0.335083007812 0.335083007812 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 2.98 0 2.98 2.98 2.98 2.91666666667 0.0533333333333 0.0209731543624 0.0209731543624 157646.666667 40.8574758561 157664 157600 157676 19292 19292 0.294372558594 0.294372558594 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 12.8933333333 0.0057735026919 12.89 12.89 12.9 12.7 0.173333333333 0.0193898681549 0.0193948797517 479310.666667 141.213785918 479332 479160 479440 67344 67344 0.256896972656 0.256896972656 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 56.4 0.03 56.4 56.37 56.43 55.81 0.53 0.0177304997983 0.0177304964539 1190862.66667 79.4313120459 1190896 1190772 1190920 254132 254132 0.242359161377 0.242359161377 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 117.83 0.31224989992 117.93 117.48 118.08 116.87 0.846666666667 0.0169736855989 0.0169592130925 1837317.33333 56.1901533485 1837312 1837264 1837376 503320 503320 0.240001678467 0.240001678467 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 254.493333333 0.185831464864 254.58 254.28 254.62 253.04 1.21666666667 0.0157175096496 0.0157121533506 2799441.33333 10.0664459137 2799440 2799432 2799452 991873 991873 0.236480951309 0.236480951309 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 835.173333333 1.87324139751 834.33 833.87 837.32 832.083333333 2.33666666667 0.0114189171633 0.0114304210133 5061216 93.5521245082 5061268 5061108 5061272 2272630 2272630 0.227263 0.227263 1 +compress ppmd expert ppmd compress:ppmd 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.01 0 0.01 0.01 0.01 0.00666666666667 0 0.390625 0.390625 7044 274.021896935 6900 6872 7360 1058 1058 0.25830078125 0.25830078125 1 +compress ppmd expert ppmd compress:ppmd 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.04 0 0.04 0.04 0.04 0.0266666666667 0.00333333333333 0.390625 0.390625 16032 162.923294835 16012 15880 16204 6267 6267 0.382507324219 0.382507324219 1 +compress ppmd expert ppmd compress:ppmd 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.21 0 0.21 0.21 0.21 0.193333333333 0.0133333333333 0.297619047619 0.297619047619 44961.3333333 263.949490118 45052 44664 45168 23186 23186 0.353790283203 0.353790283203 1 +compress ppmd expert ppmd compress:ppmd 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 1.00333333333 0.0057735026919 1 1 1.01 0.923333333333 0.0733333333333 0.249174917492 0.25 144137.333333 88.4835201229 144148 144044 144220 83269 83269 0.317646026611 0.317646026611 1 +compress ppmd expert ppmd compress:ppmd 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 4.21 0 4.21 4.21 4.21 4.01 0.19 0.237529691211 0.237529691211 399285.333333 90.0074071026 399248 399220 399388 326331 326331 0.311213493347 0.311213493347 1 +compress ppmd expert ppmd compress:ppmd 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 8.28333333333 0.0750555349947 8.28 8.21 8.36 8.04333333333 0.223333333333 0.241461900933 0.24154589372 457633.333333 182.486529183 457532 457524 457844 661953 661953 0.315643787384 0.315643787384 1 +compress ppmd expert ppmd compress:ppmd 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 16.41 0.0346410161514 16.39 16.39 16.45 16.1666666667 0.226666666667 0.243754531917 0.244051250763 459718.666667 60.5750223552 459692 459676 459788 1329257 1329257 0.316919565201 0.316919565201 1 +compress ppmd expert ppmd compress:ppmd 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 39.86 0.0435889894354 39.88 39.81 39.89 39.5433333333 0.273333333333 0.239256165872 0.239135987063 568756 154.970965022 568816 568580 568872 3154465 3154465 0.3154465 0.3154465 1 +compress rosa expert rosaplus compress:rosa 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0 0 0 0 0 0 0 5684 246.933189345 5804 5400 5848 1127 1127 0.275146484375 0.275146484375 1 +compress rosa expert rosaplus compress:rosa 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.03 0 0.03 0.03 0.03 0.02 0 0.520833333333 0.520833333333 8556 81.6823114266 8528 8492 8648 6359 6359 0.388122558594 0.388122558594 1 +compress rosa expert rosaplus compress:rosa 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.15 0 0.15 0.15 0.15 0.143333333333 0.00333333333333 0.416666666667 0.416666666667 19252 65.482822175 19268 19180 19308 22843 22843 0.348556518555 0.348556518555 1 +compress rosa expert rosaplus compress:rosa 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 0.85 0 0.85 0.85 0.85 0.833333333333 0.01 0.294117647059 0.294117647059 61973.3333333 48.0555234425 61976 61924 62020 80590 80590 0.307426452637 0.307426452637 1 +compress rosa expert rosaplus compress:rosa 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 4.76666666667 0.0152752523165 4.77 4.75 4.78 4.69666666667 0.0633333333333 0.209791647527 0.20964360587 172204 49.1528229098 172224 172148 172240 306522 306522 0.292322158813 0.292322158813 1 +compress rosa expert rosaplus compress:rosa 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 11.5466666667 0.204287379281 11.46 11.4 11.78 11.3866666667 0.146666666667 0.173245984409 0.174520069808 337397.333333 32.083225108 337400 337364 337428 608681 608681 0.290241718292 0.290241718292 1 +compress rosa expert rosaplus compress:rosa 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 27.2766666667 0.119303534454 27.33 27.14 27.36 26.9466666667 0.3 0.146647359224 0.146359312111 669366.666667 62.0107517559 669392 669296 669412 1199345 1199345 0.285946130753 0.285946130753 1 +compress rosa expert rosaplus compress:rosa 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 80.5633333333 0.310214979221 80.55 80.26 80.88 79.76 0.733333333333 0.118376896964 0.118395321714 1556642.66667 52.2047252012 1556648 1556588 1556692 2752778 2752778 0.2752778 0.2752778 1 +compress rwkv expert rwkv compress:rwkv 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.0633333333333 0.0057735026919 0.06 0.06 0.07 0.06 0 0.062003968254 0.0651041666667 6570.66666667 81.0267445559 6596 6480 6636 2368 2368 0.578125 0.578125 1 +compress rwkv expert rwkv compress:rwkv 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.253333333333 0.0057735026919 0.25 0.25 0.26 0.253333333333 0 0.0616987179487 0.0625 6613.33333333 127.582652948 6648 6472 6720 10098 10098 0.616333007812 0.616333007812 1 +compress rwkv expert rwkv compress:rwkv 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 1.02 0 1.02 1.02 1.02 1.02 0 0.0612745098039 0.0612745098039 6748 112.853887837 6764 6628 6852 32333 32333 0.493362426758 0.493362426758 1 +compress rwkv expert rwkv compress:rwkv 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 4.08333333333 0.0251661147842 4.08 4.06 4.11 4.07666666667 0 0.061226038364 0.0612745098039 7014.66666667 20.5264057578 7020 6992 7032 120713 120713 0.460483551025 0.460483551025 1 +compress rwkv expert rwkv compress:rwkv 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 16.45 0.0346410161514 16.47 16.41 16.47 16.4333333333 0 0.0607904534938 0.0607164541591 8654.66666667 54.3077649451 8684 8592 8688 475332 475332 0.453311920166 0.453311920166 1 +compress rwkv expert rwkv compress:rwkv 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 37.1566666667 0.255016339346 37.16 36.9 37.41 37.13 0 0.0538278321726 0.05382131324 10502.6666667 117.189305542 10456 10416 10636 956112 956112 0.455909729004 0.455909729004 1 +compress rwkv expert rwkv compress:rwkv 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 72.72 0.286879765756 72.86 72.39 72.91 72.67 0.00333333333333 0.0550060725133 0.0548998078507 17928 110.489818535 17892 17840 18052 3599584 3599584 0.858207702637 0.858207702637 1 +compress rwkv expert rwkv compress:rwkv 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 360.813333333 0.453908948285 361.05 360.29 361.1 360.573333333 0.0133333333333 0.0264312661207 0.0264139126549 25844 86.0697391654 25840 25760 25932 4788130 4788130 0.478813 0.478813 1 +decompress ctw expert ctw decompress:ctw 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.05 0 0.05 0.05 0.05 0.0466666666667 0 0.078125 0.078125 13437.3333333 64.0416531121 13440 13372 13500 1332 1332 0.3251953125 0.3251953125 1 +decompress ctw expert ctw decompress:ctw 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.233333333333 0.0057735026919 0.23 0.23 0.24 0.22 0.0133333333333 0.0669912439614 0.0679347826087 40985.3333333 188.226813534 40996 40792 41168 6579 6579 0.401550292969 0.401550292969 1 +decompress ctw expert ctw decompress:ctw 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 1.02333333333 0.0057735026919 1.02 1.02 1.03 0.99 0.0233333333333 0.0610762104194 0.0612745098039 92397.3333333 80.133222407 92392 92320 92480 22728 22728 0.346801757812 0.346801757812 1 +decompress ctw expert ctw decompress:ctw 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 4.28666666667 0.0057735026919 4.29 4.28 4.29 4.21 0.0633333333333 0.0583204438345 0.0582750582751 222077.333333 59.3745175419 222092 222012 222128 80004 80004 0.305191040039 0.305191040039 1 +decompress ctw expert ctw decompress:ctw 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 18.3433333333 0.0057735026919 18.34 18.34 18.35 18.0866666667 0.236666666667 0.0545157222987 0.0545256270447 552432 94.5727233403 552392 552364 552540 301713 301713 0.287735939026 0.287735939026 1 +decompress ctw expert ctw decompress:ctw 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 38.08 0.0264575131106 38.09 38.05 38.1 37.6533333333 0.386666666667 0.0525210253114 0.0525072197427 897984 52 897956 897952 898044 597003 597003 0.284673213959 0.284673213959 1 +decompress ctw expert ctw decompress:ctw 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 79.1066666667 0.254820198048 78.98 78.94 79.4 78.42 0.613333333333 0.0505649876157 0.050645733097 1474340 84.2852300228 1474348 1474252 1474420 1177962 1177962 0.280848026276 0.280848026276 1 +decompress ctw expert ctw decompress:ctw 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 199.673333333 0.132790561914 199.75 199.52 199.75 198.316666667 1.18333333333 0.0477617407285 0.0477433950641 2776528 49.9599839872 2776512 2776488 2776584 2746861 2746861 0.2746861 0.2746861 1 +decompress match expert match decompress:match 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0 0 0 0 0 0 0 5361.33333333 88.7543426168 5348 5280 5456 2874 2874 0.70166015625 0.70166015625 1 +decompress match expert match decompress:match 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.01 0 0.01 0.01 0.01 0.01 0 1.5625 1.5625 5436 98.0612053771 5440 5336 5532 13506 13506 0.824340820312 0.824340820312 1 +decompress match expert match decompress:match 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.05 0 0.05 0.05 0.05 0.05 0 1.25 1.25 6380 115.723809132 6428 6248 6464 53748 53748 0.820129394531 0.820129394531 1 +decompress match expert match decompress:match 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 0.2 0 0.2 0.2 0.2 0.2 0 1.25 1.25 7877.33333333 75.0821772016 7892 7796 7944 206203 206203 0.786602020264 0.786602020264 1 +decompress match expert match decompress:match 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 0.853333333333 0.0115470053838 0.86 0.84 0.86 0.846666666667 0 1.17201919528 1.16279069767 11396 66.813172354 11384 11336 11468 817505 817505 0.779633522034 0.779633522034 1 +decompress match expert match decompress:match 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 1.67666666667 0.0152752523165 1.68 1.66 1.69 1.67 0 1.19290914008 1.19047619048 20069.3333333 88.9344327768 20020 20016 20172 1644756 1644756 0.784280776978 0.784280776978 1 +decompress match expert match decompress:match 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 3.34666666667 0.0251661147842 3.35 3.32 3.37 3.33666666667 0.00333333333333 1.19526424934 1.19402985075 23890.6666667 54.3077649451 23920 23828 23924 3292751 3292751 0.785053014755 0.785053014755 1 +decompress match expert match decompress:match 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 8.01666666667 0.0550757054729 7.99 7.98 8.08 7.99 0.0166666666667 1.18965182305 1.1935848766 45941.3333333 161.013456995 45880 45820 46124 7859324 7859324 0.7859324 0.7859324 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.16 0 0.16 0.16 0.16 0.156666666667 0 0.0244140625 0.0244140625 18765.3333333 92.2893998969 18804 18660 18832 983 983 0.239990234375 0.239990234375 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.75 0.01 0.75 0.74 0.76 0.723333333333 0.02 0.0208358029082 0.0208333333333 58865.3333333 86.0077515886 58844 58792 58960 5490 5490 0.335083007812 0.335083007812 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 2.97 0 2.97 2.97 2.97 2.90333333333 0.06 0.0210437710438 0.0210437710438 156965.333333 98.7387124351 156992 156856 157048 19292 19292 0.294372558594 0.294372558594 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 12.9166666667 0.0115470053838 12.91 12.91 12.93 12.7166666667 0.183333333333 0.0193548490162 0.0193648334624 479588 58.9236794506 479600 479524 479640 67344 67344 0.256896972656 0.256896972656 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 56.3733333333 0.0750555349947 56.37 56.3 56.45 55.7866666667 0.53 0.0177389045941 0.0177399325883 1182824 90.0666419936 1182828 1182732 1182912 254132 254132 0.242359161377 0.242359161377 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 118.15 0.276224546339 118.06 117.93 118.46 117.25 0.783333333333 0.016927695983 0.0169405387091 1835526.66667 93.7514444333 1835564 1835420 1835596 503320 503320 0.240001678467 0.240001678467 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 254.33 0.298161030318 254.41 254 254.58 252.84 1.26 0.0157276124194 0.0157226524115 2777461.33333 42.7707064863 2777452 2777424 2777508 991873 991873 0.236480951309 0.236480951309 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 837.243333333 0.50013331556 837.23 836.75 837.75 834.096666667 2.38666666667 0.0113906495914 0.0113908282838 5041721.33333 92.1158690636 5041744 5041620 5041800 2272630 2272630 0.227263 0.227263 1 +decompress ppmd expert ppmd decompress:ppmd 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.01 0 0.01 0.01 0.01 0.00333333333333 0 0.390625 0.390625 7198.66666667 163.23398339 7236 7020 7340 1058 1058 0.25830078125 0.25830078125 1 +decompress ppmd expert ppmd decompress:ppmd 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.04 0 0.04 0.04 0.04 0.04 0 0.390625 0.390625 16117.3333333 88.4835201229 16128 16024 16200 6267 6267 0.382507324219 0.382507324219 1 +decompress ppmd expert ppmd decompress:ppmd 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.213333333333 0.0057735026919 0.21 0.21 0.22 0.196666666667 0.0133333333333 0.29310966811 0.297619047619 45034.6666667 98.1699207157 44980 44976 45148 23186 23186 0.353790283203 0.353790283203 1 +decompress ppmd expert ppmd decompress:ppmd 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 1.01333333333 0.0057735026919 1.01 1.01 1.02 0.946666666667 0.06 0.246715848055 0.247524752475 143966.666667 91.2432645916 143916 143912 144072 83269 83269 0.317646026611 0.317646026611 1 +decompress ppmd expert ppmd decompress:ppmd 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 4.23666666667 0.0115470053838 4.23 4.23 4.25 4.02333333333 0.203333333333 0.236035785473 0.236406619385 399052 121.786698781 399076 398920 399160 326331 326331 0.311213493347 0.311213493347 1 +decompress ppmd expert ppmd decompress:ppmd 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 8.29333333333 0.057735026919 8.26 8.26 8.36 8.08333333333 0.196666666667 0.24116531699 0.242130750605 457612 58.9236794506 457600 457560 457676 661953 661953 0.315643787384 0.315643787384 1 +decompress ppmd expert ppmd decompress:ppmd 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 16.5566666667 0.0635085296109 16.52 16.52 16.63 16.3133333333 0.22 0.241596888457 0.242130750605 458101.333333 224.582575756 458180 457848 458276 1329257 1329257 0.316919565201 0.316919565201 1 +decompress ppmd expert ppmd decompress:ppmd 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 40.2 0.0754983443527 40.21 40.12 40.27 39.89 0.263333333333 0.237232975058 0.237173418654 559082.666667 354.679197774 559244 558676 559328 3154465 3154465 0.3154465 0.3154465 1 +decompress rosa expert rosaplus decompress:rosa 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0 0 0 0 0 0 0 5866.66666667 65.0333247907 5844 5816 5940 1127 1127 0.275146484375 0.275146484375 1 +decompress rosa expert rosaplus decompress:rosa 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.03 0 0.03 0.03 0.03 0.0233333333333 0 0.520833333333 0.520833333333 8504 58.9236794506 8472 8468 8572 6359 6359 0.388122558594 0.388122558594 1 +decompress rosa expert rosaplus decompress:rosa 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 0.15 0 0.15 0.15 0.15 0.143333333333 0.00333333333333 0.416666666667 0.416666666667 19317.3333333 73.3575717519 19292 19260 19400 22843 22843 0.348556518555 0.348556518555 1 +decompress rosa expert rosaplus decompress:rosa 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 0.846666666667 0.0115470053838 0.84 0.84 0.86 0.83 0.01 0.295311923219 0.297619047619 61872 66.0908465674 61904 61796 61916 80590 80590 0.307426452637 0.307426452637 1 +decompress rosa expert rosaplus decompress:rosa 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 4.78333333333 0.0057735026919 4.78 4.78 4.79 4.71 0.0633333333333 0.209059436355 0.209205020921 172616 115.377640815 172584 172520 172744 306522 306522 0.292322158813 0.292322158813 1 +decompress rosa expert rosaplus decompress:rosa 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 11.4333333333 0.0115470053838 11.44 11.42 11.44 11.2966666667 0.123333333333 0.174927232721 0.174825174825 338036 106.056588669 338060 337920 338128 608681 608681 0.290241718292 0.290241718292 1 +decompress rosa expert rosaplus decompress:rosa 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 27.31 0.0556776436283 27.32 27.25 27.36 27.0033333333 0.276666666667 0.146466901856 0.146412884334 658334.666667 71.7030915187 658372 658252 658380 1199345 1199345 0.285946130753 0.285946130753 1 +decompress rosa expert rosaplus decompress:rosa 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 80.5866666667 0.0503322295685 80.58 80.54 80.64 79.81 0.706666666667 0.118341483007 0.118351243039 1554957.33333 138.236512302 1554948 1554824 1555100 2752778 2752778 0.2752778 0.2752778 1 +decompress rwkv expert rwkv decompress:rwkv 4096 3 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e 0.0666666666667 0.0057735026919 0.07 0.06 0.07 0.0633333333333 0 0.0589037698413 0.0558035714286 6648 61.5792172734 6632 6596 6716 2368 2368 0.578125 0.578125 1 +decompress rwkv expert rwkv decompress:rwkv 16384 3 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f 0.256666666667 0.0057735026919 0.26 0.25 0.26 0.256666666667 0 0.0608974358974 0.0600961538462 6530.66666667 44.9592408002 6520 6492 6580 10098 10098 0.616333007812 0.616333007812 1 +decompress rwkv expert rwkv decompress:rwkv 65536 3 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 1.03 0 1.03 1.03 1.03 1.03 0 0.0606796116505 0.0606796116505 6670.66666667 34.9475797922 6680 6632 6700 32333 32333 0.493362426758 0.493362426758 1 +decompress rwkv expert rwkv decompress:rwkv 262144 3 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 4.11 0.0346410161514 4.09 4.09 4.15 4.10333333333 0 0.0608301175362 0.0611246943765 7001.33333333 114.705419808 7016 6880 7108 120713 120713 0.460483551025 0.460483551025 1 +decompress rwkv expert rwkv decompress:rwkv 1048576 3 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 16.29 0.0264575131106 16.28 16.27 16.32 16.2766666667 0 0.0613874620753 0.0614250614251 8069.33333333 28.9367125523 8084 8036 8088 475332 475332 0.453311920166 0.453311920166 1 +decompress rwkv expert rwkv decompress:rwkv 2097152 3 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 37.2566666667 0.136503968196 37.28 37.11 37.38 37.23 0 0.0536821485709 0.0536480686695 9566.66666667 105.248911317 9512 9500 9688 956112 956112 0.455909729004 0.455909729004 1 +decompress rwkv expert rwkv decompress:rwkv 4194304 3 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 72.8666666667 0.366924152017 72.91 72.48 73.21 72.8133333333 0 0.0548957138067 0.0548621588259 14153.3333333 34.0196021925 14140 14128 14192 3599584 3599584 0.858207702637 0.858207702637 1 +decompress rwkv expert rwkv decompress:rwkv 10000000 3 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 359.19 0.676683086829 358.84 358.76 359.97 358.96 0.00333333333333 0.0265507550454 0.0265765889089 21162.6666667 56.1901533485 21168 21104 21216 4788130 4788130 0.478813 0.478813 1 diff --git a/benchmarks/bcb2c188/infotheory-two-json-raw-bcb2c188-20260601-021136.tsv b/benchmarks/bcb2c188/infotheory-two-json-raw-bcb2c188-20260601-021136.tsv new file mode 100644 index 00000000..c1337c25 --- /dev/null +++ b/benchmarks/bcb2c188/infotheory-two-json-raw-bcb2c188-20260601-021136.tsv @@ -0,0 +1,289 @@ +operation subject subject_kind expert_kind series size_bytes repetition cpu compression_backend input_sha256 suite_spec_path suite_spec_sha256 build_mode build_features archive_bytes entropy_bpb real_seconds user_seconds sys_seconds rss_kib verified +h neural_mixture mixture neural-mixture h:neural_mixture 4096 1 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1.918573669072413 0.11 0.10 0.00 12828 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1001 0.12 0.12 0.00 12672 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1001 0.12 0.12 0.00 12904 1 +h neural_mixture mixture neural-mixture h:neural_mixture 4096 2 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1.918573669072413 0.11 0.11 0.00 12632 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1001 0.12 0.12 0.00 12700 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1001 0.12 0.11 0.00 12896 1 +h ctw expert ctw h:ctw 4096 1 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.5654962454168757 0.02 0.02 0.00 7392 1 +compress ctw expert ctw compress:ctw 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1332 0.04 0.03 0.00 7320 1 +decompress ctw expert ctw decompress:ctw 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1332 0.03 0.03 0.00 7520 1 +h ctw expert ctw h:ctw 4096 2 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.5654962454168757 0.02 0.01 0.00 7328 1 +compress ctw expert ctw compress:ctw 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1332 0.04 0.03 0.00 7504 1 +decompress ctw expert ctw decompress:ctw 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1332 0.03 0.03 0.00 7648 1 +h ppmd expert ppmd h:ppmd 4096 1 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.029162762208123 0.00 0.00 0.00 8096 1 +compress ppmd expert ppmd compress:ppmd 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1058 0.00 0.00 0.00 7584 1 +decompress ppmd expert ppmd decompress:ppmd 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1058 0.00 0.00 0.00 7840 1 +h ppmd expert ppmd h:ppmd 4096 2 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.029162762208123 0.01 0.00 0.00 7688 1 +compress ppmd expert ppmd compress:ppmd 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1058 0.00 0.00 0.00 7776 1 +decompress ppmd expert ppmd decompress:ppmd 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1058 0.01 0.00 0.00 7596 1 +h rosa expert rosaplus h:rosa 4096 1 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.111721242855073 0.05 0.04 0.00 6252 1 +compress rosa expert rosaplus compress:rosa 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1127 0.00 0.00 0.00 6224 1 +decompress rosa expert rosaplus decompress:rosa 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1127 0.00 0.00 0.00 6036 1 +h rosa expert rosaplus h:rosa 4096 2 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.111721242855073 0.03 0.03 0.00 6368 1 +compress rosa expert rosaplus compress:rosa 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1127 0.00 0.00 0.00 6260 1 +decompress rosa expert rosaplus decompress:rosa 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1127 0.00 0.00 0.00 6488 1 +h match expert match h:match 4096 1 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 5.577349932524691 0.00 0.00 0.00 6124 1 +compress match expert match compress:match 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2874 0.00 0.00 0.00 6040 1 +decompress match expert match decompress:match 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2874 0.00 0.00 0.00 5660 1 +h match expert match h:match 4096 2 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 5.577349932524691 0.00 0.00 0.00 6076 1 +compress match expert match compress:match 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2874 0.00 0.00 0.00 5920 1 +decompress match expert match decompress:match 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2874 0.00 0.00 0.00 5860 1 +h rwkv7 expert rwkv7 h:rwkv7 4096 1 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 7.217014379468224 0.09 0.08 0.00 8824 1 +compress rwkv7 expert rwkv7 compress:rwkv7 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3714 0.06 0.05 0.00 8160 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3714 0.05 0.05 0.00 8300 1 +h rwkv7 expert rwkv7 h:rwkv7 4096 2 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 7.217014379468224 0.05 0.05 0.00 8724 1 +compress rwkv7 expert rwkv7 compress:rwkv7 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3714 0.06 0.05 0.00 8356 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3714 0.06 0.06 0.00 8372 1 +h neural_mixture mixture neural-mixture h:neural_mixture 16384 1 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.6924577982523554 0.48 0.46 0.01 30496 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 5533 0.52 0.50 0.01 31064 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 5533 0.52 0.50 0.01 30840 1 +h neural_mixture mixture neural-mixture h:neural_mixture 16384 2 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.6924577982523554 0.47 0.46 0.00 30456 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 5533 0.52 0.51 0.00 31052 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 5533 0.53 0.52 0.01 30880 1 +h ctw expert ctw h:ctw 16384 1 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3.2034095120908512 0.09 0.09 0.00 12476 1 +compress ctw expert ctw compress:ctw 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 6579 0.18 0.17 0.00 12520 1 +decompress ctw expert ctw decompress:ctw 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 6579 0.18 0.17 0.00 12504 1 +h ctw expert ctw h:ctw 16384 2 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3.2034095120908512 0.10 0.09 0.00 12468 1 +compress ctw expert ctw compress:ctw 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 6579 0.18 0.18 0.00 12644 1 +decompress ctw expert ctw decompress:ctw 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 6579 0.18 0.18 0.00 12460 1 +h ppmd expert ppmd h:ppmd 16384 1 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3.05111741488576 0.02 0.02 0.00 15096 1 +compress ppmd expert ppmd compress:ppmd 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 6267 0.03 0.02 0.00 15272 1 +decompress ppmd expert ppmd decompress:ppmd 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 6267 0.03 0.03 0.00 15236 1 +h ppmd expert ppmd h:ppmd 16384 2 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3.05111741488576 0.02 0.02 0.00 14984 1 +compress ppmd expert ppmd compress:ppmd 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 6267 0.03 0.02 0.00 15556 1 +decompress ppmd expert ppmd decompress:ppmd 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 6267 0.03 0.02 0.01 15404 1 +h rosa expert rosaplus h:rosa 16384 1 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3.486306229863964 0.13 0.13 0.00 8456 1 +compress rosa expert rosaplus compress:rosa 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 6359 0.02 0.02 0.00 7904 1 +decompress rosa expert rosaplus decompress:rosa 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 6359 0.02 0.02 0.00 7984 1 +h rosa expert rosaplus h:rosa 16384 2 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3.486306229863964 0.13 0.12 0.00 8340 1 +compress rosa expert rosaplus compress:rosa 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 6359 0.02 0.02 0.00 7964 1 +decompress rosa expert rosaplus decompress:rosa 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 6359 0.02 0.02 0.00 8032 1 +h match expert match h:match 16384 1 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 6.5858628640659385 0.00 0.00 0.00 6280 1 +compress match expert match compress:match 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 13506 0.01 0.00 0.00 5980 1 +decompress match expert match decompress:match 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 13506 0.01 0.01 0.00 6172 1 +h match expert match h:match 16384 2 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 6.5858628640659385 0.00 0.00 0.00 6036 1 +compress match expert match compress:match 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 13506 0.03 0.02 0.00 5972 1 +decompress match expert match decompress:match 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 13506 0.02 0.02 0.00 6060 1 +h rwkv7 expert rwkv7 h:rwkv7 16384 1 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 5.850210720420741 0.24 0.23 0.00 8792 1 +compress rwkv7 expert rwkv7 compress:rwkv7 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 12000 0.22 0.21 0.00 8272 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 12000 0.21 0.21 0.00 8344 1 +h rwkv7 expert rwkv7 h:rwkv7 16384 2 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 5.850210720420741 0.20 0.20 0.00 9092 1 +compress rwkv7 expert rwkv7 compress:rwkv7 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 12000 0.21 0.21 0.00 8100 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 12000 0.21 0.21 0.00 8060 1 +h neural_mixture mixture neural-mixture h:neural_mixture 65536 1 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.359021171161541 2.07 2.03 0.03 83564 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 19344 2.26 2.23 0.02 84640 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 19344 2.29 2.26 0.02 87648 1 +h neural_mixture mixture neural-mixture h:neural_mixture 65536 2 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.359021171161541 2.07 2.03 0.03 83696 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 19344 2.27 2.25 0.01 84572 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 19344 2.26 2.24 0.02 87672 1 +h ctw expert ctw h:ctw 65536 1 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.7721389563404757 0.48 0.47 0.00 22736 1 +compress ctw expert ctw compress:ctw 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 22728 0.81 0.80 0.00 22884 1 +decompress ctw expert ctw decompress:ctw 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 22728 0.80 0.79 0.01 23024 1 +h ctw expert ctw h:ctw 65536 2 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.7721389563404757 0.47 0.46 0.01 22684 1 +compress ctw expert ctw compress:ctw 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 22728 0.80 0.79 0.00 22892 1 +decompress ctw expert ctw decompress:ctw 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 22728 0.81 0.80 0.00 22868 1 +h ppmd expert ppmd h:ppmd 65536 1 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.828003099069916 0.11 0.09 0.02 39052 1 +compress ppmd expert ppmd compress:ppmd 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 23186 0.14 0.13 0.01 39472 1 +decompress ppmd expert ppmd decompress:ppmd 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 23186 0.14 0.13 0.01 39656 1 +h ppmd expert ppmd h:ppmd 65536 2 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.828003099069916 0.11 0.10 0.01 39092 1 +compress ppmd expert ppmd compress:ppmd 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 23186 0.14 0.13 0.00 39436 1 +decompress ppmd expert ppmd decompress:ppmd 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 23186 0.14 0.11 0.02 39476 1 +h rosa expert rosaplus h:rosa 65536 1 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3.1905669620146786 0.64 0.62 0.01 15508 1 +compress rosa expert rosaplus compress:rosa 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 22843 0.12 0.12 0.00 16476 1 +decompress rosa expert rosaplus decompress:rosa 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 22843 0.13 0.13 0.00 16480 1 +h rosa expert rosaplus h:rosa 65536 2 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3.1905669620146786 0.63 0.61 0.02 15476 1 +compress rosa expert rosaplus compress:rosa 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 22843 0.12 0.12 0.00 16452 1 +decompress rosa expert rosaplus decompress:rosa 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 22843 0.12 0.12 0.00 16348 1 +h match expert match h:match 65536 1 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 6.558825530499182 0.01 0.00 0.00 6996 1 +compress match expert match compress:match 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 53748 0.03 0.03 0.00 6712 1 +decompress match expert match decompress:match 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 53748 0.03 0.03 0.00 6928 1 +h match expert match h:match 65536 2 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 6.558825530499182 0.01 0.01 0.00 7196 1 +compress match expert match compress:match 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 53748 0.03 0.03 0.00 6764 1 +decompress match expert match decompress:match 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 53748 0.03 0.03 0.00 6820 1 +h rwkv7 expert rwkv7 h:rwkv7 65536 1 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 4.311738518194067 0.81 0.81 0.00 8548 1 +compress rwkv7 expert rwkv7 compress:rwkv7 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 35340 0.85 0.85 0.00 8220 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 35340 0.86 0.85 0.00 8236 1 +h rwkv7 expert rwkv7 h:rwkv7 65536 2 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 4.311738518194067 0.80 0.80 0.00 8740 1 +compress rwkv7 expert rwkv7 compress:rwkv7 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 35340 0.84 0.83 0.00 8400 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 35340 0.85 0.85 0.00 8376 1 +h neural_mixture mixture neural-mixture h:neural_mixture 262144 1 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.0579561160084685 8.93 8.84 0.08 246384 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 67454 9.83 9.73 0.08 243696 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 67454 9.91 9.81 0.09 248520 1 +h neural_mixture mixture neural-mixture h:neural_mixture 262144 2 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.0579561160084685 8.94 8.84 0.08 246356 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 67454 9.91 9.80 0.08 243536 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 67454 9.93 9.82 0.09 248776 1 +h ctw expert ctw h:ctw 262144 1 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.4409671415182586 2.29 2.27 0.02 48176 1 +compress ctw expert ctw compress:ctw 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 80004 3.48 3.47 0.01 48292 1 +decompress ctw expert ctw decompress:ctw 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 80004 3.49 3.48 0.00 48640 1 +h ctw expert ctw h:ctw 262144 2 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.4409671415182586 2.29 2.27 0.02 48296 1 +compress ctw expert ctw compress:ctw 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 80004 3.48 3.46 0.01 48456 1 +decompress ctw expert ctw decompress:ctw 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 80004 3.50 3.47 0.02 48432 1 +h ppmd expert ppmd h:ppmd 262144 1 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.5405956182399834 0.52 0.46 0.05 127232 1 +compress ppmd expert ppmd compress:ppmd 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 83269 0.62 0.55 0.06 127384 1 +decompress ppmd expert ppmd decompress:ppmd 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 83269 0.62 0.55 0.07 127188 1 +h ppmd expert ppmd h:ppmd 262144 2 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.5405956182399834 0.53 0.46 0.06 127040 1 +compress ppmd expert ppmd compress:ppmd 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 83269 0.62 0.54 0.07 127372 1 +decompress ppmd expert ppmd decompress:ppmd 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 83269 0.62 0.55 0.07 127532 1 +h rosa expert rosaplus h:rosa 262144 1 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.905672544873225 3.56 3.48 0.07 44176 1 +compress rosa expert rosaplus compress:rosa 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 80590 0.70 0.67 0.03 46592 1 +decompress rosa expert rosaplus decompress:rosa 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 80590 0.71 0.69 0.01 46848 1 +h rosa expert rosaplus h:rosa 262144 2 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.905672544873225 3.54 3.47 0.06 44196 1 +compress rosa expert rosaplus compress:rosa 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 80590 0.70 0.69 0.01 46664 1 +decompress rosa expert rosaplus decompress:rosa 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 80590 0.71 0.69 0.01 46816 1 +h match expert match h:match 262144 1 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 6.292259446690552 0.03 0.02 0.00 8428 1 +compress match expert match compress:match 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 206203 0.13 0.13 0.00 8688 1 +decompress match expert match decompress:match 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 206203 0.14 0.14 0.00 8488 1 +h match expert match h:match 262144 2 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 6.292259446690552 0.03 0.03 0.00 8596 1 +compress match expert match compress:match 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 206203 0.13 0.13 0.00 8456 1 +decompress match expert match decompress:match 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 206203 0.14 0.14 0.00 8512 1 +h rwkv7 expert rwkv7 h:rwkv7 262144 1 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 4.180651265210767 3.35 3.34 0.00 8980 1 +compress rwkv7 expert rwkv7 compress:rwkv7 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 137010 3.31 3.30 0.00 8280 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 137010 3.36 3.36 0.00 8440 1 +h rwkv7 expert rwkv7 h:rwkv7 262144 2 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 4.180651265210767 3.28 3.27 0.00 9080 1 +compress rwkv7 expert rwkv7 compress:rwkv7 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 137010 3.35 3.35 0.00 8352 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 137010 3.37 3.36 0.00 8588 1 +h neural_mixture mixture neural-mixture h:neural_mixture 1048576 1 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1.9396366933249274 40.68 40.35 0.29 680428 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 254251 45.12 44.78 0.29 681084 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 254251 45.20 44.81 0.34 679164 1 +h neural_mixture mixture neural-mixture h:neural_mixture 1048576 2 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1.9396366933249274 40.89 40.51 0.33 680464 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 254251 45.06 44.70 0.31 680972 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 254251 44.96 44.61 0.30 679480 1 +h ctw expert ctw h:ctw 1048576 1 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.301742747548954 11.64 11.55 0.08 115616 1 +compress ctw expert ctw compress:ctw 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 301713 15.64 15.59 0.04 116340 1 +decompress ctw expert ctw decompress:ctw 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 301713 15.70 15.60 0.08 116192 1 +h ctw expert ctw h:ctw 1048576 2 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.301742747548954 11.64 11.58 0.04 115552 1 +compress ctw expert ctw compress:ctw 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 301713 15.67 15.60 0.05 116448 1 +decompress ctw expert ctw decompress:ctw 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 301713 15.75 15.67 0.05 116200 1 +h ppmd expert ppmd h:ppmd 1048576 1 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.489564753975732 2.39 2.19 0.19 370124 1 +compress ppmd expert ppmd compress:ppmd 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 326331 2.75 2.57 0.17 370580 1 +decompress ppmd expert ppmd decompress:ppmd 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 326331 2.82 2.64 0.17 370908 1 +h ppmd expert ppmd h:ppmd 1048576 2 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.489564753975732 2.38 2.18 0.20 370488 1 +compress ppmd expert ppmd compress:ppmd 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 326331 2.74 2.58 0.15 370720 1 +decompress ppmd expert ppmd decompress:ppmd 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 326331 2.83 2.60 0.22 371052 1 +h rosa expert rosaplus h:rosa 1048576 1 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.801230278247886 19.53 19.06 0.45 194756 1 +compress rosa expert rosaplus compress:rosa 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 306522 3.86 3.80 0.05 146116 1 +decompress rosa expert rosaplus decompress:rosa 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 306522 3.91 3.86 0.04 146592 1 +h rosa expert rosaplus h:rosa 1048576 2 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.801230278247886 19.55 19.17 0.36 194708 1 +compress rosa expert rosaplus compress:rosa 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 306522 3.90 3.83 0.06 146268 1 +decompress rosa expert rosaplus decompress:rosa 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 306522 3.91 3.84 0.05 146336 1 +h match expert match h:match 1048576 1 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 6.236926355106452 0.13 0.12 0.00 11820 1 +compress match expert match compress:match 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 817505 0.53 0.53 0.00 12512 1 +decompress match expert match decompress:match 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 817505 0.55 0.55 0.00 12036 1 +h match expert match h:match 1048576 2 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 6.236926355106452 0.14 0.14 0.00 11916 1 +compress match expert match compress:match 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 817505 0.52 0.52 0.00 12392 1 +decompress match expert match decompress:match 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 817505 0.55 0.55 0.00 12004 1 +h rwkv7 expert rwkv7 h:rwkv7 1048576 1 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3.6182904815139514 13.25 13.24 0.00 9568 1 +compress rwkv7 expert rwkv7 compress:rwkv7 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 474275 13.95 13.93 0.00 9752 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 474275 13.72 13.71 0.00 9312 1 +h rwkv7 expert rwkv7 h:rwkv7 1048576 2 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3.6182904815139514 13.43 13.42 0.00 9568 1 +compress rwkv7 expert rwkv7 compress:rwkv7 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 474275 13.82 13.81 0.00 9748 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 474275 13.96 13.94 0.00 9504 1 +h neural_mixture mixture neural-mixture h:neural_mixture 2097152 1 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1.9206985464400697 86.70 86.15 0.47 1011996 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 2097152 1 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 503520 96.26 95.72 0.44 1004904 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 2097152 1 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 503520 96.03 95.48 0.47 1010532 1 +h neural_mixture mixture neural-mixture h:neural_mixture 2097152 2 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1.9206985464400697 86.72 86.16 0.48 1012192 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 503520 96.02 95.41 0.52 1004788 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 503520 96.01 95.40 0.51 1010360 1 +h ctw expert ctw h:ctw 2097152 1 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.277315456609421 25.98 25.87 0.08 185376 1 +compress ctw expert ctw compress:ctw 2097152 1 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 597003 33.17 33.07 0.07 186556 1 +decompress ctw expert ctw decompress:ctw 2097152 1 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 597003 33.25 33.14 0.07 185836 1 +h ctw expert ctw h:ctw 2097152 2 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.277315456609421 25.91 25.83 0.06 185460 1 +compress ctw expert ctw compress:ctw 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 597003 33.15 33.06 0.06 186648 1 +decompress ctw expert ctw decompress:ctw 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 597003 33.35 33.22 0.10 185916 1 +h ppmd expert ppmd h:ppmd 2097152 1 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.525080573022069 4.93 4.70 0.22 434232 1 +compress ppmd expert ppmd compress:ppmd 2097152 1 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 661953 5.65 5.45 0.19 434516 1 +decompress ppmd expert ppmd decompress:ppmd 2097152 1 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 661953 5.75 5.53 0.21 435188 1 +h ppmd expert ppmd h:ppmd 2097152 2 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.525080573022069 5.04 4.82 0.21 434040 1 +compress ppmd expert ppmd compress:ppmd 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 661953 5.66 5.45 0.19 434316 1 +decompress ppmd expert ppmd decompress:ppmd 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 661953 5.74 5.52 0.21 435028 1 +h rosa expert rosaplus h:rosa 2097152 1 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.704371387016519 46.91 45.16 1.69 317108 1 +compress rosa expert rosaplus compress:rosa 2097152 1 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 608681 9.42 9.31 0.09 281416 1 +decompress rosa expert rosaplus decompress:rosa 2097152 1 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 608681 9.51 9.38 0.12 278264 1 +h rosa expert rosaplus h:rosa 2097152 2 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.704371387016519 46.99 45.30 1.63 317248 1 +compress rosa expert rosaplus compress:rosa 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 608681 9.35 9.20 0.14 281568 1 +decompress rosa expert rosaplus decompress:rosa 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 608681 9.38 9.23 0.13 278240 1 +h match expert match h:match 2097152 1 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 6.274177264896603 0.27 0.26 0.01 19964 1 +compress match expert match compress:match 2097152 1 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1644756 1.05 1.04 0.01 22580 1 +decompress match expert match decompress:match 2097152 1 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1644756 1.13 1.11 0.01 20676 1 +h match expert match h:match 2097152 2 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 6.274177264896603 0.27 0.27 0.00 19904 1 +compress match expert match compress:match 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1644756 1.05 1.04 0.01 22532 1 +decompress match expert match decompress:match 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1644756 1.11 1.10 0.00 20648 1 +h rwkv7 expert rwkv7 h:rwkv7 2097152 1 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3.498336499657132 27.30 27.28 0.00 10656 1 +compress rwkv7 expert rwkv7 compress:rwkv7 2097152 1 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 917086 28.47 28.45 0.00 11552 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 2097152 1 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 917086 28.17 28.15 0.00 11104 1 +h rwkv7 expert rwkv7 h:rwkv7 2097152 2 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3.498336499657132 27.57 27.55 0.00 10400 1 +compress rwkv7 expert rwkv7 compress:rwkv7 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 917086 28.33 28.31 0.00 11528 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 917086 28.22 28.20 0.00 10844 1 +h neural_mixture mixture neural-mixture h:neural_mixture 4194304 1 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1.885096705283187 184.02 183.18 0.66 1397608 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 4194304 1 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 988355 205.67 204.84 0.63 1392272 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 4194304 1 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 988355 206.06 205.30 0.56 1387308 1 +h neural_mixture mixture neural-mixture h:neural_mixture 4194304 2 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1.885096705283187 184.42 183.61 0.64 1385492 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 988355 205.93 205.10 0.62 1392084 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 988355 205.97 205.16 0.61 1387296 1 +h ctw expert ctw h:ctw 4194304 1 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.246749307224066 57.18 57.04 0.09 298384 1 +compress ctw expert ctw compress:ctw 4194304 1 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1177962 69.88 69.69 0.13 300560 1 +decompress ctw expert ctw decompress:ctw 4194304 1 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1177962 70.34 70.14 0.13 299616 1 +h ctw expert ctw h:ctw 4194304 2 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.246749307224066 57.34 57.15 0.14 298288 1 +compress ctw expert ctw compress:ctw 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1177962 69.95 69.75 0.13 300880 1 +decompress ctw expert ctw decompress:ctw 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1177962 70.29 70.06 0.16 299436 1 +h ppmd expert ppmd h:ppmd 4194304 1 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.5353209835043775 10.03 9.81 0.20 436348 1 +compress ppmd expert ppmd compress:ppmd 4194304 1 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1329257 11.57 11.34 0.21 436632 1 +decompress ppmd expert ppmd decompress:ppmd 4194304 1 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1329257 11.64 11.39 0.23 435792 1 +h ppmd expert ppmd h:ppmd 4194304 2 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.5353209835043775 10.03 9.79 0.22 436164 1 +compress ppmd expert ppmd compress:ppmd 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1329257 11.69 11.46 0.22 436632 1 +decompress ppmd expert ppmd decompress:ppmd 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1329257 11.65 11.41 0.23 435676 1 +h rosa expert rosaplus h:rosa 4194304 1 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.62099230176433 113.21 108.30 4.78 626444 1 +compress rosa expert rosaplus compress:rosa 4194304 1 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1199345 23.56 23.33 0.20 538164 1 +decompress rosa expert rosaplus decompress:rosa 4194304 1 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1199345 23.66 23.42 0.22 536052 1 +h rosa expert rosaplus h:rosa 4194304 2 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.62099230176433 113.21 108.33 4.75 626224 1 +compress rosa expert rosaplus compress:rosa 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1199345 23.39 23.12 0.25 538184 1 +decompress rosa expert rosaplus decompress:rosa 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1199345 23.45 23.15 0.27 535972 1 +h match expert match h:match 4194304 1 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 6.280388008990573 0.58 0.58 0.00 21896 1 +compress match expert match compress:match 4194304 1 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3292751 2.11 2.09 0.01 29420 1 +decompress match expert match decompress:match 4194304 1 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3292751 2.23 2.21 0.01 24244 1 +h match expert match h:match 4194304 2 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 6.280388008990573 0.55 0.54 0.00 21768 1 +compress match expert match compress:match 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3292751 2.11 2.11 0.00 29524 1 +decompress match expert match decompress:match 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3292751 2.25 2.24 0.00 24404 1 +h rwkv7 expert rwkv7 h:rwkv7 4194304 1 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3.3357009871104677 55.08 55.05 0.00 12612 1 +compress rwkv7 expert rwkv7 compress:rwkv7 4194304 1 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1748887 57.04 57.00 0.00 15456 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 4194304 1 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1748887 56.86 56.82 0.00 13792 1 +h rwkv7 expert rwkv7 h:rwkv7 4194304 2 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3.3357009871104677 56.08 56.04 0.00 12640 1 +compress rwkv7 expert rwkv7 compress:rwkv7 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1748887 55.89 55.85 0.00 15456 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1748887 56.33 56.29 0.00 13640 1 +h neural_mixture mixture neural-mixture h:neural_mixture 10000000 1 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1.8161754737084215 471.99 470.32 1.22 2510036 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 10000000 1 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2270248 535.84 534.03 1.24 2492592 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 10000000 1 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2270248 534.97 533.29 1.13 2499828 1 +h neural_mixture mixture neural-mixture h:neural_mixture 10000000 2 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1.8161754737084215 472.77 471.16 1.17 2509912 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2270248 535.56 533.78 1.22 2492740 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2270248 535.96 534.11 1.33 2499908 1 +h ctw expert ctw h:ctw 10000000 1 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.1974739848061873 152.06 151.65 0.28 541152 1 +compress ctw expert ctw compress:ctw 10000000 1 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2746861 177.70 177.27 0.27 546652 1 +decompress ctw expert ctw decompress:ctw 10000000 1 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2746861 179.92 179.50 0.25 543716 1 +h ctw expert ctw h:ctw 10000000 2 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.1974739848061873 152.31 151.88 0.30 541056 1 +compress ctw expert ctw compress:ctw 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2746861 177.59 177.19 0.25 546652 1 +decompress ctw expert ctw decompress:ctw 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2746861 178.56 178.10 0.31 543816 1 +h ppmd expert ppmd h:ppmd 10000000 1 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.523558147667331 24.41 24.12 0.26 550776 1 +compress ppmd expert ppmd compress:ppmd 10000000 1 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3154465 27.82 27.49 0.30 551024 1 +decompress ppmd expert ppmd decompress:ppmd 10000000 1 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3154465 28.62 28.30 0.29 553168 1 +h ppmd expert ppmd h:ppmd 10000000 2 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.523558147667331 24.38 24.04 0.31 550952 1 +compress ppmd expert ppmd compress:ppmd 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3154465 28.10 27.80 0.27 550768 1 +decompress ppmd expert ppmd decompress:ppmd 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3154465 28.23 27.93 0.27 553012 1 +h rosa expert rosaplus h:rosa 10000000 1 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.484818977912099 334.54 321.81 12.36 1422504 1 +compress rosa expert rosaplus compress:rosa 10000000 1 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2752778 73.44 72.78 0.59 1263248 1 +decompress rosa expert rosaplus decompress:rosa 10000000 1 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2752778 74.72 74.10 0.55 1262816 1 +h rosa expert rosaplus h:rosa 10000000 2 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.484818977912099 334.93 322.21 12.34 1422776 1 +compress rosa expert rosaplus compress:rosa 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2752778 74.09 73.39 0.62 1263456 1 +decompress rosa expert rosaplus decompress:rosa 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2752778 74.04 73.36 0.61 1262760 1 +h match expert match h:match 10000000 1 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 6.287444365927943 1.42 1.39 0.02 40692 1 +compress match expert match compress:match 10000000 1 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 7859324 5.16 5.14 0.01 55932 1 +decompress match expert match decompress:match 10000000 1 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 7859324 5.38 5.36 0.01 46480 1 +h match expert match h:match 10000000 2 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 6.287444365927943 1.35 1.33 0.01 41112 1 +compress match expert match compress:match 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 7859324 5.10 5.08 0.01 55932 1 +decompress match expert match decompress:match 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 7859324 5.36 5.33 0.02 46520 1 +h rwkv7 expert rwkv7 h:rwkv7 10000000 1 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3.174902781170517 133.15 133.07 0.00 18080 1 +compress rwkv7 expert rwkv7 compress:rwkv7 10000000 1 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3968645 137.63 137.53 0.00 25332 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 10000000 1 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3968645 136.47 136.38 0.00 21664 1 +h rwkv7 expert rwkv7 h:rwkv7 10000000 2 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3.174902781170517 132.46 132.38 0.00 18376 1 +compress rwkv7 expert rwkv7 compress:rwkv7 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3968645 135.96 135.87 0.00 25368 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3968645 137.66 137.58 0.00 21536 1 diff --git a/benchmarks/bcb2c188/infotheory-two-json-summary-bcb2c188-20260601-021136.tsv b/benchmarks/bcb2c188/infotheory-two-json-summary-bcb2c188-20260601-021136.tsv new file mode 100644 index 00000000..cc73900b --- /dev/null +++ b/benchmarks/bcb2c188/infotheory-two-json-summary-bcb2c188-20260601-021136.tsv @@ -0,0 +1,145 @@ +operation subject subject_kind expert_kind series size_bytes repeats cpu compression_backend input_sha256 suite_spec_path suite_spec_sha256 build_mode build_features real_seconds_mean real_seconds_stdev real_seconds_median real_seconds_min real_seconds_max user_seconds_mean sys_seconds_mean throughput_mib_s_mean throughput_mib_s_median rss_kib_mean rss_kib_stdev rss_kib_median rss_kib_min rss_kib_max archive_bytes_mean archive_bytes_median archive_ratio_mean archive_ratio_median entropy_bpb_mean entropy_bpb_median verified_all +h ctw expert ctw h:ctw 4096 2 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.02 0 0.02 0.02 0.02 0.015 0 0.1953125 0.1953125 7360 45.2548339959 7360 7328 7392 2.56549624542 2.56549624542 1 +h ctw expert ctw h:ctw 16384 2 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.095 0.00707106781187 0.095 0.09 0.1 0.09 0 0.164930555556 0.164930555556 12472 5.65685424949 12472 12468 12476 3.20340951209 3.20340951209 1 +h ctw expert ctw h:ctw 65536 2 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.475 0.00707106781187 0.475 0.47 0.48 0.465 0.005 0.131593528369 0.131593528369 22710 36.7695526217 22710 22684 22736 2.77213895634 2.77213895634 1 +h ctw expert ctw h:ctw 262144 2 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.29 0 2.29 2.29 2.29 2.27 0.02 0.109170305677 0.109170305677 48236 84.8528137424 48236 48176 48296 2.44096714152 2.44096714152 1 +h ctw expert ctw h:ctw 1048576 2 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 11.64 0 11.64 11.64 11.64 11.565 0.06 0.085910652921 0.085910652921 115584 45.2548339959 115584 115552 115616 2.30174274755 2.30174274755 1 +h ctw expert ctw h:ctw 2097152 2 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 25.945 0.0494974746831 25.945 25.91 25.98 25.85 0.07 0.0770862840489 0.0770862840489 185418 59.3969696197 185418 185376 185460 2.27731545661 2.27731545661 1 +h ctw expert ctw h:ctw 4194304 2 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 57.26 0.11313708499 57.26 57.18 57.34 57.095 0.115 0.0698569299331 0.0698569299331 298336 67.8822509939 298336 298288 298384 2.24674930722 2.24674930722 1 +h ctw expert ctw h:ctw 10000000 2 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 152.185 0.176776695297 152.185 152.06 152.31 151.765 0.29 0.0626655031573 0.0626655031573 541104 67.8822509939 541104 541056 541152 2.19747398481 2.19747398481 1 +h match expert match h:match 4096 2 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0 0 0 0 0 0 0 6100 33.941125497 6100 6076 6124 5.57734993252 5.57734993252 1 +h match expert match h:match 16384 2 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0 0 0 0 0 0 0 6158 172.53405461 6158 6036 6280 6.58586286407 6.58586286407 1 +h match expert match h:match 65536 2 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.01 0 0.01 0.01 0.01 0.005 0 6.25 6.25 7096 141.421356237 7096 6996 7196 6.5588255305 6.5588255305 1 +h match expert match h:match 262144 2 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.03 0 0.03 0.03 0.03 0.025 0 8.33333333333 8.33333333333 8512 118.793939239 8512 8428 8596 6.29225944669 6.29225944669 1 +h match expert match h:match 1048576 2 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.135 0.00707106781187 0.135 0.13 0.14 0.13 0 7.41758241758 7.41758241758 11868 67.8822509939 11868 11820 11916 6.23692635511 6.23692635511 1 +h match expert match h:match 2097152 2 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.27 0 0.27 0.27 0.27 0.265 0.005 7.40740740741 7.40740740741 19934 42.4264068712 19934 19904 19964 6.2741772649 6.2741772649 1 +h match expert match h:match 4194304 2 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.565 0.0212132034356 0.565 0.55 0.58 0.56 0 7.08463949843 7.08463949843 21832 90.5096679919 21832 21768 21896 6.28038800899 6.28038800899 1 +h match expert match h:match 10000000 2 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1.385 0.0494974746831 1.385 1.35 1.42 1.36 0.015 6.89013525416 6.89013525416 40902 296.984848098 40902 40692 41112 6.28744436593 6.28744436593 1 +h neural_mixture mixture neural-mixture h:neural_mixture 4096 2 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.11 0 0.11 0.11 0.11 0.105 0 0.0355113636364 0.0355113636364 12730 138.592929113 12730 12632 12828 1.91857366907 1.91857366907 1 +h neural_mixture mixture neural-mixture h:neural_mixture 16384 2 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.475 0.00707106781187 0.475 0.47 0.48 0.46 0.005 0.0328983820922 0.0328983820922 30476 28.2842712475 30476 30456 30496 2.69245779825 2.69245779825 1 +h neural_mixture mixture neural-mixture h:neural_mixture 65536 2 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.07 0 2.07 2.07 2.07 2.03 0.03 0.030193236715 0.030193236715 83630 93.3380951166 83630 83564 83696 2.35902117116 2.35902117116 1 +h neural_mixture mixture neural-mixture h:neural_mixture 262144 2 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 8.935 0.00707106781187 8.935 8.93 8.94 8.84 0.08 0.0279798632666 0.0279798632666 246370 19.7989898732 246370 246356 246384 2.05795611601 2.05795611601 1 +h neural_mixture mixture neural-mixture h:neural_mixture 1048576 2 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 40.785 0.148492424049 40.785 40.68 40.89 40.43 0.31 0.024518980703 0.024518980703 680446 25.4558441227 680446 680428 680464 1.93963669332 1.93963669332 1 +h neural_mixture mixture neural-mixture h:neural_mixture 2097152 2 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 86.71 0.0141421356237 86.71 86.7 86.72 86.155 0.475 0.0230653906885 0.0230653906885 1012094 138.592929113 1012094 1011996 1012192 1.92069854644 1.92069854644 1 +h neural_mixture mixture neural-mixture h:neural_mixture 4194304 2 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 184.22 0.282842712475 184.22 184.02 184.42 183.395 0.65 0.0217131946294 0.0217131946294 1391550 8567.30576086 1391550 1385492 1397608 1.88509670528 1.88509670528 1 +h neural_mixture mixture neural-mixture h:neural_mixture 10000000 2 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 472.38 0.551543289325 472.38 471.99 472.77 470.74 1.195 0.0201887244688 0.0201887244688 2509974 87.6812408671 2509974 2509912 2510036 1.81617547371 1.81617547371 1 +h ppmd expert ppmd h:ppmd 4096 2 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.005 0.00707106781187 0.005 0 0.01 0 0 7892 288.499566724 7892 7688 8096 2.02916276221 2.02916276221 1 +h ppmd expert ppmd h:ppmd 16384 2 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.02 0 0.02 0.02 0.02 0.02 0 0.78125 0.78125 15040 79.1959594929 15040 14984 15096 3.05111741489 3.05111741489 1 +h ppmd expert ppmd h:ppmd 65536 2 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.11 0 0.11 0.11 0.11 0.095 0.015 0.568181818182 0.568181818182 39072 28.2842712475 39072 39052 39092 2.82800309907 2.82800309907 1 +h ppmd expert ppmd h:ppmd 262144 2 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.525 0.00707106781187 0.525 0.52 0.53 0.46 0.055 0.476233671988 0.476233671988 127136 135.764501988 127136 127040 127232 2.54059561824 2.54059561824 1 +h ppmd expert ppmd h:ppmd 1048576 2 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.385 0.00707106781187 2.385 2.38 2.39 2.185 0.195 0.419289054534 0.419289054534 370306 257.386868352 370306 370124 370488 2.48956475398 2.48956475398 1 +h ppmd expert ppmd h:ppmd 2097152 2 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 4.985 0.0777817459305 4.985 4.93 5.04 4.76 0.215 0.401252455005 0.401252455005 434136 135.764501988 434136 434040 434232 2.52508057302 2.52508057302 1 +h ppmd expert ppmd h:ppmd 4194304 2 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 10.03 0 10.03 10.03 10.03 9.8 0.21 0.398803589232 0.398803589232 436256 130.107647738 436256 436164 436348 2.5353209835 2.5353209835 1 +h ppmd expert ppmd h:ppmd 10000000 2 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 24.395 0.0212132034356 24.395 24.38 24.41 24.08 0.285 0.390930386132 0.390930386132 550864 124.450793489 550864 550776 550952 2.52355814767 2.52355814767 1 +h rosa expert rosaplus h:rosa 4096 2 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.04 0.0141421356237 0.04 0.03 0.05 0.035 0 0.104166666667 0.104166666667 6310 82.0243866176 6310 6252 6368 2.11172124286 2.11172124286 1 +h rosa expert rosaplus h:rosa 16384 2 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.13 0 0.13 0.13 0.13 0.125 0 0.120192307692 0.120192307692 8398 82.0243866176 8398 8340 8456 3.48630622986 3.48630622986 1 +h rosa expert rosaplus h:rosa 65536 2 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.635 0.00707106781187 0.635 0.63 0.64 0.615 0.015 0.0984312996032 0.0984312996032 15492 22.627416998 15492 15476 15508 3.19056696201 3.19056696201 1 +h rosa expert rosaplus h:rosa 262144 2 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3.55 0.0141421356237 3.55 3.54 3.56 3.475 0.065 0.0704230940138 0.0704230940138 44186 14.1421356237 44186 44176 44196 2.90567254487 2.90567254487 1 +h rosa expert rosaplus h:rosa 1048576 2 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 19.54 0.0141421356237 19.54 19.53 19.55 19.115 0.405 0.0511770860752 0.0511770860752 194732 33.941125497 194732 194708 194756 2.80123027825 2.80123027825 1 +h rosa expert rosaplus h:rosa 2097152 2 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 46.95 0.0565685424949 46.95 46.91 46.99 45.23 1.66 0.0425985399725 0.0425985399725 317178 98.9949493661 317178 317108 317248 2.70437138702 2.70437138702 1 +h rosa expert rosaplus h:rosa 4194304 2 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 113.21 0 113.21 113.21 113.21 108.315 4.765 0.0353325677944 0.0353325677944 626334 155.563491861 626334 626224 626444 2.62099230176 2.62099230176 1 +h rosa expert rosaplus h:rosa 10000000 2 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 334.735 0.275771644663 334.735 334.54 334.93 322.01 12.35 0.0284904369143 0.0284904369143 1422640 192.333044483 1422640 1422504 1422776 2.48481897791 2.48481897791 1 +h rwkv7 expert rwkv7 h:rwkv7 4096 2 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.07 0.0282842712475 0.07 0.05 0.09 0.065 0 0.0607638888889 0.0607638888889 8774 70.7106781187 8774 8724 8824 7.21701437947 7.21701437947 1 +h rwkv7 expert rwkv7 h:rwkv7 16384 2 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.22 0.0282842712475 0.22 0.2 0.24 0.215 0 0.0716145833333 0.0716145833333 8942 212.132034356 8942 8792 9092 5.85021072042 5.85021072042 1 +h rwkv7 expert rwkv7 h:rwkv7 65536 2 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.805 0.00707106781187 0.805 0.8 0.81 0.805 0 0.0776427469136 0.0776427469136 8644 135.764501988 8644 8548 8740 4.31173851819 4.31173851819 1 +h rwkv7 expert rwkv7 h:rwkv7 262144 2 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3.315 0.0494974746831 3.315 3.28 3.35 3.305 0 0.0754231889334 0.0754231889334 9030 70.7106781187 9030 8980 9080 4.18065126521 4.18065126521 1 +h rwkv7 expert rwkv7 h:rwkv7 1048576 2 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 13.34 0.127279220614 13.34 13.25 13.43 13.33 0 0.0749659309628 0.0749659309628 9568 0 9568 9568 9568 3.61829048151 3.61829048151 1 +h rwkv7 expert rwkv7 h:rwkv7 2097152 2 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 27.435 0.19091883092 27.435 27.3 27.57 27.415 0 0.0729013460243 0.0729013460243 10528 181.019335984 10528 10400 10656 3.49833649966 3.49833649966 1 +h rwkv7 expert rwkv7 h:rwkv7 4194304 2 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 55.58 0.707106781187 55.58 55.08 56.08 55.545 0 0.071974158713 0.071974158713 12626 19.7989898732 12626 12612 12640 3.33570098711 3.33570098711 1 +h rwkv7 expert rwkv7 h:rwkv7 10000000 2 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 132.805 0.487903679019 132.805 132.46 133.15 132.725 0 0.0718106059525 0.0718106059525 18228 209.303607231 18228 18080 18376 3.17490278117 3.17490278117 1 +compress ctw expert ctw compress:ctw 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.04 0 0.04 0.04 0.04 0.03 0 0.09765625 0.09765625 7412 130.107647738 7412 7320 7504 1332 1332 0.3251953125 0.3251953125 1 +compress ctw expert ctw compress:ctw 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.18 0 0.18 0.18 0.18 0.175 0 0.0868055555556 0.0868055555556 12582 87.6812408671 12582 12520 12644 6579 6579 0.401550292969 0.401550292969 1 +compress ctw expert ctw compress:ctw 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.805 0.00707106781187 0.805 0.8 0.81 0.795 0 0.0776427469136 0.0776427469136 22888 5.65685424949 22888 22884 22892 22728 22728 0.346801757812 0.346801757812 1 +compress ctw expert ctw compress:ctw 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3.48 0 3.48 3.48 3.48 3.465 0.01 0.0718390804598 0.0718390804598 48374 115.965512115 48374 48292 48456 80004 80004 0.305191040039 0.305191040039 1 +compress ctw expert ctw compress:ctw 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 15.655 0.0212132034356 15.655 15.64 15.67 15.595 0.045 0.0638774141215 0.0638774141215 116394 76.3675323681 116394 116340 116448 301713 301713 0.287735939026 0.287735939026 1 +compress ctw expert ctw compress:ctw 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 33.16 0.0141421356237 33.16 33.15 33.17 33.065 0.065 0.0603136363657 0.0603136363657 186602 65.0538238692 186602 186556 186648 597003 597003 0.284673213959 0.284673213959 1 +compress ctw expert ctw compress:ctw 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 69.915 0.0494974746831 69.915 69.88 69.95 69.72 0.13 0.0572123435948 0.0572123435948 300720 226.27416998 300720 300560 300880 1177962 1177962 0.280848026276 0.280848026276 1 +compress ctw expert ctw compress:ctw 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 177.645 0.0777817459305 177.645 177.59 177.7 177.23 0.26 0.0536842808873 0.0536842808873 546652 0 546652 546652 546652 2746861 2746861 0.2746861 0.2746861 1 +compress match expert match compress:match 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0 0 0 0 0 0 0 5980 84.8528137424 5980 5920 6040 2874 2874 0.70166015625 0.70166015625 1 +compress match expert match compress:match 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.02 0.0141421356237 0.02 0.01 0.03 0.01 0 1.04166666667 1.04166666667 5976 5.65685424949 5976 5972 5980 13506 13506 0.824340820312 0.824340820312 1 +compress match expert match compress:match 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.03 0 0.03 0.03 0.03 0.03 0 2.08333333333 2.08333333333 6738 36.7695526217 6738 6712 6764 53748 53748 0.820129394531 0.820129394531 1 +compress match expert match compress:match 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.13 0 0.13 0.13 0.13 0.13 0 1.92307692308 1.92307692308 8572 164.048773235 8572 8456 8688 206203 206203 0.786602020264 0.786602020264 1 +compress match expert match compress:match 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.525 0.00707106781187 0.525 0.52 0.53 0.525 0 1.90493468795 1.90493468795 12452 84.8528137424 12452 12392 12512 817505 817505 0.779633522034 0.779633522034 1 +compress match expert match compress:match 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1.05 0 1.05 1.05 1.05 1.04 0.01 1.90476190476 1.90476190476 22556 33.941125497 22556 22532 22580 1644756 1644756 0.784280776978 0.784280776978 1 +compress match expert match compress:match 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.11 0 2.11 2.11 2.11 2.1 0.005 1.89573459716 1.89573459716 29472 73.5391052434 29472 29420 29524 3292751 3292751 0.785053014755 0.785053014755 1 +compress match expert match compress:match 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 5.13 0.0424264068712 5.13 5.1 5.16 5.11 0.01 1.85907783978 1.85907783978 55932 0 55932 55932 55932 7859324 7859324 0.7859324 0.7859324 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.12 0 0.12 0.12 0.12 0.12 0 0.0325520833333 0.0325520833333 12686 19.7989898732 12686 12672 12700 1001 1001 0.244384765625 0.244384765625 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.52 0 0.52 0.52 0.52 0.505 0.005 0.0300480769231 0.0300480769231 31058 8.48528137424 31058 31052 31064 5533 5533 0.337707519531 0.337707519531 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.265 0.00707106781187 2.265 2.26 2.27 2.24 0.015 0.0275939534521 0.0275939534521 84606 48.0832611207 84606 84572 84640 19344 19344 0.295166015625 0.295166015625 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 9.87 0.0565685424949 9.87 9.83 9.91 9.765 0.08 0.0253296966698 0.0253296966698 243616 113.13708499 243616 243536 243696 67454 67454 0.257316589355 0.257316589355 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 45.09 0.0424264068712 45.09 45.06 45.12 44.74 0.3 0.0221778763068 0.0221778763068 681028 79.1959594929 681028 680972 681084 254251 254251 0.242472648621 0.242472648621 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 96.14 0.169705627485 96.14 96.02 96.26 95.565 0.48 0.0208030280415 0.0208030280415 1004846 82.0243866176 1004846 1004788 1004904 503520 503520 0.240097045898 0.240097045898 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 205.8 0.183847763109 205.8 205.67 205.93 204.97 0.625 0.0194363537225 0.0194363537225 1392178 132.936074863 1392178 1392084 1392272 988355 988355 0.235642194748 0.235642194748 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 535.7 0.197989898732 535.7 535.56 535.84 533.905 1.23 0.0178023965193 0.0178023965193 2492666 104.651803616 2492666 2492592 2492740 2270248 2270248 0.2270248 0.2270248 1 +compress ppmd expert ppmd compress:ppmd 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0 0 0 0 0 0 0 7680 135.764501988 7680 7584 7776 1058 1058 0.25830078125 0.25830078125 1 +compress ppmd expert ppmd compress:ppmd 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.03 0 0.03 0.03 0.03 0.02 0 0.520833333333 0.520833333333 15414 200.818325857 15414 15272 15556 6267 6267 0.382507324219 0.382507324219 1 +compress ppmd expert ppmd compress:ppmd 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.14 0 0.14 0.14 0.14 0.13 0.005 0.446428571429 0.446428571429 39454 25.4558441227 39454 39436 39472 23186 23186 0.353790283203 0.353790283203 1 +compress ppmd expert ppmd compress:ppmd 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.62 0 0.62 0.62 0.62 0.545 0.065 0.403225806452 0.403225806452 127378 8.48528137424 127378 127372 127384 83269 83269 0.317646026611 0.317646026611 1 +compress ppmd expert ppmd compress:ppmd 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.745 0.00707106781187 2.745 2.74 2.75 2.575 0.16 0.364299933643 0.364299933643 370650 98.9949493661 370650 370580 370720 326331 326331 0.311213493347 0.311213493347 1 +compress ppmd expert ppmd compress:ppmd 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 5.655 0.00707106781187 5.655 5.65 5.66 5.45 0.19 0.353669595672 0.353669595672 434416 141.421356237 434416 434316 434516 661953 661953 0.315643787384 0.315643787384 1 +compress ppmd expert ppmd compress:ppmd 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 11.63 0.0848528137424 11.63 11.57 11.69 11.4 0.215 0.343947245649 0.343947245649 436632 0 436632 436632 436632 1329257 1329257 0.316919565201 0.316919565201 1 +compress ppmd expert ppmd compress:ppmd 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 27.96 0.197989898732 27.96 27.82 28.1 27.645 0.285 0.341093786527 0.341093786527 550896 181.019335984 550896 550768 551024 3154465 3154465 0.3154465 0.3154465 1 +compress rosa expert rosaplus compress:rosa 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0 0 0 0 0 0 0 6242 25.4558441227 6242 6224 6260 1127 1127 0.275146484375 0.275146484375 1 +compress rosa expert rosaplus compress:rosa 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.02 0 0.02 0.02 0.02 0.02 0 0.78125 0.78125 7934 42.4264068712 7934 7904 7964 6359 6359 0.388122558594 0.388122558594 1 +compress rosa expert rosaplus compress:rosa 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.12 0 0.12 0.12 0.12 0.12 0 0.520833333333 0.520833333333 16464 16.9705627485 16464 16452 16476 22843 22843 0.348556518555 0.348556518555 1 +compress rosa expert rosaplus compress:rosa 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.7 0 0.7 0.7 0.7 0.68 0.02 0.357142857143 0.357142857143 46628 50.9116882454 46628 46592 46664 80590 80590 0.307426452637 0.307426452637 1 +compress rosa expert rosaplus compress:rosa 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3.88 0.0282842712475 3.88 3.86 3.9 3.815 0.055 0.257738806962 0.257738806962 146192 107.48023074 146192 146116 146268 306522 306522 0.292322158813 0.292322158813 1 +compress rosa expert rosaplus compress:rosa 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 9.385 0.0494974746831 9.385 9.35 9.42 9.255 0.115 0.213108984184 0.213108984184 281492 107.48023074 281492 281416 281568 608681 608681 0.290241718292 0.290241718292 1 +compress rosa expert rosaplus compress:rosa 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 23.475 0.120208152802 23.475 23.39 23.56 23.225 0.225 0.170396270227 0.170396270227 538174 14.1421356237 538174 538164 538184 1199345 1199345 0.285946130753 0.285946130753 1 +compress rosa expert rosaplus compress:rosa 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 73.765 0.459619407771 73.765 73.44 74.09 73.085 0.605 0.129287986079 0.129287986079 1263352 147.078210487 1263352 1263248 1263456 2752778 2752778 0.2752778 0.2752778 1 +compress rwkv7 expert rwkv7 compress:rwkv7 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.06 0 0.06 0.06 0.06 0.05 0 0.0651041666667 0.0651041666667 8258 138.592929113 8258 8160 8356 3714 3714 0.90673828125 0.90673828125 1 +compress rwkv7 expert rwkv7 compress:rwkv7 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.215 0.00707106781187 0.215 0.21 0.22 0.21 0 0.0727137445887 0.0727137445887 8186 121.622366364 8186 8100 8272 12000 12000 0.732421875 0.732421875 1 +compress rwkv7 expert rwkv7 compress:rwkv7 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.845 0.00707106781187 0.845 0.84 0.85 0.84 0 0.0739670868347 0.0739670868347 8310 127.279220614 8310 8220 8400 35340 35340 0.539245605469 0.539245605469 1 +compress rwkv7 expert rwkv7 compress:rwkv7 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3.33 0.0282842712475 3.33 3.31 3.35 3.325 0 0.075077783289 0.075077783289 8316 50.9116882454 8316 8280 8352 137010 137010 0.522651672363 0.522651672363 1 +compress rwkv7 expert rwkv7 compress:rwkv7 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 13.885 0.0919238815543 13.885 13.82 13.95 13.87 0 0.0720217439792 0.0720217439792 9750 2.82842712475 9750 9748 9752 474275 474275 0.452303886414 0.452303886414 1 +compress rwkv7 expert rwkv7 compress:rwkv7 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 28.4 0.0989949493661 28.4 28.33 28.47 28.38 0 0.0704229630437 0.0704229630437 11540 16.9705627485 11540 11528 11552 917086 917086 0.437300682068 0.437300682068 1 +compress rwkv7 expert rwkv7 compress:rwkv7 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 56.465 0.813172798365 56.465 55.89 57.04 56.425 0 0.0708476904519 0.0708476904519 15456 0 15456 15456 15456 1748887 1748887 0.416967153549 0.416967153549 1 +compress rwkv7 expert rwkv7 compress:rwkv7 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 136.795 1.18086832458 136.795 135.96 137.63 136.7 0 0.0697181805467 0.0697181805467 25350 25.4558441227 25350 25332 25368 3968645 3968645 0.3968645 0.3968645 1 +decompress ctw expert ctw decompress:ctw 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.03 0 0.03 0.03 0.03 0.03 0 0.130208333333 0.130208333333 7584 90.5096679919 7584 7520 7648 1332 1332 0.3251953125 0.3251953125 1 +decompress ctw expert ctw decompress:ctw 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.18 0 0.18 0.18 0.18 0.175 0 0.0868055555556 0.0868055555556 12482 31.1126983722 12482 12460 12504 6579 6579 0.401550292969 0.401550292969 1 +decompress ctw expert ctw decompress:ctw 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.805 0.00707106781187 0.805 0.8 0.81 0.795 0.005 0.0776427469136 0.0776427469136 22946 110.308657865 22946 22868 23024 22728 22728 0.346801757812 0.346801757812 1 +decompress ctw expert ctw decompress:ctw 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3.495 0.00707106781187 3.495 3.49 3.5 3.475 0.01 0.0715309046255 0.0715309046255 48536 147.078210487 48536 48432 48640 80004 80004 0.305191040039 0.305191040039 1 +decompress ctw expert ctw decompress:ctw 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 15.725 0.0353553390593 15.725 15.7 15.75 15.635 0.065 0.063593165504 0.063593165504 116196 5.65685424949 116196 116192 116200 301713 301713 0.287735939026 0.287735939026 1 +decompress ctw expert ctw decompress:ctw 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 33.3 0.0707106781187 33.3 33.25 33.35 33.18 0.085 0.0600601954662 0.0600601954662 185876 56.5685424949 185876 185836 185916 597003 597003 0.284673213959 0.284673213959 1 +decompress ctw expert ctw decompress:ctw 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 70.315 0.0353553390593 70.315 70.29 70.34 70.1 0.145 0.0568868734359 0.0568868734359 299526 127.279220614 299526 299436 299616 1177962 1177962 0.280848026276 0.280848026276 1 +decompress ctw expert ctw decompress:ctw 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 179.24 0.961665222414 179.24 178.56 179.92 178.8 0.28 0.0532073221787 0.0532073221787 543766 70.7106781187 543766 543716 543816 2746861 2746861 0.2746861 0.2746861 1 +decompress match expert match decompress:match 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0 0 0 0 0 0 0 5760 141.421356237 5760 5660 5860 2874 2874 0.70166015625 0.70166015625 1 +decompress match expert match decompress:match 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.015 0.00707106781187 0.015 0.01 0.02 0.015 0 1.171875 1.171875 6116 79.1959594929 6116 6060 6172 13506 13506 0.824340820312 0.824340820312 1 +decompress match expert match decompress:match 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.03 0 0.03 0.03 0.03 0.03 0 2.08333333333 2.08333333333 6874 76.3675323681 6874 6820 6928 53748 53748 0.820129394531 0.820129394531 1 +decompress match expert match decompress:match 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.14 0 0.14 0.14 0.14 0.14 0 1.78571428571 1.78571428571 8500 16.9705627485 8500 8488 8512 206203 206203 0.786602020264 0.786602020264 1 +decompress match expert match decompress:match 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.55 0 0.55 0.55 0.55 0.55 0 1.81818181818 1.81818181818 12020 22.627416998 12020 12004 12036 817505 817505 0.779633522034 0.779633522034 1 +decompress match expert match decompress:match 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 1.12 0.0141421356237 1.12 1.11 1.13 1.105 0.005 1.78585665311 1.78585665311 20662 19.7989898732 20662 20648 20676 1644756 1644756 0.784280776978 0.784280776978 1 +decompress match expert match decompress:match 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.24 0.0141421356237 2.24 2.23 2.25 2.225 0.005 1.78574987544 1.78574987544 24324 113.13708499 24324 24244 24404 3292751 3292751 0.785053014755 0.785053014755 1 +decompress match expert match decompress:match 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 5.37 0.0141421356237 5.37 5.36 5.38 5.345 0.015 1.77593598426 1.77593598426 46500 28.2842712475 46500 46480 46520 7859324 7859324 0.7859324 0.7859324 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.12 0 0.12 0.12 0.12 0.115 0 0.0325520833333 0.0325520833333 12900 5.65685424949 12900 12896 12904 1001 1001 0.244384765625 0.244384765625 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.525 0.00707106781187 0.525 0.52 0.53 0.51 0.01 0.0297646044993 0.0297646044993 30860 28.2842712475 30860 30840 30880 5533 5533 0.337707519531 0.337707519531 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.275 0.0212132034356 2.275 2.26 2.29 2.25 0.02 0.0274737218379 0.0274737218379 87660 16.9705627485 87660 87648 87672 19344 19344 0.295166015625 0.295166015625 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 9.92 0.0141421356237 9.92 9.91 9.93 9.815 0.09 0.025201638513 0.025201638513 248648 181.019335984 248648 248520 248776 67454 67454 0.257316589355 0.257316589355 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 45.08 0.169705627485 45.08 44.96 45.2 44.71 0.32 0.0221829433439 0.0221829433439 679322 223.445742855 679322 679164 679480 254251 254251 0.242472648621 0.242472648621 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 96.02 0.0141421356237 96.02 96.01 96.03 95.44 0.49 0.0208289941855 0.0208289941855 1010446 121.622366364 1010446 1010360 1010532 503520 503520 0.240097045898 0.240097045898 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 206.015 0.0636396103068 206.015 205.97 206.06 205.23 0.585 0.0194160628636 0.0194160628636 1387302 8.48528137424 1387302 1387296 1387308 988355 988355 0.235642194748 0.235642194748 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 535.465 0.700035713375 535.465 534.97 535.96 533.7 1.23 0.0178102234766 0.0178102234766 2499868 56.5685424949 2499868 2499828 2499908 2270248 2270248 0.2270248 0.2270248 1 +decompress ppmd expert ppmd decompress:ppmd 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.005 0.00707106781187 0.005 0 0.01 0 0 7718 172.53405461 7718 7596 7840 1058 1058 0.25830078125 0.25830078125 1 +decompress ppmd expert ppmd decompress:ppmd 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.03 0 0.03 0.03 0.03 0.025 0.005 0.520833333333 0.520833333333 15320 118.793939239 15320 15236 15404 6267 6267 0.382507324219 0.382507324219 1 +decompress ppmd expert ppmd decompress:ppmd 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.14 0 0.14 0.14 0.14 0.12 0.015 0.446428571429 0.446428571429 39566 127.279220614 39566 39476 39656 23186 23186 0.353790283203 0.353790283203 1 +decompress ppmd expert ppmd decompress:ppmd 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.62 0 0.62 0.62 0.62 0.55 0.07 0.403225806452 0.403225806452 127360 243.244732728 127360 127188 127532 83269 83269 0.317646026611 0.317646026611 1 +decompress ppmd expert ppmd decompress:ppmd 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 2.825 0.00707106781187 2.825 2.82 2.83 2.62 0.195 0.353983409769 0.353983409769 370980 101.823376491 370980 370908 371052 326331 326331 0.311213493347 0.311213493347 1 +decompress ppmd expert ppmd decompress:ppmd 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 5.745 0.00707106781187 5.745 5.74 5.75 5.525 0.21 0.348129071353 0.348129071353 435108 113.13708499 435108 435028 435188 661953 661953 0.315643787384 0.315643787384 1 +decompress ppmd expert ppmd decompress:ppmd 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 11.645 0.00707106781187 11.645 11.64 11.65 11.4 0.23 0.343495125584 0.343495125584 435734 82.0243866176 435734 435676 435792 1329257 1329257 0.316919565201 0.316919565201 1 +decompress ppmd expert ppmd decompress:ppmd 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 28.425 0.275771644663 28.425 28.23 28.62 28.115 0.28 0.335521266534 0.335521266534 553090 110.308657865 553090 553012 553168 3154465 3154465 0.3154465 0.3154465 1 +decompress rosa expert rosaplus decompress:rosa 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0 0 0 0 0 0 0 6262 319.612265096 6262 6036 6488 1127 1127 0.275146484375 0.275146484375 1 +decompress rosa expert rosaplus decompress:rosa 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.02 0 0.02 0.02 0.02 0.02 0 0.78125 0.78125 8008 33.941125497 8008 7984 8032 6359 6359 0.388122558594 0.388122558594 1 +decompress rosa expert rosaplus decompress:rosa 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.125 0.00707106781187 0.125 0.12 0.13 0.125 0 0.500801282051 0.500801282051 16414 93.3380951166 16414 16348 16480 22843 22843 0.348556518555 0.348556518555 1 +decompress rosa expert rosaplus decompress:rosa 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.71 0 0.71 0.71 0.71 0.69 0.01 0.352112676056 0.352112676056 46832 22.627416998 46832 46816 46848 80590 80590 0.307426452637 0.307426452637 1 +decompress rosa expert rosaplus decompress:rosa 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3.91 0 3.91 3.91 3.91 3.85 0.045 0.255754475703 0.255754475703 146464 181.019335984 146464 146336 146592 306522 306522 0.292322158813 0.292322158813 1 +decompress rosa expert rosaplus decompress:rosa 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 9.445 0.0919238815543 9.445 9.38 9.51 9.305 0.125 0.211762279185 0.211762279185 278252 16.9705627485 278252 278240 278264 608681 608681 0.290241718292 0.290241718292 1 +decompress rosa expert rosaplus decompress:rosa 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 23.555 0.148492424049 23.555 23.45 23.66 23.285 0.245 0.169818700243 0.169818700243 536012 56.5685424949 536012 535972 536052 1199345 1199345 0.285946130753 0.285946130753 1 +decompress rosa expert rosaplus decompress:rosa 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 74.38 0.480832611207 74.38 74.04 74.72 73.73 0.58 0.128219177735 0.128219177735 1262788 39.5979797464 1262788 1262760 1262816 2752778 2752778 0.2752778 0.2752778 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.055 0.00707106781187 0.055 0.05 0.06 0.055 0 0.0716145833333 0.0716145833333 8336 50.9116882454 8336 8300 8372 3714 3714 0.90673828125 0.90673828125 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.21 0 0.21 0.21 0.21 0.21 0 0.0744047619048 0.0744047619048 8202 200.818325857 8202 8060 8344 12000 12000 0.732421875 0.732421875 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 0.855 0.00707106781187 0.855 0.85 0.86 0.85 0 0.0731019151847 0.0731019151847 8306 98.9949493661 8306 8236 8376 35340 35340 0.539245605469 0.539245605469 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 3.365 0.00707106781187 3.365 3.36 3.37 3.36 0 0.0742943690829 0.0742943690829 8514 104.651803616 8514 8440 8588 137010 137010 0.522651672363 0.522651672363 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 13.84 0.169705627485 13.84 13.72 13.96 13.825 0 0.0722597675992 0.0722597675992 9408 135.764501988 9408 9312 9504 474275 474275 0.452303886414 0.452303886414 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 28.195 0.0353553390593 28.195 28.17 28.22 28.175 0 0.0709346186349 0.0709346186349 10974 183.847763109 10974 10844 11104 917086 917086 0.437300682068 0.437300682068 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 56.595 0.374766594029 56.595 56.33 56.86 56.555 0 0.0706791713247 0.0706791713247 13716 107.48023074 13716 13640 13792 1748887 1748887 0.416967153549 0.416967153549 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory-bench-bcb2c188/configs/bench/two.json e596d7e9d03d5a82ac5785d9e3c63db62ffd9e15d18e952283fd22a4b3522a8d native cli 137.065 0.841457069612 137.065 136.47 137.66 136.98 0 0.0695795635722 0.0695795635722 21600 90.5096679919 21600 21536 21664 3968645 3968645 0.3968645 0.3968645 1 diff --git a/benchmarks/current/infotheory-two-json-raw-20260601-021136.tsv b/benchmarks/current/infotheory-two-json-raw-20260601-021136.tsv new file mode 100644 index 00000000..5ed8291d --- /dev/null +++ b/benchmarks/current/infotheory-two-json-raw-20260601-021136.tsv @@ -0,0 +1,289 @@ +operation subject subject_kind expert_kind series size_bytes repetition cpu compression_backend input_sha256 suite_spec_path suite_spec_sha256 build_mode build_features archive_bytes entropy_bpb real_seconds user_seconds sys_seconds rss_kib verified +h neural_mixture mixture neural-mixture h:neural_mixture 4096 1 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 1.918573669072413 0.13 0.11 0.00 12560 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 1001 0.12 0.12 0.00 12632 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 1001 0.12 0.12 0.00 12888 1 +h neural_mixture mixture neural-mixture h:neural_mixture 4096 2 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 1.918573669072413 0.11 0.10 0.00 12764 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 1001 0.12 0.12 0.00 12640 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 1001 0.12 0.12 0.00 12764 1 +h fac-ctw expert fac-ctw h:fac-ctw 4096 1 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2.5654962454168757 0.02 0.02 0.00 7496 1 +compress fac-ctw expert fac-ctw compress:fac-ctw 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 1332 0.04 0.04 0.00 7584 1 +decompress fac-ctw expert fac-ctw decompress:fac-ctw 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 1332 0.03 0.03 0.00 7496 1 +h fac-ctw expert fac-ctw h:fac-ctw 4096 2 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2.5654962454168757 0.02 0.02 0.00 7464 1 +compress fac-ctw expert fac-ctw compress:fac-ctw 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 1332 0.04 0.04 0.00 7372 1 +decompress fac-ctw expert fac-ctw decompress:fac-ctw 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 1332 0.03 0.03 0.00 7636 1 +h ppmd expert ppmd h:ppmd 4096 1 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2.029162762208123 0.00 0.00 0.00 7968 1 +compress ppmd expert ppmd compress:ppmd 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 1058 0.01 0.01 0.00 7556 1 +decompress ppmd expert ppmd decompress:ppmd 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 1058 0.01 0.00 0.00 7544 1 +h ppmd expert ppmd h:ppmd 4096 2 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2.029162762208123 0.01 0.01 0.00 7888 1 +compress ppmd expert ppmd compress:ppmd 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 1058 0.02 0.01 0.00 7632 1 +decompress ppmd expert ppmd decompress:ppmd 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 1058 0.02 0.01 0.00 7680 1 +h rosa expert rosaplus h:rosa 4096 1 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2.111721242855073 0.06 0.04 0.01 6548 1 +compress rosa expert rosaplus compress:rosa 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 1127 0.00 0.00 0.00 6112 1 +decompress rosa expert rosaplus decompress:rosa 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 1127 0.00 0.00 0.00 6220 1 +h rosa expert rosaplus h:rosa 4096 2 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2.111721242855073 0.03 0.03 0.00 6472 1 +compress rosa expert rosaplus compress:rosa 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 1127 0.00 0.00 0.00 6280 1 +decompress rosa expert rosaplus decompress:rosa 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 1127 0.00 0.00 0.00 6140 1 +h match expert match h:match 4096 1 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 5.577349932524691 0.00 0.00 0.00 6120 1 +compress match expert match compress:match 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2874 0.00 0.00 0.00 5868 1 +decompress match expert match decompress:match 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2874 0.01 0.00 0.00 5860 1 +h match expert match h:match 4096 2 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 5.577349932524691 0.00 0.00 0.00 6224 1 +compress match expert match compress:match 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2874 0.00 0.00 0.00 5868 1 +decompress match expert match decompress:match 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2874 0.00 0.00 0.00 5876 1 +h rwkv7 expert rwkv7 h:rwkv7 4096 1 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 7.217014379468224 0.09 0.08 0.00 9196 1 +compress rwkv7 expert rwkv7 compress:rwkv7 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 3714 0.05 0.05 0.00 8480 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 4096 1 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 3714 0.05 0.05 0.00 8412 1 +h rwkv7 expert rwkv7 h:rwkv7 4096 2 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 7.217014379468224 0.05 0.05 0.00 8956 1 +compress rwkv7 expert rwkv7 compress:rwkv7 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 3714 0.05 0.05 0.00 8428 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 3714 0.05 0.05 0.00 8204 1 +h neural_mixture mixture neural-mixture h:neural_mixture 16384 1 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2.6924577982523554 0.47 0.45 0.01 30840 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 5533 0.52 0.51 0.01 30688 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 5533 0.52 0.51 0.00 30548 1 +h neural_mixture mixture neural-mixture h:neural_mixture 16384 2 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2.6924577982523554 0.47 0.46 0.00 31072 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 5533 0.51 0.50 0.00 30584 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 5533 0.52 0.50 0.01 30624 1 +h fac-ctw expert fac-ctw h:fac-ctw 16384 1 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 3.2034095120908512 0.10 0.10 0.00 12684 1 +compress fac-ctw expert fac-ctw compress:fac-ctw 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 6579 0.18 0.18 0.00 12772 1 +decompress fac-ctw expert fac-ctw decompress:fac-ctw 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 6579 0.18 0.17 0.00 12628 1 +h fac-ctw expert fac-ctw h:fac-ctw 16384 2 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 3.2034095120908512 0.10 0.10 0.00 12624 1 +compress fac-ctw expert fac-ctw compress:fac-ctw 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 6579 0.18 0.17 0.00 12560 1 +decompress fac-ctw expert fac-ctw decompress:fac-ctw 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 6579 0.18 0.17 0.00 12604 1 +h ppmd expert ppmd h:ppmd 16384 1 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 3.05111741488576 0.02 0.02 0.00 15204 1 +compress ppmd expert ppmd compress:ppmd 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 6267 0.03 0.03 0.00 15552 1 +decompress ppmd expert ppmd decompress:ppmd 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 6267 0.03 0.02 0.00 15348 1 +h ppmd expert ppmd h:ppmd 16384 2 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 3.05111741488576 0.02 0.02 0.00 15204 1 +compress ppmd expert ppmd compress:ppmd 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 6267 0.03 0.02 0.00 15484 1 +decompress ppmd expert ppmd decompress:ppmd 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 6267 0.03 0.03 0.00 15228 1 +h rosa expert rosaplus h:rosa 16384 1 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 3.486306229863964 0.13 0.12 0.00 8460 1 +compress rosa expert rosaplus compress:rosa 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 6359 0.02 0.02 0.00 8024 1 +decompress rosa expert rosaplus decompress:rosa 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 6359 0.02 0.02 0.00 8032 1 +h rosa expert rosaplus h:rosa 16384 2 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 3.486306229863964 0.13 0.12 0.01 8396 1 +compress rosa expert rosaplus compress:rosa 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 6359 0.02 0.02 0.00 7904 1 +decompress rosa expert rosaplus decompress:rosa 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 6359 0.02 0.02 0.00 8048 1 +h match expert match h:match 16384 1 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 6.5858628640659385 0.00 0.00 0.00 6336 1 +compress match expert match compress:match 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 13506 0.01 0.01 0.00 5728 1 +decompress match expert match decompress:match 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 13506 0.01 0.01 0.00 6028 1 +h match expert match h:match 16384 2 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 6.5858628640659385 0.00 0.00 0.00 6328 1 +compress match expert match compress:match 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 13506 0.02 0.02 0.00 6048 1 +decompress match expert match decompress:match 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 13506 0.02 0.01 0.00 5856 1 +h rwkv7 expert rwkv7 h:rwkv7 16384 1 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 5.850210720420741 0.24 0.24 0.00 8928 1 +compress rwkv7 expert rwkv7 compress:rwkv7 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 12000 0.21 0.21 0.00 8428 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 16384 1 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 12000 0.21 0.21 0.00 8464 1 +h rwkv7 expert rwkv7 h:rwkv7 16384 2 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 5.850210720420741 0.20 0.20 0.00 8992 1 +compress rwkv7 expert rwkv7 compress:rwkv7 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 12000 0.21 0.21 0.00 8240 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 12000 0.21 0.21 0.00 8308 1 +h neural_mixture mixture neural-mixture h:neural_mixture 65536 1 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2.359021171161541 2.06 2.02 0.03 85408 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 19344 2.26 2.23 0.02 87672 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 19344 2.26 2.23 0.02 86452 1 +h neural_mixture mixture neural-mixture h:neural_mixture 65536 2 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2.359021171161541 2.06 2.03 0.02 85288 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 19344 2.25 2.23 0.02 87724 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 19344 2.25 2.22 0.02 86344 1 +h fac-ctw expert fac-ctw h:fac-ctw 65536 1 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2.7721389563404757 0.48 0.47 0.01 22728 1 +compress fac-ctw expert fac-ctw compress:fac-ctw 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 22728 0.80 0.79 0.01 22852 1 +decompress fac-ctw expert fac-ctw decompress:fac-ctw 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 22728 0.81 0.81 0.00 22972 1 +h fac-ctw expert fac-ctw h:fac-ctw 65536 2 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2.7721389563404757 0.48 0.47 0.00 22604 1 +compress fac-ctw expert fac-ctw compress:fac-ctw 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 22728 0.80 0.79 0.00 22744 1 +decompress fac-ctw expert fac-ctw decompress:fac-ctw 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 22728 0.80 0.79 0.00 23036 1 +h ppmd expert ppmd h:ppmd 65536 1 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2.828003099069916 0.11 0.10 0.01 39240 1 +compress ppmd expert ppmd compress:ppmd 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 23186 0.14 0.14 0.00 39592 1 +decompress ppmd expert ppmd decompress:ppmd 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 23186 0.14 0.13 0.01 39524 1 +h ppmd expert ppmd h:ppmd 65536 2 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2.828003099069916 0.12 0.10 0.01 39220 1 +compress ppmd expert ppmd compress:ppmd 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 23186 0.14 0.12 0.02 39524 1 +decompress ppmd expert ppmd decompress:ppmd 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 23186 0.14 0.11 0.03 39456 1 +h rosa expert rosaplus h:rosa 65536 1 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 3.1905669620146786 0.65 0.63 0.01 15632 1 +compress rosa expert rosaplus compress:rosa 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 22843 0.12 0.12 0.00 16340 1 +decompress rosa expert rosaplus decompress:rosa 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 22843 0.13 0.13 0.00 16360 1 +h rosa expert rosaplus h:rosa 65536 2 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 3.1905669620146786 0.64 0.63 0.01 15720 1 +compress rosa expert rosaplus compress:rosa 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 22843 0.12 0.12 0.00 16140 1 +decompress rosa expert rosaplus decompress:rosa 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 22843 0.12 0.12 0.00 16400 1 +h match expert match h:match 65536 1 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 6.558825530499182 0.01 0.01 0.00 7048 1 +compress match expert match compress:match 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 53748 0.03 0.03 0.00 6720 1 +decompress match expert match decompress:match 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 53748 0.03 0.03 0.00 7012 1 +h match expert match h:match 65536 2 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 6.558825530499182 0.01 0.01 0.00 7088 1 +compress match expert match compress:match 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 53748 0.03 0.03 0.00 6712 1 +decompress match expert match decompress:match 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 53748 0.03 0.03 0.00 6976 1 +h rwkv7 expert rwkv7 h:rwkv7 65536 1 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 4.311738518194067 0.83 0.83 0.00 9056 1 +compress rwkv7 expert rwkv7 compress:rwkv7 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 35340 0.83 0.83 0.00 8464 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 65536 1 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 35340 0.87 0.87 0.00 8344 1 +h rwkv7 expert rwkv7 h:rwkv7 65536 2 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 4.311738518194067 0.81 0.80 0.00 8752 1 +compress rwkv7 expert rwkv7 compress:rwkv7 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 35340 0.83 0.83 0.00 8328 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 35340 0.85 0.84 0.00 8252 1 +h neural_mixture mixture neural-mixture h:neural_mixture 262144 1 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2.0579561160084685 8.91 8.82 0.08 251756 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 67454 9.81 9.73 0.06 246796 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 67454 9.87 9.77 0.09 249832 1 +h neural_mixture mixture neural-mixture h:neural_mixture 262144 2 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2.0579561160084685 8.93 8.84 0.08 251652 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 67454 9.87 9.79 0.06 246852 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 67454 9.85 9.77 0.06 249952 1 +h fac-ctw expert fac-ctw h:fac-ctw 262144 1 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2.4409671415182586 2.32 2.29 0.02 48456 1 +compress fac-ctw expert fac-ctw compress:fac-ctw 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 80004 3.49 3.48 0.00 48360 1 +decompress fac-ctw expert fac-ctw decompress:fac-ctw 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 80004 3.49 3.48 0.00 48416 1 +h fac-ctw expert fac-ctw h:fac-ctw 262144 2 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2.4409671415182586 2.32 2.30 0.01 48368 1 +compress fac-ctw expert fac-ctw compress:fac-ctw 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 80004 3.49 3.47 0.01 48460 1 +decompress fac-ctw expert fac-ctw decompress:fac-ctw 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 80004 3.51 3.50 0.00 48424 1 +h ppmd expert ppmd h:ppmd 262144 1 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2.5405956182399834 0.52 0.46 0.05 127004 1 +compress ppmd expert ppmd compress:ppmd 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 83269 0.62 0.54 0.08 127296 1 +decompress ppmd expert ppmd decompress:ppmd 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 83269 0.62 0.55 0.06 127288 1 +h ppmd expert ppmd h:ppmd 262144 2 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2.5405956182399834 0.52 0.45 0.06 127120 1 +compress ppmd expert ppmd compress:ppmd 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 83269 0.63 0.55 0.08 127364 1 +decompress ppmd expert ppmd decompress:ppmd 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 83269 0.62 0.55 0.06 127164 1 +h rosa expert rosaplus h:rosa 262144 1 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2.905672544873225 3.61 3.53 0.06 44536 1 +compress rosa expert rosaplus compress:rosa 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 80590 0.70 0.68 0.01 46408 1 +decompress rosa expert rosaplus decompress:rosa 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 80590 0.70 0.69 0.01 46688 1 +h rosa expert rosaplus h:rosa 262144 2 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2.905672544873225 3.58 3.46 0.11 44308 1 +compress rosa expert rosaplus compress:rosa 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 80590 0.70 0.67 0.01 46428 1 +decompress rosa expert rosaplus decompress:rosa 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 80590 0.70 0.68 0.02 46704 1 +h match expert match h:match 262144 1 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 6.292259446690552 0.03 0.03 0.00 8376 1 +compress match expert match compress:match 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 206203 0.13 0.13 0.00 8800 1 +decompress match expert match decompress:match 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 206203 0.13 0.13 0.00 8152 1 +h match expert match h:match 262144 2 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 6.292259446690552 0.03 0.03 0.00 8536 1 +compress match expert match compress:match 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 206203 0.13 0.13 0.00 8556 1 +decompress match expert match decompress:match 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 206203 0.13 0.13 0.00 8308 1 +h rwkv7 expert rwkv7 h:rwkv7 262144 1 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 4.180651265210767 3.29 3.29 0.00 9000 1 +compress rwkv7 expert rwkv7 compress:rwkv7 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 137010 3.37 3.36 0.00 8544 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 262144 1 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 137010 3.29 3.29 0.00 8440 1 +h rwkv7 expert rwkv7 h:rwkv7 262144 2 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 4.180651265210767 3.24 3.23 0.00 9012 1 +compress rwkv7 expert rwkv7 compress:rwkv7 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 137010 3.31 3.31 0.00 8340 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 137010 3.29 3.28 0.00 8368 1 +h neural_mixture mixture neural-mixture h:neural_mixture 1048576 1 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 1.9396366933249272 40.69 40.35 0.30 679968 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 254251 44.66 44.30 0.31 692620 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 254251 44.92 44.59 0.29 679340 1 +h neural_mixture mixture neural-mixture h:neural_mixture 1048576 2 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 1.9396366933249272 40.58 40.29 0.25 679916 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 254251 44.71 44.39 0.27 692892 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 254251 44.93 44.61 0.28 679312 1 +h fac-ctw expert fac-ctw h:fac-ctw 1048576 1 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2.301742747548954 11.73 11.68 0.04 115564 1 +compress fac-ctw expert fac-ctw compress:fac-ctw 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 301713 15.70 15.63 0.06 116320 1 +decompress fac-ctw expert fac-ctw decompress:fac-ctw 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 301713 15.78 15.74 0.03 116008 1 +h fac-ctw expert fac-ctw h:fac-ctw 1048576 2 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2.301742747548954 11.70 11.65 0.04 115548 1 +compress fac-ctw expert fac-ctw compress:fac-ctw 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 301713 15.77 15.71 0.05 116260 1 +decompress fac-ctw expert fac-ctw decompress:fac-ctw 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 301713 15.76 15.70 0.04 116052 1 +h ppmd expert ppmd h:ppmd 1048576 1 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2.489564753975732 2.38 2.22 0.16 370460 1 +compress ppmd expert ppmd compress:ppmd 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 326331 2.74 2.58 0.15 370844 1 +decompress ppmd expert ppmd decompress:ppmd 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 326331 2.78 2.61 0.16 371240 1 +h ppmd expert ppmd h:ppmd 1048576 2 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2.489564753975732 2.43 2.25 0.16 370376 1 +compress ppmd expert ppmd compress:ppmd 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 326331 2.75 2.57 0.18 370616 1 +decompress ppmd expert ppmd decompress:ppmd 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 326331 2.77 2.56 0.20 371064 1 +h rosa expert rosaplus h:rosa 1048576 1 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2.801230278247886 19.75 19.31 0.42 194648 1 +compress rosa expert rosaplus compress:rosa 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 306522 3.87 3.80 0.06 145880 1 +decompress rosa expert rosaplus decompress:rosa 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 306522 3.89 3.86 0.02 146400 1 +h rosa expert rosaplus h:rosa 1048576 2 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2.801230278247886 19.68 19.28 0.38 194776 1 +compress rosa expert rosaplus compress:rosa 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 306522 3.84 3.81 0.02 146176 1 +decompress rosa expert rosaplus decompress:rosa 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 306522 3.88 3.84 0.04 146428 1 +h match expert match h:match 1048576 1 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 6.236926355106452 0.13 0.13 0.00 12188 1 +compress match expert match compress:match 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 817505 0.52 0.51 0.00 12636 1 +decompress match expert match decompress:match 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 817505 0.54 0.54 0.00 11888 1 +h match expert match h:match 1048576 2 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 6.236926355106452 0.13 0.13 0.00 11896 1 +compress match expert match compress:match 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 817505 0.53 0.53 0.00 12684 1 +decompress match expert match decompress:match 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 817505 0.54 0.54 0.00 11940 1 +h rwkv7 expert rwkv7 h:rwkv7 1048576 1 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 3.6182904815139514 13.44 13.43 0.00 9728 1 +compress rwkv7 expert rwkv7 compress:rwkv7 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 474275 13.64 13.63 0.00 9996 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 1048576 1 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 474275 14.14 14.13 0.00 9380 1 +h rwkv7 expert rwkv7 h:rwkv7 1048576 2 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 3.6182904815139514 13.15 13.14 0.00 9724 1 +compress rwkv7 expert rwkv7 compress:rwkv7 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 474275 13.39 13.38 0.00 9980 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 474275 13.87 13.86 0.00 9288 1 +h neural_mixture mixture neural-mixture h:neural_mixture 2097152 1 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 1.9206985464400694 86.51 85.95 0.47 1011720 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 2097152 1 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 503520 95.66 95.05 0.52 1017140 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 2097152 1 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 503520 96.01 95.48 0.43 999676 1 +h neural_mixture mixture neural-mixture h:neural_mixture 2097152 2 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 1.9206985464400694 86.41 85.90 0.43 1011616 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 503520 95.75 95.18 0.47 1016952 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 503520 95.92 95.42 0.41 999576 1 +h fac-ctw expert fac-ctw h:fac-ctw 2097152 1 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2.277315456609421 26.08 25.97 0.08 185652 1 +compress fac-ctw expert fac-ctw compress:fac-ctw 2097152 1 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 597003 33.24 33.11 0.09 186784 1 +decompress fac-ctw expert fac-ctw decompress:fac-ctw 2097152 1 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 597003 33.40 33.30 0.07 186064 1 +h fac-ctw expert fac-ctw h:fac-ctw 2097152 2 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2.277315456609421 26.10 26.00 0.08 185568 1 +compress fac-ctw expert fac-ctw compress:fac-ctw 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 597003 33.23 33.11 0.09 186512 1 +decompress fac-ctw expert fac-ctw decompress:fac-ctw 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 597003 33.33 33.22 0.07 186116 1 +h ppmd expert ppmd h:ppmd 2097152 1 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2.525080573022069 4.96 4.78 0.17 434228 1 +compress ppmd expert ppmd compress:ppmd 2097152 1 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 661953 5.77 5.58 0.18 434388 1 +decompress ppmd expert ppmd decompress:ppmd 2097152 1 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 661953 5.78 5.55 0.21 435020 1 +h ppmd expert ppmd h:ppmd 2097152 2 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2.525080573022069 4.96 4.71 0.24 434228 1 +compress ppmd expert ppmd compress:ppmd 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 661953 5.70 5.49 0.19 434460 1 +decompress ppmd expert ppmd decompress:ppmd 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 661953 5.75 5.52 0.23 435044 1 +h rosa expert rosaplus h:rosa 2097152 1 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2.704371387016519 47.26 45.56 1.65 317332 1 +compress rosa expert rosaplus compress:rosa 2097152 1 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 608681 9.31 9.19 0.11 281596 1 +decompress rosa expert rosaplus decompress:rosa 2097152 1 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 608681 9.46 9.37 0.07 278232 1 +h rosa expert rosaplus h:rosa 2097152 2 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2.704371387016519 47.30 45.60 1.65 317352 1 +compress rosa expert rosaplus compress:rosa 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 608681 9.29 9.14 0.13 281632 1 +decompress rosa expert rosaplus decompress:rosa 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 608681 9.42 9.32 0.08 278216 1 +h match expert match h:match 2097152 1 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 6.274177264896603 0.27 0.27 0.00 19820 1 +compress match expert match compress:match 2097152 1 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 1644756 1.06 1.05 0.00 22628 1 +decompress match expert match decompress:match 2097152 1 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 1644756 1.11 1.10 0.00 20528 1 +h match expert match h:match 2097152 2 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 6.274177264896603 0.27 0.27 0.00 19888 1 +compress match expert match compress:match 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 1644756 1.06 1.05 0.01 22352 1 +decompress match expert match decompress:match 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 1644756 1.11 1.10 0.01 20504 1 +h rwkv7 expert rwkv7 h:rwkv7 2097152 1 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 3.498336499657132 26.82 26.81 0.00 10684 1 +compress rwkv7 expert rwkv7 compress:rwkv7 2097152 1 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 917086 27.61 27.59 0.00 11812 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 2097152 1 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 917086 27.90 27.87 0.00 10912 1 +h rwkv7 expert rwkv7 h:rwkv7 2097152 2 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 3.498336499657132 27.06 27.05 0.00 10912 1 +compress rwkv7 expert rwkv7 compress:rwkv7 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 917086 27.66 27.64 0.00 12056 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 917086 28.06 28.04 0.00 10912 1 +h neural_mixture mixture neural-mixture h:neural_mixture 4194304 1 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 1.8850967052831877 183.73 182.94 0.62 1394212 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 4194304 1 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 988355 204.77 203.90 0.67 1407032 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 4194304 1 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 988355 205.12 204.31 0.60 1393856 1 +h neural_mixture mixture neural-mixture h:neural_mixture 4194304 2 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 1.8850967052831877 183.82 182.99 0.67 1394372 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 988355 205.03 204.20 0.63 1407052 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 988355 205.14 204.30 0.64 1380620 1 +h fac-ctw expert fac-ctw h:fac-ctw 4194304 1 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2.246749307224066 57.49 57.29 0.15 298232 1 +compress fac-ctw expert fac-ctw compress:fac-ctw 4194304 1 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 1177962 70.41 70.22 0.13 300768 1 +decompress fac-ctw expert fac-ctw decompress:fac-ctw 4194304 1 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 1177962 70.32 70.14 0.11 299680 1 +h fac-ctw expert fac-ctw h:fac-ctw 4194304 2 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2.246749307224066 57.49 57.26 0.18 298452 1 +compress fac-ctw expert fac-ctw compress:fac-ctw 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 1177962 70.33 70.11 0.15 300512 1 +decompress fac-ctw expert fac-ctw decompress:fac-ctw 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 1177962 70.45 70.26 0.13 299560 1 +h ppmd expert ppmd h:ppmd 4194304 1 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2.5353209835043775 10.05 9.81 0.22 436300 1 +compress ppmd expert ppmd compress:ppmd 4194304 1 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 1329257 11.49 11.26 0.22 436316 1 +decompress ppmd expert ppmd decompress:ppmd 4194304 1 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 1329257 11.62 11.41 0.19 435836 1 +h ppmd expert ppmd h:ppmd 4194304 2 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2.5353209835043775 10.07 9.85 0.20 436304 1 +compress ppmd expert ppmd compress:ppmd 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 1329257 11.59 11.37 0.21 436784 1 +decompress ppmd expert ppmd decompress:ppmd 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 1329257 11.68 11.44 0.22 435636 1 +h rosa expert rosaplus h:rosa 4194304 1 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2.62099230176433 113.53 108.84 4.55 626396 1 +compress rosa expert rosaplus compress:rosa 4194304 1 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 1199345 23.16 22.93 0.21 538360 1 +decompress rosa expert rosaplus decompress:rosa 4194304 1 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 1199345 23.62 23.41 0.19 535924 1 +h rosa expert rosaplus h:rosa 4194304 2 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2.62099230176433 113.59 108.85 4.61 626388 1 +compress rosa expert rosaplus compress:rosa 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 1199345 23.44 23.16 0.26 538168 1 +decompress rosa expert rosaplus decompress:rosa 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 1199345 23.54 23.29 0.23 535744 1 +h match expert match h:match 4194304 1 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 6.280388008990573 0.54 0.53 0.01 21736 1 +compress match expert match compress:match 4194304 1 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 3292751 2.15 2.14 0.01 29556 1 +decompress match expert match decompress:match 4194304 1 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 3292751 2.21 2.20 0.00 24260 1 +h match expert match h:match 4194304 2 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 6.280388008990573 0.57 0.57 0.00 21724 1 +compress match expert match compress:match 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 3292751 2.12 2.11 0.00 29416 1 +decompress match expert match decompress:match 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 3292751 2.20 2.19 0.00 24208 1 +h rwkv7 expert rwkv7 h:rwkv7 4194304 1 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 3.3357009871104677 53.99 53.95 0.00 12864 1 +compress rwkv7 expert rwkv7 compress:rwkv7 4194304 1 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 1748887 55.62 55.58 0.00 15584 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 4194304 1 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 1748887 56.60 56.56 0.00 13804 1 +h rwkv7 expert rwkv7 h:rwkv7 4194304 2 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 3.3357009871104677 54.24 54.20 0.00 12584 1 +compress rwkv7 expert rwkv7 compress:rwkv7 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 1748887 55.23 55.19 0.00 15584 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 1748887 56.12 56.09 0.00 13780 1 +h neural_mixture mixture neural-mixture h:neural_mixture 10000000 1 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 1.8161754737084215 473.22 471.49 1.27 2510660 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 10000000 1 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2270248 531.77 530.06 1.17 2487508 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 10000000 1 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2270248 534.82 532.92 1.37 2508208 1 +h neural_mixture mixture neural-mixture h:neural_mixture 10000000 2 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 1.8161754737084215 471.49 469.84 1.20 2510796 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2270248 533.19 531.37 1.27 2487500 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2270248 534.15 532.44 1.19 2508236 1 +h fac-ctw expert fac-ctw h:fac-ctw 10000000 1 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2.1974739848061873 152.98 152.59 0.26 540976 1 +compress fac-ctw expert fac-ctw compress:fac-ctw 10000000 1 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2746861 178.16 177.73 0.28 546328 1 +decompress fac-ctw expert fac-ctw decompress:fac-ctw 10000000 1 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2746861 179.32 178.88 0.28 543820 1 +h fac-ctw expert fac-ctw h:fac-ctw 10000000 2 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2.1974739848061873 153.12 152.74 0.25 541128 1 +compress fac-ctw expert fac-ctw compress:fac-ctw 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2746861 178.84 178.36 0.32 546352 1 +decompress fac-ctw expert fac-ctw decompress:fac-ctw 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2746861 179.15 178.70 0.29 543836 1 +h ppmd expert ppmd h:ppmd 10000000 1 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2.523558147667331 24.25 23.94 0.29 550948 1 +compress ppmd expert ppmd compress:ppmd 10000000 1 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 3154465 27.65 27.35 0.28 551004 1 +decompress ppmd expert ppmd decompress:ppmd 10000000 1 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 3154465 28.16 27.86 0.27 552892 1 +h ppmd expert ppmd h:ppmd 10000000 2 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2.523558147667331 24.22 23.95 0.24 550876 1 +compress ppmd expert ppmd compress:ppmd 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 3154465 28.05 27.78 0.25 551064 1 +decompress ppmd expert ppmd decompress:ppmd 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 3154465 28.08 27.76 0.29 552888 1 +h rosa expert rosaplus h:rosa 10000000 1 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2.484818977912099 335.71 322.86 12.48 1422860 1 +compress rosa expert rosaplus compress:rosa 10000000 1 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2752778 73.83 73.26 0.50 1263648 1 +decompress rosa expert rosaplus decompress:rosa 10000000 1 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2752778 74.45 73.86 0.50 1262784 1 +h rosa expert rosaplus h:rosa 10000000 2 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2.484818977912099 336.15 323.39 12.39 1422852 1 +compress rosa expert rosaplus compress:rosa 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2752778 74.25 73.58 0.60 1263560 1 +decompress rosa expert rosaplus decompress:rosa 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2752778 75.01 74.37 0.57 1262688 1 +h match expert match h:match 10000000 1 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 6.287444365927943 1.33 1.31 0.01 40692 1 +compress match expert match compress:match 10000000 1 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 7859324 5.12 5.09 0.02 55896 1 +decompress match expert match decompress:match 10000000 1 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 7859324 5.30 5.28 0.01 46484 1 +h match expert match h:match 10000000 2 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 6.287444365927943 1.34 1.32 0.01 41096 1 +compress match expert match compress:match 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 7859324 5.08 5.06 0.01 55860 1 +decompress match expert match decompress:match 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 7859324 5.32 5.30 0.01 46296 1 +h rwkv7 expert rwkv7 h:rwkv7 10000000 1 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 3.174902781170517 131.00 130.92 0.00 18472 1 +compress rwkv7 expert rwkv7 compress:rwkv7 10000000 1 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 3968645 134.54 134.45 0.00 25684 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 10000000 1 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 3968645 135.85 135.77 0.00 21464 1 +h rwkv7 expert rwkv7 h:rwkv7 10000000 2 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 3.174902781170517 132.37 132.29 0.00 18152 1 +compress rwkv7 expert rwkv7 compress:rwkv7 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 3968645 135.84 135.75 0.01 25440 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 3968645 136.07 135.97 0.01 21844 1 diff --git a/benchmarks/current/infotheory-two-json-summary-20260601-021136.tsv b/benchmarks/current/infotheory-two-json-summary-20260601-021136.tsv new file mode 100644 index 00000000..ea2f7812 --- /dev/null +++ b/benchmarks/current/infotheory-two-json-summary-20260601-021136.tsv @@ -0,0 +1,145 @@ +operation subject subject_kind expert_kind series size_bytes repeats cpu compression_backend input_sha256 suite_spec_path suite_spec_sha256 build_mode build_features real_seconds_mean real_seconds_stdev real_seconds_median real_seconds_min real_seconds_max user_seconds_mean sys_seconds_mean throughput_mib_s_mean throughput_mib_s_median rss_kib_mean rss_kib_stdev rss_kib_median rss_kib_min rss_kib_max archive_bytes_mean archive_bytes_median archive_ratio_mean archive_ratio_median entropy_bpb_mean entropy_bpb_median verified_all +h fac-ctw expert fac-ctw h:fac-ctw 4096 2 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 0.02 0 0.02 0.02 0.02 0.02 0 0.1953125 0.1953125 7480 22.627416998 7480 7464 7496 2.56549624542 2.56549624542 1 +h fac-ctw expert fac-ctw h:fac-ctw 16384 2 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 0.1 0 0.1 0.1 0.1 0.1 0 0.15625 0.15625 12654 42.4264068712 12654 12624 12684 3.20340951209 3.20340951209 1 +h fac-ctw expert fac-ctw h:fac-ctw 65536 2 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 0.48 0 0.48 0.48 0.48 0.47 0.005 0.130208333333 0.130208333333 22666 87.6812408671 22666 22604 22728 2.77213895634 2.77213895634 1 +h fac-ctw expert fac-ctw h:fac-ctw 262144 2 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2.32 0 2.32 2.32 2.32 2.295 0.015 0.10775862069 0.10775862069 48412 62.2253967444 48412 48368 48456 2.44096714152 2.44096714152 1 +h fac-ctw expert fac-ctw h:fac-ctw 1048576 2 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 11.715 0.0212132034356 11.715 11.7 11.73 11.665 0.04 0.0853607886856 0.0853607886856 115556 11.313708499 115556 115548 115564 2.30174274755 2.30174274755 1 +h fac-ctw expert fac-ctw h:fac-ctw 2097152 2 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 26.09 0.0141421356237 26.09 26.08 26.1 25.985 0.08 0.0766577345274 0.0766577345274 185610 59.3969696197 185610 185568 185652 2.27731545661 2.27731545661 1 +h fac-ctw expert fac-ctw h:fac-ctw 4194304 2 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 57.49 0 57.49 57.49 57.49 57.275 0.165 0.0695773177944 0.0695773177944 298342 155.563491861 298342 298232 298452 2.24674930722 2.24674930722 1 +h fac-ctw expert fac-ctw h:fac-ctw 10000000 2 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 153.05 0.0989949493661 153.05 152.98 153.12 152.665 0.255 0.0623113045345 0.0623113045345 541052 107.48023074 541052 540976 541128 2.19747398481 2.19747398481 1 +h match expert match h:match 4096 2 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 0 0 0 0 0 0 0 6172 73.5391052434 6172 6120 6224 5.57734993252 5.57734993252 1 +h match expert match h:match 16384 2 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 0 0 0 0 0 0 0 6332 5.65685424949 6332 6328 6336 6.58586286407 6.58586286407 1 +h match expert match h:match 65536 2 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 0.01 0 0.01 0.01 0.01 0.01 0 6.25 6.25 7068 28.2842712475 7068 7048 7088 6.5588255305 6.5588255305 1 +h match expert match h:match 262144 2 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 0.03 0 0.03 0.03 0.03 0.03 0 8.33333333333 8.33333333333 8456 113.13708499 8456 8376 8536 6.29225944669 6.29225944669 1 +h match expert match h:match 1048576 2 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 0.13 0 0.13 0.13 0.13 0.13 0 7.69230769231 7.69230769231 12042 206.475180106 12042 11896 12188 6.23692635511 6.23692635511 1 +h match expert match h:match 2097152 2 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 0.27 0 0.27 0.27 0.27 0.27 0 7.40740740741 7.40740740741 19854 48.0832611207 19854 19820 19888 6.2741772649 6.2741772649 1 +h match expert match h:match 4194304 2 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 0.555 0.0212132034356 0.555 0.54 0.57 0.55 0.005 7.21247563353 7.21247563353 21730 8.48528137424 21730 21724 21736 6.28038800899 6.28038800899 1 +h match expert match h:match 10000000 2 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 1.335 0.00707106781187 1.335 1.33 1.34 1.315 0.01 7.14372804625 7.14372804625 40894 285.671139599 40894 40692 41096 6.28744436593 6.28744436593 1 +h neural_mixture mixture neural-mixture h:neural_mixture 4096 2 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 0.12 0.0141421356237 0.12 0.11 0.13 0.105 0 0.0327797202797 0.0327797202797 12662 144.249783362 12662 12560 12764 1.91857366907 1.91857366907 1 +h neural_mixture mixture neural-mixture h:neural_mixture 16384 2 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 0.47 0 0.47 0.47 0.47 0.455 0.005 0.0332446808511 0.0332446808511 30956 164.048773235 30956 30840 31072 2.69245779825 2.69245779825 1 +h neural_mixture mixture neural-mixture h:neural_mixture 65536 2 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2.06 0 2.06 2.06 2.06 2.025 0.025 0.0303398058252 0.0303398058252 85348 84.8528137424 85348 85288 85408 2.35902117116 2.35902117116 1 +h neural_mixture mixture neural-mixture h:neural_mixture 262144 2 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 8.92 0.0141421356237 8.92 8.91 8.93 8.83 0.08 0.0280269410542 0.0280269410542 251704 73.5391052434 251704 251652 251756 2.05795611601 2.05795611601 1 +h neural_mixture mixture neural-mixture h:neural_mixture 1048576 2 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 40.635 0.0777817459305 40.635 40.58 40.69 40.32 0.275 0.0246093720192 0.0246093720192 679942 36.7695526217 679942 679916 679968 1.93963669332 1.93963669332 1 +h neural_mixture mixture neural-mixture h:neural_mixture 2097152 2 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 86.46 0.0707106781187 86.46 86.41 86.51 85.925 0.45 0.0231320919369 0.0231320919369 1011668 73.5391052434 1011668 1011616 1011720 1.92069854644 1.92069854644 1 +h neural_mixture mixture neural-mixture h:neural_mixture 4194304 2 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 183.775 0.0636396103068 183.775 183.73 183.82 182.965 0.645 0.021765747462 0.021765747462 1394292 113.13708499 1394292 1394212 1394372 1.88509670528 1.88509670528 1 +h neural_mixture mixture neural-mixture h:neural_mixture 10000000 2 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 472.355 1.22329473145 472.355 471.49 473.22 470.665 1.235 0.0201898469274 0.0201898469274 2510728 96.1665222414 2510728 2510660 2510796 1.81617547371 1.81617547371 1 +h ppmd expert ppmd h:ppmd 4096 2 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 0.005 0.00707106781187 0.005 0 0.01 0.005 0 7928 56.5685424949 7928 7888 7968 2.02916276221 2.02916276221 1 +h ppmd expert ppmd h:ppmd 16384 2 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 0.02 0 0.02 0.02 0.02 0.02 0 0.78125 0.78125 15204 0 15204 15204 15204 3.05111741489 3.05111741489 1 +h ppmd expert ppmd h:ppmd 65536 2 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 0.115 0.00707106781187 0.115 0.11 0.12 0.1 0.01 0.544507575758 0.544507575758 39230 14.1421356237 39230 39220 39240 2.82800309907 2.82800309907 1 +h ppmd expert ppmd h:ppmd 262144 2 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 0.52 0 0.52 0.52 0.52 0.455 0.055 0.480769230769 0.480769230769 127062 82.0243866176 127062 127004 127120 2.54059561824 2.54059561824 1 +h ppmd expert ppmd h:ppmd 1048576 2 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2.405 0.0353553390593 2.405 2.38 2.43 2.235 0.16 0.415845350486 0.415845350486 370418 59.3969696197 370418 370376 370460 2.48956475398 2.48956475398 1 +h ppmd expert ppmd h:ppmd 2097152 2 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 4.96 0 4.96 4.96 4.96 4.745 0.205 0.403225806452 0.403225806452 434228 0 434228 434228 434228 2.52508057302 2.52508057302 1 +h ppmd expert ppmd h:ppmd 4194304 2 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 10.06 0.0141421356237 10.06 10.05 10.07 9.83 0.21 0.397614707001 0.397614707001 436302 2.82842712475 436302 436300 436304 2.5353209835 2.5353209835 1 +h ppmd expert ppmd h:ppmd 10000000 2 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 24.235 0.0212132034356 24.235 24.22 24.25 23.945 0.265 0.393511319062 0.393511319062 550912 50.9116882454 550912 550876 550948 2.52355814767 2.52355814767 1 +h rosa expert rosaplus h:rosa 4096 2 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 0.045 0.0212132034356 0.045 0.03 0.06 0.035 0.005 0.09765625 0.09765625 6510 53.7401153702 6510 6472 6548 2.11172124286 2.11172124286 1 +h rosa expert rosaplus h:rosa 16384 2 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 0.13 0 0.13 0.13 0.13 0.12 0.005 0.120192307692 0.120192307692 8428 45.2548339959 8428 8396 8460 3.48630622986 3.48630622986 1 +h rosa expert rosaplus h:rosa 65536 2 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 0.645 0.00707106781187 0.645 0.64 0.65 0.63 0.01 0.0969050480769 0.0969050480769 15676 62.2253967444 15676 15632 15720 3.19056696201 3.19056696201 1 +h rosa expert rosaplus h:rosa 262144 2 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 3.595 0.0212132034356 3.595 3.58 3.61 3.495 0.085 0.0695422398985 0.0695422398985 44422 161.220346111 44422 44308 44536 2.90567254487 2.90567254487 1 +h rosa expert rosaplus h:rosa 1048576 2 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 19.715 0.0494974746831 19.715 19.68 19.75 19.295 0.4 0.0507229597612 0.0507229597612 194712 90.5096679919 194712 194648 194776 2.80123027825 2.80123027825 1 +h rosa expert rosaplus h:rosa 2097152 2 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 47.28 0.0282842712475 47.28 47.26 47.3 45.58 1.65 0.0423011920025 0.0423011920025 317342 14.1421356237 317342 317332 317352 2.70437138702 2.70437138702 1 +h rosa expert rosaplus h:rosa 4194304 2 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 113.56 0.0424264068712 113.56 113.53 113.59 108.845 4.58 0.0352236727647 0.0352236727647 626392 5.65685424949 626392 626388 626396 2.62099230176 2.62099230176 1 +h rosa expert rosaplus h:rosa 10000000 2 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 335.93 0.311126983722 335.93 335.71 336.15 323.125 12.435 0.028389090746 0.028389090746 1422856 5.65685424949 1422856 1422852 1422860 2.48481897791 2.48481897791 1 +h rwkv7 expert rwkv7 h:rwkv7 4096 2 11 - 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 0.07 0.0282842712475 0.07 0.05 0.09 0.065 0 0.0607638888889 0.0607638888889 9076 169.705627485 9076 8956 9196 7.21701437947 7.21701437947 1 +h rwkv7 expert rwkv7 h:rwkv7 16384 2 11 - f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 0.22 0.0282842712475 0.22 0.2 0.24 0.22 0 0.0716145833333 0.0716145833333 8960 45.2548339959 8960 8928 8992 5.85021072042 5.85021072042 1 +h rwkv7 expert rwkv7 h:rwkv7 65536 2 11 - 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 0.82 0.0141421356237 0.82 0.81 0.83 0.815 0 0.0762308493232 0.0762308493232 8904 214.960461481 8904 8752 9056 4.31173851819 4.31173851819 1 +h rwkv7 expert rwkv7 h:rwkv7 262144 2 11 - 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 3.265 0.0353553390593 3.265 3.24 3.29 3.26 0 0.0765741678862 0.0765741678862 9006 8.48528137424 9006 9000 9012 4.18065126521 4.18065126521 1 +h rwkv7 expert rwkv7 h:rwkv7 1048576 2 11 - 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 13.295 0.205060966544 13.295 13.15 13.44 13.285 0 0.0752251946406 0.0752251946406 9726 2.82842712475 9726 9724 9728 3.61829048151 3.61829048151 1 +h rwkv7 expert rwkv7 h:rwkv7 2097152 2 11 - 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 26.94 0.169705627485 26.94 26.82 27.06 26.93 0 0.0742405227591 0.0742405227591 10798 161.220346111 10798 10684 10912 3.49833649966 3.49833649966 1 +h rwkv7 expert rwkv7 h:rwkv7 4194304 2 11 - 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 54.115 0.176776695297 54.115 53.99 54.24 54.075 0 0.0739170533601 0.0739170533601 12724 197.989898732 12724 12584 12864 3.33570098711 3.33570098711 1 +h rwkv7 expert rwkv7 h:rwkv7 10000000 2 11 - 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 131.685 0.968736290226 131.685 131 132.37 131.605 0 0.0724228364952 0.0724228364952 18312 226.27416998 18312 18152 18472 3.17490278117 3.17490278117 1 +compress fac-ctw expert fac-ctw compress:fac-ctw 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 0.04 0 0.04 0.04 0.04 0.04 0 0.09765625 0.09765625 7478 149.906637612 7478 7372 7584 1332 1332 0.3251953125 0.3251953125 1 +compress fac-ctw expert fac-ctw compress:fac-ctw 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 0.18 0 0.18 0.18 0.18 0.175 0 0.0868055555556 0.0868055555556 12666 149.906637612 12666 12560 12772 6579 6579 0.401550292969 0.401550292969 1 +compress fac-ctw expert fac-ctw compress:fac-ctw 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 0.8 0 0.8 0.8 0.8 0.79 0.005 0.078125 0.078125 22798 76.3675323681 22798 22744 22852 22728 22728 0.346801757812 0.346801757812 1 +compress fac-ctw expert fac-ctw compress:fac-ctw 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 3.49 0 3.49 3.49 3.49 3.475 0.005 0.0716332378223 0.0716332378223 48410 70.7106781187 48410 48360 48460 80004 80004 0.305191040039 0.305191040039 1 +compress fac-ctw expert fac-ctw compress:fac-ctw 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 15.735 0.0494974746831 15.735 15.7 15.77 15.67 0.055 0.0635529042082 0.0635529042082 116290 42.4264068712 116290 116260 116320 301713 301713 0.287735939026 0.287735939026 1 +compress fac-ctw expert fac-ctw compress:fac-ctw 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 33.235 0.00707106781187 33.235 33.23 33.24 33.11 0.09 0.0601775250569 0.0601775250569 186648 192.333044483 186648 186512 186784 597003 597003 0.284673213959 0.284673213959 1 +compress fac-ctw expert fac-ctw compress:fac-ctw 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 70.37 0.0565685424949 70.37 70.33 70.41 70.165 0.14 0.0568424227998 0.0568424227998 300640 181.019335984 300640 300512 300768 1177962 1177962 0.280848026276 0.280848026276 1 +compress fac-ctw expert fac-ctw compress:fac-ctw 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 178.5 0.480832611207 178.5 178.16 178.84 178.045 0.3 0.0534273264124 0.0534273264124 546340 16.9705627485 546340 546328 546352 2746861 2746861 0.2746861 0.2746861 1 +compress match expert match compress:match 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 0 0 0 0 0 0 0 5868 0 5868 5868 5868 2874 2874 0.70166015625 0.70166015625 1 +compress match expert match compress:match 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 0.015 0.00707106781187 0.015 0.01 0.02 0.015 0 1.171875 1.171875 5888 226.27416998 5888 5728 6048 13506 13506 0.824340820312 0.824340820312 1 +compress match expert match compress:match 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 0.03 0 0.03 0.03 0.03 0.03 0 2.08333333333 2.08333333333 6716 5.65685424949 6716 6712 6720 53748 53748 0.820129394531 0.820129394531 1 +compress match expert match compress:match 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 0.13 0 0.13 0.13 0.13 0.13 0 1.92307692308 1.92307692308 8678 172.53405461 8678 8556 8800 206203 206203 0.786602020264 0.786602020264 1 +compress match expert match compress:match 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 0.525 0.00707106781187 0.525 0.52 0.53 0.52 0 1.90493468795 1.90493468795 12660 33.941125497 12660 12636 12684 817505 817505 0.779633522034 0.779633522034 1 +compress match expert match compress:match 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 1.06 0 1.06 1.06 1.06 1.05 0.005 1.88679245283 1.88679245283 22490 195.161471607 22490 22352 22628 1644756 1644756 0.784280776978 0.784280776978 1 +compress match expert match compress:match 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2.135 0.0212132034356 2.135 2.12 2.15 2.125 0.005 1.87362878455 1.87362878455 29486 98.9949493661 29486 29416 29556 3292751 3292751 0.785053014755 0.785053014755 1 +compress match expert match compress:match 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 5.1 0.0282842712475 5.1 5.08 5.12 5.075 0.015 1.86997839785 1.86997839785 55878 25.4558441227 55878 55860 55896 7859324 7859324 0.7859324 0.7859324 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 0.12 0 0.12 0.12 0.12 0.12 0 0.0325520833333 0.0325520833333 12636 5.65685424949 12636 12632 12640 1001 1001 0.244384765625 0.244384765625 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 0.515 0.00707106781187 0.515 0.51 0.52 0.505 0.005 0.0303426659125 0.0303426659125 30636 73.5391052434 30636 30584 30688 5533 5533 0.337707519531 0.337707519531 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2.255 0.00707106781187 2.255 2.25 2.26 2.23 0.02 0.0277163225172 0.0277163225172 87698 36.7695526217 87698 87672 87724 19344 19344 0.295166015625 0.295166015625 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 9.84 0.0424264068712 9.84 9.81 9.87 9.76 0.06 0.0254067402223 0.0254067402223 246824 39.5979797464 246824 246796 246852 67454 67454 0.257316589355 0.257316589355 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 44.685 0.0353553390593 44.685 44.66 44.71 44.345 0.29 0.0223788813474 0.0223788813474 692756 192.333044483 692756 692620 692892 254251 254251 0.242472648621 0.242472648621 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 95.705 0.0636396103068 95.705 95.66 95.75 95.115 0.495 0.0208975543824 0.0208975543824 1017046 132.936074863 1017046 1016952 1017140 503520 503520 0.240097045898 0.240097045898 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 204.9 0.183847763108 204.9 204.77 205.03 204.05 0.65 0.0195217257693 0.0195217257693 1407042 14.1421356237 1407042 1407032 1407052 988355 988355 0.235642194748 0.235642194748 1 +compress neural_mixture mixture neural-mixture compress:neural_mixture 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 532.48 1.00409162928 532.48 531.77 533.19 530.715 1.22 0.0179100813544 0.0179100813544 2487504 5.65685424949 2487504 2487500 2487508 2270248 2270248 0.2270248 0.2270248 1 +compress ppmd expert ppmd compress:ppmd 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 0.015 0.00707106781187 0.015 0.01 0.02 0.01 0 0.29296875 0.29296875 7594 53.7401153702 7594 7556 7632 1058 1058 0.25830078125 0.25830078125 1 +compress ppmd expert ppmd compress:ppmd 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 0.03 0 0.03 0.03 0.03 0.025 0 0.520833333333 0.520833333333 15518 48.0832611207 15518 15484 15552 6267 6267 0.382507324219 0.382507324219 1 +compress ppmd expert ppmd compress:ppmd 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 0.14 0 0.14 0.14 0.14 0.13 0.01 0.446428571429 0.446428571429 39558 48.0832611207 39558 39524 39592 23186 23186 0.353790283203 0.353790283203 1 +compress ppmd expert ppmd compress:ppmd 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 0.625 0.00707106781187 0.625 0.62 0.63 0.545 0.08 0.400025601639 0.400025601639 127330 48.0832611207 127330 127296 127364 83269 83269 0.317646026611 0.317646026611 1 +compress ppmd expert ppmd compress:ppmd 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2.745 0.00707106781187 2.745 2.74 2.75 2.575 0.165 0.364299933643 0.364299933643 370730 161.220346111 370730 370616 370844 326331 326331 0.311213493347 0.311213493347 1 +compress ppmd expert ppmd compress:ppmd 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 5.735 0.0494974746831 5.735 5.7 5.77 5.535 0.185 0.348748821795 0.348748821795 434424 50.9116882454 434424 434388 434460 661953 661953 0.315643787384 0.315643787384 1 +compress ppmd expert ppmd compress:ppmd 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 11.54 0.0707106781187 11.54 11.49 11.59 11.315 0.215 0.346626957755 0.346626957755 436550 330.925973595 436550 436316 436784 1329257 1329257 0.316919565201 0.316919565201 1 +compress ppmd expert ppmd compress:ppmd 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 27.85 0.282842712475 27.85 27.65 28.05 27.565 0.265 0.342450090247 0.342450090247 551034 42.4264068712 551034 551004 551064 3154465 3154465 0.3154465 0.3154465 1 +compress rosa expert rosaplus compress:rosa 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 0 0 0 0 0 0 0 6196 118.793939239 6196 6112 6280 1127 1127 0.275146484375 0.275146484375 1 +compress rosa expert rosaplus compress:rosa 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 0.02 0 0.02 0.02 0.02 0.02 0 0.78125 0.78125 7964 84.8528137424 7964 7904 8024 6359 6359 0.388122558594 0.388122558594 1 +compress rosa expert rosaplus compress:rosa 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 0.12 0 0.12 0.12 0.12 0.12 0 0.520833333333 0.520833333333 16240 141.421356237 16240 16140 16340 22843 22843 0.348556518555 0.348556518555 1 +compress rosa expert rosaplus compress:rosa 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 0.7 0 0.7 0.7 0.7 0.675 0.01 0.357142857143 0.357142857143 46418 14.1421356237 46418 46408 46428 80590 80590 0.307426452637 0.307426452637 1 +compress rosa expert rosaplus compress:rosa 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 3.855 0.0212132034356 3.855 3.84 3.87 3.805 0.04 0.259407299742 0.259407299742 146028 209.303607231 146028 145880 146176 306522 306522 0.292322158813 0.292322158813 1 +compress rosa expert rosaplus compress:rosa 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 9.3 0.0141421356237 9.3 9.29 9.31 9.165 0.12 0.215054012087 0.215054012087 281614 25.4558441227 281614 281596 281632 608681 608681 0.290241718292 0.290241718292 1 +compress rosa expert rosaplus compress:rosa 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 23.3 0.197989898732 23.3 23.16 23.44 23.045 0.235 0.17168001792 0.17168001792 538264 135.764501988 538264 538168 538360 1199345 1199345 0.285946130753 0.285946130753 1 +compress rosa expert rosaplus compress:rosa 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 74.04 0.296984848098 74.04 73.83 74.25 73.42 0.55 0.128806319344 0.128806319344 1263604 62.2253967444 1263604 1263560 1263648 2752778 2752778 0.2752778 0.2752778 1 +compress rwkv7 expert rwkv7 compress:rwkv7 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 0.05 0 0.05 0.05 0.05 0.05 0 0.078125 0.078125 8454 36.7695526217 8454 8428 8480 3714 3714 0.90673828125 0.90673828125 1 +compress rwkv7 expert rwkv7 compress:rwkv7 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 0.21 0 0.21 0.21 0.21 0.21 0 0.0744047619048 0.0744047619048 8334 132.936074863 8334 8240 8428 12000 12000 0.732421875 0.732421875 1 +compress rwkv7 expert rwkv7 compress:rwkv7 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 0.83 0 0.83 0.83 0.83 0.83 0 0.0753012048193 0.0753012048193 8396 96.1665222414 8396 8328 8464 35340 35340 0.539245605469 0.539245605469 1 +compress rwkv7 expert rwkv7 compress:rwkv7 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 3.34 0.0424264068712 3.34 3.31 3.37 3.335 0 0.0748563385837 0.0748563385837 8442 144.249783362 8442 8340 8544 137010 137010 0.522651672363 0.522651672363 1 +compress rwkv7 expert rwkv7 compress:rwkv7 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 13.515 0.176776695297 13.515 13.39 13.64 13.505 0 0.0739981909728 0.0739981909728 9988 11.313708499 9988 9980 9996 474275 474275 0.452303886414 0.452303886414 1 +compress rwkv7 expert rwkv7 compress:rwkv7 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 27.635 0.0353553390593 27.635 27.61 27.66 27.615 0 0.0723720512677 0.0723720512677 11934 172.53405461 11934 11812 12056 917086 917086 0.437300682068 0.437300682068 1 +compress rwkv7 expert rwkv7 compress:rwkv7 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 55.425 0.275771644663 55.425 55.23 55.62 55.385 0 0.0721704918981 0.0721704918981 15584 0 15584 15584 15584 1748887 1748887 0.416967153549 0.416967153549 1 +compress rwkv7 expert rwkv7 compress:rwkv7 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 135.19 0.919238815543 135.19 134.54 135.84 135.1 0.005 0.0705448896601 0.0705448896601 25562 172.53405461 25562 25440 25684 3968645 3968645 0.3968645 0.3968645 1 +decompress fac-ctw expert fac-ctw decompress:fac-ctw 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 0.03 0 0.03 0.03 0.03 0.03 0 0.130208333333 0.130208333333 7566 98.9949493661 7566 7496 7636 1332 1332 0.3251953125 0.3251953125 1 +decompress fac-ctw expert fac-ctw decompress:fac-ctw 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 0.18 0 0.18 0.18 0.18 0.17 0 0.0868055555556 0.0868055555556 12616 16.9705627485 12616 12604 12628 6579 6579 0.401550292969 0.401550292969 1 +decompress fac-ctw expert fac-ctw decompress:fac-ctw 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 0.805 0.00707106781187 0.805 0.8 0.81 0.8 0 0.0776427469136 0.0776427469136 23004 45.2548339959 23004 22972 23036 22728 22728 0.346801757812 0.346801757812 1 +decompress fac-ctw expert fac-ctw decompress:fac-ctw 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 3.5 0.0141421356237 3.5 3.49 3.51 3.49 0 0.0714291545237 0.0714291545237 48420 5.65685424949 48420 48416 48424 80004 80004 0.305191040039 0.305191040039 1 +decompress fac-ctw expert fac-ctw decompress:fac-ctw 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 15.77 0.0141421356237 15.77 15.76 15.78 15.72 0.035 0.0634115663984 0.0634115663984 116030 31.1126983722 116030 116008 116052 301713 301713 0.287735939026 0.287735939026 1 +decompress fac-ctw expert fac-ctw decompress:fac-ctw 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 33.365 0.0494974746831 33.365 33.33 33.4 33.26 0.07 0.0599431200605 0.0599431200605 186090 36.7695526217 186090 186064 186116 597003 597003 0.284673213959 0.284673213959 1 +decompress fac-ctw expert fac-ctw decompress:fac-ctw 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 70.385 0.0919238815543 70.385 70.32 70.45 70.2 0.12 0.0568303390119 0.0568303390119 299620 84.8528137424 299620 299560 299680 1177962 1177962 0.280848026276 0.280848026276 1 +decompress fac-ctw expert fac-ctw decompress:fac-ctw 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 179.235 0.120208152802 179.235 179.15 179.32 178.79 0.285 0.0532080526063 0.0532080526063 543828 11.313708499 543828 543820 543836 2746861 2746861 0.2746861 0.2746861 1 +decompress match expert match decompress:match 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 0.005 0.00707106781187 0.005 0 0.01 0 0 5868 11.313708499 5868 5860 5876 2874 2874 0.70166015625 0.70166015625 1 +decompress match expert match decompress:match 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 0.015 0.00707106781187 0.015 0.01 0.02 0.01 0 1.171875 1.171875 5942 121.622366364 5942 5856 6028 13506 13506 0.824340820312 0.824340820312 1 +decompress match expert match decompress:match 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 0.03 0 0.03 0.03 0.03 0.03 0 2.08333333333 2.08333333333 6994 25.4558441227 6994 6976 7012 53748 53748 0.820129394531 0.820129394531 1 +decompress match expert match decompress:match 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 0.13 0 0.13 0.13 0.13 0.13 0 1.92307692308 1.92307692308 8230 110.308657865 8230 8152 8308 206203 206203 0.786602020264 0.786602020264 1 +decompress match expert match decompress:match 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 0.54 0 0.54 0.54 0.54 0.54 0 1.85185185185 1.85185185185 11914 36.7695526217 11914 11888 11940 817505 817505 0.779633522034 0.779633522034 1 +decompress match expert match decompress:match 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 1.11 0 1.11 1.11 1.11 1.1 0.005 1.8018018018 1.8018018018 20516 16.9705627485 20516 20504 20528 1644756 1644756 0.784280776978 0.784280776978 1 +decompress match expert match decompress:match 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2.205 0.00707106781187 2.205 2.2 2.21 2.195 0 1.81406828466 1.81406828466 24234 36.7695526217 24234 24208 24260 3292751 3292751 0.785053014755 0.785053014755 1 +decompress match expert match decompress:match 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 5.31 0.0141421356237 5.31 5.3 5.32 5.29 0.01 1.79600319908 1.79600319908 46390 132.936074863 46390 46296 46484 7859324 7859324 0.7859324 0.7859324 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 0.12 0 0.12 0.12 0.12 0.12 0 0.0325520833333 0.0325520833333 12826 87.6812408671 12826 12764 12888 1001 1001 0.244384765625 0.244384765625 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 0.52 0 0.52 0.52 0.52 0.505 0.005 0.0300480769231 0.0300480769231 30586 53.7401153702 30586 30548 30624 5533 5533 0.337707519531 0.337707519531 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2.255 0.00707106781187 2.255 2.25 2.26 2.225 0.02 0.0277163225172 0.0277163225172 86398 76.3675323681 86398 86344 86452 19344 19344 0.295166015625 0.295166015625 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 9.86 0.0141421356237 9.86 9.85 9.87 9.77 0.075 0.0253549956542 0.0253549956542 249892 84.8528137424 249892 249832 249952 67454 67454 0.257316589355 0.257316589355 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 44.925 0.00707106781186 44.925 44.92 44.93 44.6 0.285 0.0222593213664 0.0222593213664 679326 19.7989898732 679326 679312 679340 254251 254251 0.242472648621 0.242472648621 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 95.965 0.0636396103068 95.965 95.92 96.01 95.45 0.42 0.0208409361723 0.0208409361723 999626 70.7106781187 999626 999576 999676 503520 503520 0.240097045898 0.240097045898 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 205.13 0.0141421356237 205.13 205.12 205.14 204.305 0.62 0.0194998294228 0.0194998294228 1387238 9359.26535579 1387238 1380620 1393856 988355 988355 0.235642194748 0.235642194748 1 +decompress neural_mixture mixture neural-mixture decompress:neural_mixture 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 534.485 0.473761543395 534.485 534.15 534.82 532.68 1.28 0.0178428710076 0.0178428710076 2508222 19.7989898732 2508222 2508208 2508236 2270248 2270248 0.2270248 0.2270248 1 +decompress ppmd expert ppmd decompress:ppmd 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 0.015 0.00707106781187 0.015 0.01 0.02 0.005 0 0.29296875 0.29296875 7612 96.1665222414 7612 7544 7680 1058 1058 0.25830078125 0.25830078125 1 +decompress ppmd expert ppmd decompress:ppmd 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 0.03 0 0.03 0.03 0.03 0.025 0 0.520833333333 0.520833333333 15288 84.8528137424 15288 15228 15348 6267 6267 0.382507324219 0.382507324219 1 +decompress ppmd expert ppmd decompress:ppmd 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 0.14 0 0.14 0.14 0.14 0.12 0.02 0.446428571429 0.446428571429 39490 48.0832611207 39490 39456 39524 23186 23186 0.353790283203 0.353790283203 1 +decompress ppmd expert ppmd decompress:ppmd 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 0.62 0 0.62 0.62 0.62 0.55 0.06 0.403225806452 0.403225806452 127226 87.6812408671 127226 127164 127288 83269 83269 0.317646026611 0.317646026611 1 +decompress ppmd expert ppmd decompress:ppmd 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 2.775 0.00707106781187 2.775 2.77 2.78 2.585 0.18 0.36036153027 0.36036153027 371152 124.450793489 371152 371064 371240 326331 326331 0.311213493347 0.311213493347 1 +decompress ppmd expert ppmd decompress:ppmd 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 5.765 0.0212132034356 5.765 5.75 5.78 5.535 0.22 0.346923424101 0.346923424101 435032 16.9705627485 435032 435020 435044 661953 661953 0.315643787384 0.315643787384 1 +decompress ppmd expert ppmd decompress:ppmd 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 11.65 0.0424264068712 11.65 11.62 11.68 11.425 0.205 0.343349916299 0.343349916299 435736 141.421356237 435736 435636 435836 1329257 1329257 0.316919565201 0.316919565201 1 +decompress ppmd expert ppmd decompress:ppmd 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 28.12 0.0565685424949 28.12 28.08 28.16 27.81 0.28 0.339145179982 0.339145179982 552890 2.82842712475 552890 552888 552892 3154465 3154465 0.3154465 0.3154465 1 +decompress rosa expert rosaplus decompress:rosa 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 0 0 0 0 0 0 0 6180 56.5685424949 6180 6140 6220 1127 1127 0.275146484375 0.275146484375 1 +decompress rosa expert rosaplus decompress:rosa 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 0.02 0 0.02 0.02 0.02 0.02 0 0.78125 0.78125 8040 11.313708499 8040 8032 8048 6359 6359 0.388122558594 0.388122558594 1 +decompress rosa expert rosaplus decompress:rosa 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 0.125 0.00707106781187 0.125 0.12 0.13 0.125 0 0.500801282051 0.500801282051 16380 28.2842712475 16380 16360 16400 22843 22843 0.348556518555 0.348556518555 1 +decompress rosa expert rosaplus decompress:rosa 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 0.7 0 0.7 0.7 0.7 0.685 0.015 0.357142857143 0.357142857143 46696 11.313708499 46696 46688 46704 80590 80590 0.307426452637 0.307426452637 1 +decompress rosa expert rosaplus decompress:rosa 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 3.885 0.00707106781187 3.885 3.88 3.89 3.85 0.03 0.257400683752 0.257400683752 146414 19.7989898732 146414 146400 146428 306522 306522 0.292322158813 0.292322158813 1 +decompress rosa expert rosaplus decompress:rosa 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 9.44 0.0282842712475 9.44 9.42 9.46 9.345 0.075 0.21186535777 0.21186535777 278224 11.313708499 278224 278216 278232 608681 608681 0.290241718292 0.290241718292 1 +decompress rosa expert rosaplus decompress:rosa 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 23.58 0.0565685424949 23.58 23.54 23.62 23.35 0.21 0.169635772285 0.169635772285 535834 127.279220614 535834 535744 535924 1199345 1199345 0.285946130753 0.285946130753 1 +decompress rosa expert rosaplus decompress:rosa 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 74.73 0.395979797464 74.73 74.45 75.01 74.115 0.535 0.12761778468 0.12761778468 1262736 67.8822509939 1262736 1262688 1262784 2752778 2752778 0.2752778 0.2752778 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 4096 2 11 rate-ac 652ed0541c8b9e2d8194c2a1cd20a3ea516efaa542082535c0ef57267e1ca49e /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 0.05 0 0.05 0.05 0.05 0.05 0 0.078125 0.078125 8308 147.078210487 8308 8204 8412 3714 3714 0.90673828125 0.90673828125 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 16384 2 11 rate-ac f72f174851ed11ae77711d9c7d2df9e3c34666e09adadb61a1081ff54df0196f /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 0.21 0 0.21 0.21 0.21 0.21 0 0.0744047619048 0.0744047619048 8386 110.308657865 8386 8308 8464 12000 12000 0.732421875 0.732421875 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 65536 2 11 rate-ac 05fc5f44993ef0557959db76bf47e45badb2dd9c69d93ce08911935b5e52bf40 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 0.86 0.0141421356237 0.86 0.85 0.87 0.855 0 0.0726842461122 0.0726842461122 8298 65.0538238692 8298 8252 8344 35340 35340 0.539245605469 0.539245605469 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 262144 2 11 rate-ac 0cdfd008d5327addbf5b118b4a8d616a8cc2971627727b10bfb20e97920881e7 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 3.29 0 3.29 3.29 3.29 3.285 0 0.0759878419453 0.0759878419453 8404 50.9116882454 8404 8368 8440 137010 137010 0.522651672363 0.522651672363 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 1048576 2 11 rate-ac 4fb5efa9f35df431737731bf3c8f38a467b69731940ff82a4ee0e218aae58834 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 14.005 0.19091883092 14.005 13.87 14.14 13.995 0 0.0714097056013 0.0714097056013 9334 65.0538238692 9334 9288 9380 474275 474275 0.452303886414 0.452303886414 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 2097152 2 11 rate-ac 9dd214e64458c0c36f75a6100aed1a90a7b76ff9dfbb85dc032dea17a1963f50 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 27.98 0.11313708499 27.98 27.9 28.06 27.955 0 0.0714802126524 0.0714802126524 10912 0 10912 10912 10912 917086 917086 0.437300682068 0.437300682068 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 4194304 2 11 rate-ac 5ba647fec2ba51b0383cbe7147e15a478d85613245b7131c41764dbdd5062f77 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 56.36 0.33941125497 56.36 56.12 56.6 56.325 0 0.0709736077915 0.0709736077915 13792 16.9705627485 13792 13780 13804 1748887 1748887 0.416967153549 0.416967153549 1 +decompress rwkv7 expert rwkv7 decompress:rwkv7 10000000 2 11 rate-ac 5985c81c39d927ae0e169625790ca4d9e7d1531270c8b09ad73176a375bb3d97 /home/theo/dev/infotheory/configs/bench/two.json a2181cf7f6579787b6fad37008a3aa9a6ef88363dc47a0e225d765d7b1ed207b native cli 135.96 0.155563491861 135.96 135.85 136.07 135.87 0.005 0.0701437879276 0.0701437879276 21654 268.700576851 21654 21464 21844 3968645 3968645 0.3968645 0.3968645 1 diff --git a/configs/aixi/builtin_tictactoe.json b/configs/aixi/builtin_tictactoe.json new file mode 100644 index 00000000..e177ad9b --- /dev/null +++ b/configs/aixi/builtin_tictactoe.json @@ -0,0 +1,42 @@ +{ + "assets": [], + "controller": { + "agent_horizon": 4, + "discount_gamma": 1, + "exploration_exploitation_ratio": 1.4, + "kind": "mc_aixi", + "num_simulations": 500, + "predictor": { + "base_depth": 32, + "encoding_bits": 8, + "kind": "fac-ctw", + "num_percept_bits": 21 + } + }, + "environment": { + "kind": "builtin", + "name": "tic_tac_toe" + }, + "interface": { + "agent_actions": 9, + "max_reward": 2, + "min_reward": -3, + "observation_bits": 18, + "observation_key_mode": "full_stream", + "observation_stream_len": 1, + "reward_bits": 3, + "reward_offset": 3 + }, + "kind": "planner_run", + "runtime": { + "eval_cycles": 5000, + "explore_epsilon": 0.9999, + "explore_gamma": 0.999999, + "learn_cycles": 500000, + "log_every": 1, + "perf": false, + "terminate_lifetime": 20, + "vm_perf_only": false + }, + "schema_version": 1 +} diff --git a/configs/aixi/paper_biased_rps.json b/configs/aixi/paper_biased_rps.json new file mode 100644 index 00000000..656c63c5 --- /dev/null +++ b/configs/aixi/paper_biased_rps.json @@ -0,0 +1,42 @@ +{ + "assets": [], + "controller": { + "agent_horizon": 4, + "discount_gamma": 1, + "exploration_exploitation_ratio": 0.5, + "kind": "mc_aixi", + "num_simulations": 100, + "predictor": { + "base_depth": 32, + "encoding_bits": 8, + "kind": "fac-ctw", + "num_percept_bits": 4 + } + }, + "environment": { + "kind": "builtin", + "name": "biased_rock_paper_scissor" + }, + "interface": { + "agent_actions": 3, + "max_reward": 1, + "min_reward": -1, + "observation_bits": 2, + "observation_key_mode": "full_stream", + "observation_stream_len": 1, + "reward_bits": 2, + "reward_offset": 1 + }, + "kind": "planner_run", + "runtime": { + "eval_cycles": 200, + "explore_epsilon": 0.999, + "explore_gamma": 0.99999, + "learn_cycles": 15000, + "log_every": 1, + "perf": false, + "terminate_lifetime": 20, + "vm_perf_only": false + }, + "schema_version": 1 +} diff --git a/configs/aixi/paper_extended_tiger.json b/configs/aixi/paper_extended_tiger.json new file mode 100644 index 00000000..5c466924 --- /dev/null +++ b/configs/aixi/paper_extended_tiger.json @@ -0,0 +1,42 @@ +{ + "assets": [], + "controller": { + "agent_horizon": 4, + "discount_gamma": 1, + "exploration_exploitation_ratio": 1.4, + "kind": "mc_aixi", + "num_simulations": 500, + "predictor": { + "base_depth": 96, + "encoding_bits": 8, + "kind": "fac-ctw", + "num_percept_bits": 11 + } + }, + "environment": { + "kind": "builtin", + "name": "extended_tiger" + }, + "interface": { + "agent_actions": 4, + "max_reward": 30, + "min_reward": -100, + "observation_bits": 3, + "observation_key_mode": "full_stream", + "observation_stream_len": 1, + "reward_bits": 8, + "reward_offset": 100 + }, + "kind": "planner_run", + "runtime": { + "eval_cycles": 5000, + "explore_epsilon": 0.99, + "explore_gamma": 0.99999, + "learn_cycles": 50000, + "log_every": 1, + "perf": false, + "terminate_lifetime": 20, + "vm_perf_only": false + }, + "schema_version": 1 +} diff --git a/configs/aixi/paper_kuhn_poker.json b/configs/aixi/paper_kuhn_poker.json new file mode 100644 index 00000000..c325b6d2 --- /dev/null +++ b/configs/aixi/paper_kuhn_poker.json @@ -0,0 +1,42 @@ +{ + "assets": [], + "controller": { + "agent_horizon": 2, + "discount_gamma": 1, + "exploration_exploitation_ratio": 1.4, + "kind": "mc_aixi", + "num_simulations": 200, + "predictor": { + "base_depth": 16, + "encoding_bits": 8, + "kind": "fac-ctw", + "num_percept_bits": 7 + } + }, + "environment": { + "kind": "builtin", + "name": "kuhn_poker" + }, + "interface": { + "agent_actions": 2, + "max_reward": 2, + "min_reward": -2, + "observation_bits": 4, + "observation_key_mode": "full_stream", + "observation_stream_len": 1, + "reward_bits": 3, + "reward_offset": 2 + }, + "kind": "planner_run", + "runtime": { + "eval_cycles": 200, + "explore_epsilon": 0.99, + "explore_gamma": 0.9999, + "learn_cycles": 25000, + "log_every": 1, + "perf": false, + "terminate_lifetime": 20, + "vm_perf_only": false + }, + "schema_version": 1 +} diff --git a/configs/aixi/paper_tictactoe.json b/configs/aixi/paper_tictactoe.json new file mode 100644 index 00000000..abad2d69 --- /dev/null +++ b/configs/aixi/paper_tictactoe.json @@ -0,0 +1,42 @@ +{ + "assets": [], + "controller": { + "agent_horizon": 9, + "discount_gamma": 1, + "exploration_exploitation_ratio": 1.4, + "kind": "mc_aixi", + "num_simulations": 500, + "predictor": { + "base_depth": 64, + "encoding_bits": 8, + "kind": "fac-ctw", + "num_percept_bits": 21 + } + }, + "environment": { + "kind": "builtin", + "name": "tic_tac_toe" + }, + "interface": { + "agent_actions": 9, + "max_reward": 2, + "min_reward": -3, + "observation_bits": 18, + "observation_key_mode": "full_stream", + "observation_stream_len": 1, + "reward_bits": 3, + "reward_offset": 3 + }, + "kind": "planner_run", + "runtime": { + "eval_cycles": 5000, + "explore_epsilon": 0.9999, + "explore_gamma": 0.999999, + "learn_cycles": 500000, + "log_every": 1, + "perf": false, + "terminate_lifetime": 20, + "vm_perf_only": false + }, + "schema_version": 1 +} diff --git a/aixi_confs/sudo_baseline.txt b/configs/aixi/sudo_baseline.txt similarity index 100% rename from aixi_confs/sudo_baseline.txt rename to configs/aixi/sudo_baseline.txt diff --git a/configs/aixi/ui_biased_rps.json b/configs/aixi/ui_biased_rps.json new file mode 100644 index 00000000..72dd07d6 --- /dev/null +++ b/configs/aixi/ui_biased_rps.json @@ -0,0 +1,42 @@ +{ + "assets": [], + "controller": { + "agent_horizon": 4, + "discount_gamma": 1, + "exploration_exploitation_ratio": 0.6, + "kind": "mc_aixi", + "num_simulations": 120, + "predictor": { + "base_depth": 32, + "encoding_bits": 8, + "kind": "fac-ctw", + "num_percept_bits": 4 + } + }, + "environment": { + "kind": "builtin", + "name": "biased_rock_paper_scissor" + }, + "interface": { + "agent_actions": 3, + "max_reward": 1, + "min_reward": -1, + "observation_bits": 2, + "observation_key_mode": "full_stream", + "observation_stream_len": 1, + "reward_bits": 2, + "reward_offset": 1 + }, + "kind": "planner_run", + "runtime": { + "eval_cycles": 200, + "explore_epsilon": 0.995, + "explore_gamma": 0.999, + "learn_cycles": 3000, + "log_every": 50, + "perf": false, + "terminate_lifetime": 20, + "vm_perf_only": false + }, + "schema_version": 1 +} diff --git a/configs/aixi/ui_coin_flip.json b/configs/aixi/ui_coin_flip.json new file mode 100644 index 00000000..b89e4a0e --- /dev/null +++ b/configs/aixi/ui_coin_flip.json @@ -0,0 +1,42 @@ +{ + "assets": [], + "controller": { + "agent_horizon": 2, + "discount_gamma": 1, + "exploration_exploitation_ratio": 1, + "kind": "mc_aixi", + "num_simulations": 120, + "predictor": { + "base_depth": 16, + "encoding_bits": 8, + "kind": "fac-ctw", + "num_percept_bits": 2 + } + }, + "environment": { + "kind": "builtin", + "name": "coin_flip" + }, + "interface": { + "agent_actions": 2, + "max_reward": 1, + "min_reward": 0, + "observation_bits": 1, + "observation_key_mode": "full_stream", + "observation_stream_len": 1, + "reward_bits": 1, + "reward_offset": 0 + }, + "kind": "planner_run", + "runtime": { + "eval_cycles": 200, + "explore_epsilon": 0.995, + "explore_gamma": 0.999, + "learn_cycles": 2000, + "log_every": 50, + "perf": false, + "terminate_lifetime": 20, + "vm_perf_only": false + }, + "schema_version": 1 +} diff --git a/configs/aixi/ui_kuhn_poker.json b/configs/aixi/ui_kuhn_poker.json new file mode 100644 index 00000000..386a43ce --- /dev/null +++ b/configs/aixi/ui_kuhn_poker.json @@ -0,0 +1,42 @@ +{ + "assets": [], + "controller": { + "agent_horizon": 2, + "discount_gamma": 1, + "exploration_exploitation_ratio": 1.4, + "kind": "mc_aixi", + "num_simulations": 200, + "predictor": { + "base_depth": 36, + "encoding_bits": 8, + "kind": "fac-ctw", + "num_percept_bits": 7 + } + }, + "environment": { + "kind": "builtin", + "name": "kuhn_poker" + }, + "interface": { + "agent_actions": 2, + "max_reward": 2, + "min_reward": -2, + "observation_bits": 4, + "observation_key_mode": "full_stream", + "observation_stream_len": 1, + "reward_bits": 3, + "reward_offset": 2 + }, + "kind": "planner_run", + "runtime": { + "eval_cycles": 500, + "explore_epsilon": 0.99, + "explore_gamma": 0.9999, + "learn_cycles": 5000, + "log_every": 100, + "perf": false, + "terminate_lifetime": 20, + "vm_perf_only": false + }, + "schema_version": 1 +} diff --git a/configs/aixi/vm_example.json b/configs/aixi/vm_example.json new file mode 100644 index 00000000..c8f9b68e --- /dev/null +++ b/configs/aixi/vm_example.json @@ -0,0 +1,90 @@ +{ + "assets": [ + { + "id": "firecracker", + "path": "..\/..\/vendor\/nyx-lite\/vm_image\/vmconfig.json" + } + ], + "controller": { + "agent_horizon": 8, + "discount_gamma": 1, + "exploration_exploitation_ratio": 1.4, + "kind": "mc_aixi", + "num_simulations": 1, + "predictor": { + "base_depth": 32, + "encoding_bits": 8, + "kind": "fac-ctw", + "num_percept_bits": 16 + } + }, + "environment": { + "action_source": { + "actions": [ + { + "name": "ping", + "payload": "50494e47" + }, + { + "name": "status", + "payload": "535441545553" + } + ], + "encoding": "hex", + "kind": "literal" + }, + "boot_timeout_ms": 30000, + "debug_mode": false, + "episode_steps": 8, + "firecracker_config_asset": "firecracker", + "instance_id": "aixi-nyx", + "kind": "nyx_vm", + "observation_bits": 8, + "observation_pad_byte": 0, + "observation_policy": "from_guest", + "observation_stream_len": 1, + "observation_stream_mode": "pad_truncate", + "protocol": { + "action_prefix": "ACT ", + "action_suffix": "\n", + "data_prefix": "DATA ", + "done_prefix": "DONE ", + "obs_prefix": "OBS ", + "rew_prefix": "REW ", + "wire_encoding": "hex" + }, + "reward_bits": 8, + "reward_policy": { + "kind": "from_guest" + }, + "shared_memory_policy": "snapshot", + "shared_region_name": "shared", + "shared_region_size": 4096, + "stats_backend": { + "depth": 32, + "kind": "ctw" + }, + "step_cost": 1, + "step_timeout_ms": 5000 + }, + "interface": { + "agent_actions": 2, + "max_reward": 127, + "min_reward": -128, + "observation_bits": 8, + "observation_key_mode": "full_stream", + "observation_stream_len": 1, + "reward_bits": 8, + "reward_offset": 128 + }, + "kind": "planner_run", + "runtime": { + "explore_epsilon": 1, + "explore_gamma": 1, + "log_every": 1, + "perf": true, + "terminate_lifetime": 50, + "vm_perf_only": false + }, + "schema_version": 1 +} diff --git a/configs/aixi/vm_perf_debug.json b/configs/aixi/vm_perf_debug.json new file mode 100644 index 00000000..9a8bcbba --- /dev/null +++ b/configs/aixi/vm_perf_debug.json @@ -0,0 +1,86 @@ +{ + "assets": [ + { + "id": "firecracker", + "path": "..\/..\/vendor\/nyx-lite\/vm_image\/vmconfig.json" + } + ], + "controller": { + "agent_horizon": 1, + "discount_gamma": 1, + "exploration_exploitation_ratio": 1, + "kind": "mc_aixi", + "num_simulations": 1, + "predictor": { + "base_depth": 8, + "encoding_bits": 8, + "kind": "fac-ctw", + "num_percept_bits": 16 + } + }, + "environment": { + "action_source": { + "actions": [ + { + "name": "ping", + "payload": "50494e47" + } + ], + "encoding": "hex", + "kind": "literal" + }, + "boot_timeout_ms": 30000, + "debug_mode": true, + "episode_steps": 1000, + "firecracker_config_asset": "firecracker", + "instance_id": "aixi-nyx", + "kind": "nyx_vm", + "observation_bits": 8, + "observation_pad_byte": 0, + "observation_policy": "from_guest", + "observation_stream_len": 1, + "observation_stream_mode": "pad_truncate", + "protocol": { + "action_prefix": "ACT ", + "action_suffix": "\n", + "data_prefix": "DATA ", + "done_prefix": "DONE ", + "obs_prefix": "OBS ", + "rew_prefix": "REW ", + "wire_encoding": "hex" + }, + "reward_bits": 8, + "reward_policy": { + "kind": "from_guest" + }, + "shared_memory_policy": "snapshot", + "shared_region_name": "shared", + "shared_region_size": 4096, + "stats_backend": { + "depth": 8, + "kind": "ctw" + }, + "step_cost": 0, + "step_timeout_ms": 1000 + }, + "interface": { + "agent_actions": 1, + "max_reward": 127, + "min_reward": -128, + "observation_bits": 8, + "observation_key_mode": "full_stream", + "observation_stream_len": 1, + "reward_bits": 8, + "reward_offset": 128 + }, + "kind": "planner_run", + "runtime": { + "explore_epsilon": 0, + "explore_gamma": 1, + "log_every": 1, + "perf": true, + "terminate_lifetime": 5, + "vm_perf_only": true + }, + "schema_version": 1 +} diff --git a/configs/aixi/vm_perf_fast.json b/configs/aixi/vm_perf_fast.json new file mode 100644 index 00000000..613c0816 --- /dev/null +++ b/configs/aixi/vm_perf_fast.json @@ -0,0 +1,86 @@ +{ + "assets": [ + { + "id": "firecracker", + "path": "..\/..\/vendor\/nyx-lite\/vm_image\/vmconfig.json" + } + ], + "controller": { + "agent_horizon": 1, + "discount_gamma": 1, + "exploration_exploitation_ratio": 1, + "kind": "mc_aixi", + "num_simulations": 1, + "predictor": { + "base_depth": 8, + "encoding_bits": 8, + "kind": "fac-ctw", + "num_percept_bits": 16 + } + }, + "environment": { + "action_source": { + "actions": [ + { + "name": "ping", + "payload": "50494e47" + } + ], + "encoding": "hex", + "kind": "literal" + }, + "boot_timeout_ms": 30000, + "debug_mode": false, + "episode_steps": 1000, + "firecracker_config_asset": "firecracker", + "instance_id": "aixi-nyx", + "kind": "nyx_vm", + "observation_bits": 8, + "observation_pad_byte": 0, + "observation_policy": "from_guest", + "observation_stream_len": 1, + "observation_stream_mode": "pad_truncate", + "protocol": { + "action_prefix": "ACT ", + "action_suffix": "\n", + "data_prefix": "DATA ", + "done_prefix": "DONE ", + "obs_prefix": "OBS ", + "rew_prefix": "REW ", + "wire_encoding": "hex" + }, + "reward_bits": 8, + "reward_policy": { + "kind": "from_guest" + }, + "shared_memory_policy": "snapshot", + "shared_region_name": "shared", + "shared_region_size": 4096, + "stats_backend": { + "depth": 8, + "kind": "ctw" + }, + "step_cost": 0, + "step_timeout_ms": 50 + }, + "interface": { + "agent_actions": 1, + "max_reward": 127, + "min_reward": -128, + "observation_bits": 8, + "observation_key_mode": "full_stream", + "observation_stream_len": 1, + "reward_bits": 8, + "reward_offset": 128 + }, + "kind": "planner_run", + "runtime": { + "explore_epsilon": 0, + "explore_gamma": 1, + "log_every": 1, + "perf": true, + "terminate_lifetime": 2000, + "vm_perf_only": true + }, + "schema_version": 1 +} diff --git a/configs/aixi/vm_sudo_fuzz.json b/configs/aixi/vm_sudo_fuzz.json new file mode 100644 index 00000000..32c0027f --- /dev/null +++ b/configs/aixi/vm_sudo_fuzz.json @@ -0,0 +1,123 @@ +{ + "assets": [ + { + "id": "firecracker", + "path": "..\/..\/vendor\/nyx-lite\/vm_image\/vmconfig.json" + } + ], + "controller": { + "agent_horizon": 4, + "discount_gamma": 0.95, + "exploration_exploitation_ratio": 2, + "kind": "mc_aixi", + "num_simulations": 1, + "predictor": { + "base_depth": 32, + "encoding_bits": 8, + "kind": "fac-ctw", + "num_percept_bits": 528 + } + }, + "environment": { + "action_source": { + "dictionary": [], + "encoding": "hex", + "kind": "fuzz", + "max_len": 512, + "min_len": 1, + "mutators": [ + "flip_bit", + "insert_byte", + "delete_byte", + "havoc", + "splice_seed" + ], + "rng_seed": 12345, + "seeds": [ + "7375646f202d6c0a", + "7375646f202d7520726f6f742069640a", + "7375646f202d7523302069640a", + "7375646f202d75232d312069640a", + "7375646f2077686f616d690a", + "7375646f202d730a", + "7375646f202d690a", + "7375646f202d6b0a", + "7375646f202d760a", + "7375646f20656e760a", + "7375646f202d452077686f616d690a", + "7375646f202d482077686f616d690a", + "7375646f202d7520726f6f74202d6720726f6f742069640a", + "7375646f202d2d2069640a", + "7375646f202d6e2077686f616d690a", + "7375646f202d70202750617373776f72643a272077686f616d690a", + "7375646f202d532077686f616d690a", + "7375646f202d412077686f616d690a", + "7375646f202d622077686f616d690a", + "7375646f202d4320352077686f616d690a", + "7375646f202d2d0a", + "7375646f202d75200a", + "7375646f202d752027272069640a", + "7375646f202d7520726f6f742069640a", + "414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141410a", + "7375646f20", + "0a0a0a0a0a", + "00000000", + "7375646f202d752024286964202d75292069640a", + "7375646f202d75206077686f616d69602069640a" + ] + }, + "boot_timeout_ms": 30000, + "crash_log": "sudo_crashes.jsonl", + "debug_mode": false, + "episode_steps": 1, + "firecracker_config_asset": "firecracker", + "instance_id": "aixi-nyx", + "kind": "nyx_vm", + "observation_bits": 8, + "observation_pad_byte": 0, + "observation_policy": "shared_memory", + "observation_stream_len": 64, + "observation_stream_mode": "pad_truncate", + "protocol": { + "action_prefix": "ACT ", + "action_suffix": "\n", + "data_prefix": "DATA ", + "done_prefix": "DONE ", + "obs_prefix": "OBS ", + "rew_prefix": "REW ", + "wire_encoding": "hex" + }, + "reward_bits": 16, + "reward_policy": { + "kind": "from_guest" + }, + "shared_memory_policy": "snapshot", + "shared_region_name": "shared", + "shared_region_size": 4096, + "stats_backend": { + "kind": "rosaplus" + }, + "step_cost": 0, + "step_timeout_ms": 3000 + }, + "interface": { + "agent_actions": 5, + "max_reward": 32767, + "min_reward": -32768, + "observation_bits": 8, + "observation_key_mode": "full_stream", + "observation_stream_len": 64, + "reward_bits": 16, + "reward_offset": 32768 + }, + "kind": "planner_run", + "runtime": { + "explore_epsilon": 0, + "explore_gamma": 1, + "log_every": 500, + "perf": true, + "terminate_lifetime": 10000, + "vm_perf_only": false + }, + "schema_version": 1 +} diff --git a/configs/aixi/vm_sudo_fuzz_test.json b/configs/aixi/vm_sudo_fuzz_test.json new file mode 100644 index 00000000..a55448bf --- /dev/null +++ b/configs/aixi/vm_sudo_fuzz_test.json @@ -0,0 +1,123 @@ +{ + "assets": [ + { + "id": "firecracker", + "path": "..\/..\/vendor\/nyx-lite\/vm_image\/vmconfig.json" + } + ], + "controller": { + "agent_horizon": 4, + "discount_gamma": 0.95, + "exploration_exploitation_ratio": 2, + "kind": "mc_aixi", + "num_simulations": 1, + "predictor": { + "base_depth": 32, + "encoding_bits": 8, + "kind": "fac-ctw", + "num_percept_bits": 528 + } + }, + "environment": { + "action_source": { + "dictionary": [], + "encoding": "hex", + "kind": "fuzz", + "max_len": 512, + "min_len": 1, + "mutators": [ + "flip_bit", + "insert_byte", + "delete_byte", + "havoc", + "splice_seed" + ], + "rng_seed": 12345, + "seeds": [ + "7375646f202d6c0a", + "7375646f202d7520726f6f742069640a", + "7375646f202d7523302069640a", + "7375646f202d75232d312069640a", + "7375646f2077686f616d690a", + "7375646f202d730a", + "7375646f202d690a", + "7375646f202d6b0a", + "7375646f202d760a", + "7375646f20656e760a", + "7375646f202d452077686f616d690a", + "7375646f202d482077686f616d690a", + "7375646f202d7520726f6f74202d6720726f6f742069640a", + "7375646f202d2d2069640a", + "7375646f202d6e2077686f616d690a", + "7375646f202d70202750617373776f72643a272077686f616d690a", + "7375646f202d532077686f616d690a", + "7375646f202d412077686f616d690a", + "7375646f202d622077686f616d690a", + "7375646f202d4320352077686f616d690a", + "7375646f202d2d0a", + "7375646f202d75200a", + "7375646f202d752027272069640a", + "7375646f202d7520726f6f742069640a", + "414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141414141410a", + "7375646f20", + "0a0a0a0a0a", + "00000000", + "7375646f202d752024286964202d75292069640a", + "7375646f202d75206077686f616d69602069640a" + ] + }, + "boot_timeout_ms": 30000, + "crash_log": "sudo_crashes.jsonl", + "debug_mode": false, + "episode_steps": 1, + "firecracker_config_asset": "firecracker", + "instance_id": "aixi-nyx", + "kind": "nyx_vm", + "observation_bits": 8, + "observation_pad_byte": 0, + "observation_policy": "shared_memory", + "observation_stream_len": 64, + "observation_stream_mode": "pad_truncate", + "protocol": { + "action_prefix": "ACT ", + "action_suffix": "\n", + "data_prefix": "DATA ", + "done_prefix": "DONE ", + "obs_prefix": "OBS ", + "rew_prefix": "REW ", + "wire_encoding": "hex" + }, + "reward_bits": 16, + "reward_policy": { + "kind": "from_guest" + }, + "shared_memory_policy": "snapshot", + "shared_region_name": "shared", + "shared_region_size": 4096, + "stats_backend": { + "kind": "rosaplus" + }, + "step_cost": 0, + "step_timeout_ms": 3000 + }, + "interface": { + "agent_actions": 5, + "max_reward": 32767, + "min_reward": -32768, + "observation_bits": 8, + "observation_key_mode": "full_stream", + "observation_stream_len": 64, + "reward_bits": 16, + "reward_offset": 32768 + }, + "kind": "planner_run", + "runtime": { + "explore_epsilon": 0, + "explore_gamma": 1, + "log_every": 50, + "perf": true, + "terminate_lifetime": 500, + "vm_perf_only": false + }, + "schema_version": 1 +} diff --git a/configs/aixi/vm_trace_example.json b/configs/aixi/vm_trace_example.json new file mode 100644 index 00000000..8fde8bd9 --- /dev/null +++ b/configs/aixi/vm_trace_example.json @@ -0,0 +1,95 @@ +{ + "assets": [ + { + "id": "firecracker", + "path": "..\/..\/vendor\/nyx-lite\/vm_image\/vmconfig.json" + } + ], + "controller": { + "agent_horizon": 8, + "discount_gamma": 1, + "exploration_exploitation_ratio": 1.4, + "kind": "mc_aixi", + "num_simulations": 100, + "predictor": { + "base_depth": 32, + "encoding_bits": 8, + "kind": "fac-ctw", + "num_percept_bits": 528 + } + }, + "environment": { + "action_source": { + "actions": [ + { + "name": "ping", + "payload": "50494e47" + }, + { + "name": "status", + "payload": "535441545553" + } + ], + "encoding": "hex", + "kind": "literal" + }, + "boot_timeout_ms": 30000, + "debug_mode": false, + "episode_steps": 8, + "firecracker_config_asset": "firecracker", + "instance_id": "aixi-nyx", + "kind": "nyx_vm", + "observation_bits": 8, + "observation_pad_byte": 0, + "observation_policy": "shared_memory", + "observation_stream_len": 64, + "observation_stream_mode": "pad_truncate", + "protocol": { + "action_prefix": "ACT ", + "action_suffix": "\n", + "data_prefix": "DATA ", + "done_prefix": "DONE ", + "obs_prefix": "OBS ", + "rew_prefix": "REW ", + "wire_encoding": "hex" + }, + "reward_bits": 16, + "reward_policy": { + "kind": "from_guest" + }, + "shared_memory_policy": "snapshot", + "shared_region_name": "shared", + "shared_region_size": 4096, + "stats_backend": { + "depth": 32, + "kind": "ctw" + }, + "step_cost": 1, + "step_timeout_ms": 5000, + "trace": { + "max_bytes": 1048576, + "reset_on_episode": false, + "shared_region_name": "trace" + } + }, + "interface": { + "agent_actions": 2, + "max_reward": 32767, + "min_reward": -32768, + "observation_bits": 8, + "observation_key_mode": "full_stream", + "observation_stream_len": 64, + "reward_bits": 16, + "reward_offset": 32768 + }, + "kind": "planner_run", + "runtime": { + "explore_epsilon": 0, + "explore_gamma": 1, + "log_every": 1, + "perf": false, + "terminate_lifetime": 50, + "vm_perf_only": false + }, + "schema_version": 1 +} diff --git a/configs/bench/extra.json b/configs/bench/extra.json new file mode 100644 index 00000000..9ca26cdb --- /dev/null +++ b/configs/bench/extra.json @@ -0,0 +1,23 @@ +{ + "kind": "neural", + "alpha": 0.03, + "experts": [ + { + "name": "mamba", + "kind": "mamba", + "method": "cfg:hidden=64,layers=1,intermediate=128,state=16,conv=4,dt_rank=16,seed=26,train=adam,lr=0.001,stride=1;policy:schedule=0..10%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|10%..100%:infer", + "log_prior": 0.0 + }, + { + "name": "particle", + "kind": "particle", + "spec_path": "particle_fast.json", + "log_prior": 0.0 + }, + { + "name": "sparse-match", + "kind": "sparse-match", + "log_prior": 0.0 + } + ] +} diff --git a/configs/bench/mixture.json b/configs/bench/mixture.json new file mode 100644 index 00000000..fdff2f55 --- /dev/null +++ b/configs/bench/mixture.json @@ -0,0 +1,38 @@ +{ + "experts": [ + { + "encoding_bits": 8, + "log_prior": 0, + "base_depth": 32, + "name": "fac-ctw", + "kind": "fac-ctw", + "num_percept_bits": 8 + }, + { + "log_prior": 0, + "order": 12, + "name": "ppmd", + "kind": "ppmd", + "memory_mb": 256 + }, + { + "name": "rosa", + "log_prior": 0, + "kind": "rosaplus", + "max_order": -1 + }, + { + "name": "match", + "log_prior": 0, + "kind": "match" + }, + { + "name": "rwkv7", + "method": "cfg:hidden=64,layers=1,intermediate=64,decay_rank=16,a_rank=16,v_rank=16,g_rank=16,seed=22,train=adam,lr=0.0009,stride=1;policy:schedule=0..0.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|0.5%..1%:infer|1%..1.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|1.5%..2%:infer|2%..2.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|2.5%..3%:infer|3%..3.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|3.5%..4%:infer|4%..4.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|4.5%..5%:infer|5%..5.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|5.5%..6%:infer|6%..6.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|6.5%..7%:infer|7%..7.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|7.5%..8%:infer|8%..8.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|8.5%..9%:infer|9%..9.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|9.5%..12%:train(scope=all,opt=adam,lr=0.0001,stride=1,bptt=8,clip=0,momentum=0.9)|12%..17%:infer|17%..18%:train(scope=all,opt=adam,lr=0.0001,stride=1,bptt=8,clip=0,momentum=0.9)|18%..100%:infer", + "kind": "rwkv7", + "log_prior": 0 + } + ], + "kind": "neural", + "alpha": 0.1 +} diff --git a/configs/bench/particle_fast.json b/configs/bench/particle_fast.json new file mode 100644 index 00000000..5cbf9cd4 --- /dev/null +++ b/configs/bench/particle_fast.json @@ -0,0 +1,30 @@ +{ + "num_particles": 1, + "context_window": 4, + "unroll_steps": 1, + "num_cells": 2, + "cell_dim": 4, + "num_rules": 1, + "selector_hidden": 4, + "rule_hidden": 4, + "noise_dim": 0, + "deterministic": true, + "enable_noise": false, + "noise_scale": 0.0, + "noise_anneal_steps": 0, + "learning_rate_readout": 0.0003, + "learning_rate_selector": 0.0, + "learning_rate_rule": 0.0, + "bptt_depth": 1, + "optimizer_momentum": 0.0, + "grad_clip": 1.0, + "state_clip": 8.0, + "forget_lambda": 0.0, + "resample_threshold": 0.000001, + "mutate_fraction": 0.0, + "mutate_scale": 0.0, + "mutate_model_params": false, + "diagnostics_interval": 0, + "min_prob": 5.960464477539063e-08, + "seed": 42 +} diff --git a/configs/bench/two.json b/configs/bench/two.json new file mode 100644 index 00000000..56aeb5ed --- /dev/null +++ b/configs/bench/two.json @@ -0,0 +1,39 @@ +{ + "kind": "neural", + "experts": [ + { + "encoding_bits": 8, + "name": "fac-ctw", + "kind": "fac-ctw", + "num_percept_bits": 8, + "msb_first": true, + "log_prior": 0, + "base_depth": 32 + }, + { + "order": 12, + "memory_mb": 256, + "name": "ppmd", + "kind": "ppmd", + "log_prior": 0 + }, + { + "kind": "rosaplus", + "log_prior": 0, + "name": "rosa", + "max_order": -1 + }, + { + "kind": "match", + "name": "match", + "log_prior": 0 + }, + { + "kind": "rwkv7", + "log_prior": 0, + "name": "rwkv7", + "method": "cfg:hidden=64,layers=1,intermediate=64,decay_rank=16,a_rank=16,v_rank=16,g_rank=16,seed=22,train=adam,lr=0.0009,stride=1;policy:schedule=0..0.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|0.5%..1%:infer|1%..1.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|1.5%..2%:infer|2%..2.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|2.5%..3%:infer|3%..3.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|3.5%..4%:infer|4%..4.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|4.5%..5%:infer|5%..5.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|5.5%..6%:infer|6%..6.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|6.5%..7%:infer|7%..7.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|7.5%..8%:infer|8%..8.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|8.5%..9%:infer|9%..9.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|9.5%..12%:train(scope=all,opt=adam,lr=0.0001,stride=1,bptt=8,clip=0,momentum=0.9)|12%..17%:infer|17%..18%:train(scope=all,opt=adam,lr=0.0001,stride=1,bptt=8,clip=0,momentum=0.9)|18%..100%:infer" + } + ], + "alpha": 0.03 +} diff --git a/benchman/Cargo.toml b/crates/benchman/Cargo.toml similarity index 100% rename from benchman/Cargo.toml rename to crates/benchman/Cargo.toml diff --git a/benchman/src/log_loss.rs b/crates/benchman/src/log_loss.rs similarity index 100% rename from benchman/src/log_loss.rs rename to crates/benchman/src/log_loss.rs diff --git a/benchman/src/main.rs b/crates/benchman/src/main.rs similarity index 98% rename from benchman/src/main.rs rename to crates/benchman/src/main.rs index e83ebdd3..f802e6ac 100644 --- a/benchman/src/main.rs +++ b/crates/benchman/src/main.rs @@ -63,14 +63,14 @@ impl BenchSuite { fn spec_label(self) -> &'static str { match self { - Self::TwoJson => "examples/two.json", - Self::Extra => "examples/extra.json", + Self::TwoJson => "configs/bench/two.json", + Self::Extra => "configs/bench/extra.json", } } fn focus_subjects(self) -> &'static [&'static str] { match self { - Self::TwoJson => &["neural_mixture", "rwkv"], + Self::TwoJson => &["neural_mixture", "rwkv7"], Self::Extra => &["neural_mixture", "mamba"], } } @@ -923,7 +923,7 @@ fn parse_subject_filter(raw: Option<&str>) -> Result>> { .map(str::trim) .filter(|s| !s.is_empty()) { - selected.insert(token.to_string()); + selected.insert(canonicalize_subject(token).to_string()); } if selected.is_empty() { bail!("subject filter was provided but contained no subjects"); @@ -931,6 +931,10 @@ fn parse_subject_filter(raw: Option<&str>) -> Result>> { Ok(Some(selected)) } +fn canonicalize_subject(subject: &str) -> &str { + if subject == "rwkv" { "rwkv7" } else { subject } +} + fn ensure_plot_artifacts(inputs: &ResolvedInputs) -> Result<()> { let plot_dir = &inputs.plot_dir; let should_rebuild = if plot_dir.exists() { @@ -1133,10 +1137,11 @@ fn run_plot_script(inputs: &ResolvedInputs) -> Result<()> { fn find_repo_root() -> Result { let from_manifest = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - if let Some(parent) = from_manifest.parent() { - let candidate = parent.to_path_buf(); - if candidate.join("scripts/plot_two_json.sh").is_file() { - return Ok(candidate); + for candidate in from_manifest.ancestors().skip(1) { + if candidate.join("scripts/plot_two_json.sh").is_file() + && candidate.join("projman.sh").is_file() + { + return Ok(candidate.to_path_buf()); } } @@ -1258,7 +1263,7 @@ fn load_summary_rows( ) })?; - let subject = get_field(&row, idx_subject).trim().to_string(); + let subject = canonicalize_subject(get_field(&row, idx_subject).trim()).to_string(); if let Some(selected) = subject_filter && !selected.contains(&subject) { @@ -1266,7 +1271,9 @@ fn load_summary_rows( } let operation = get_field(&row, idx_operation).trim().to_string(); - let series = get_field(&row, idx_series).trim().to_string(); + let series = get_field(&row, idx_series) + .trim() + .replace(":rwkv", ":rwkv7"); let size_bytes = parse_size_bytes(get_field(&row, idx_size_bytes), row_idx + 2, path)?; let row = RenderRow { @@ -1345,7 +1352,7 @@ fn load_raw_rows( ) })?; - let subject = get_field(&row, idx_subject).trim().to_string(); + let subject = canonicalize_subject(get_field(&row, idx_subject).trim()).to_string(); if let Some(selected) = subject_filter && !selected.contains(&subject) { diff --git a/crates/infotheory/Cargo.toml b/crates/infotheory/Cargo.toml new file mode 100644 index 00000000..b7d2ba95 --- /dev/null +++ b/crates/infotheory/Cargo.toml @@ -0,0 +1,155 @@ +[package] +name = "infotheory" +version = "1.2.0" +edition = "2024" +license = "ISC OR Apache-2.0" +homepage = "https://infotheory.tech" +description = "The algorithmic information theory library." +autobins = false + +[dependencies] +zpaq_rs = { version = "1.0.5", path = "../../vendor/zpaq_rs", optional = true } +nyx-lite = { path = "../../vendor/nyx-lite", optional = true } +gameengine = { version = "0.3.1", path = "../../vendor/gameengine", optional = true, default-features = false } +rayon = "1.11.0" +num_cpus = "1.17.0" +once_cell = "1.21.3" +serde_json = "1.0.149" +anyhow = "1.0.100" +ahash = { version = "0.8.12", default-features = false, features = ["std", "no-rng"] } +crc32fast = "1.5.0" +# Only used by the `aixi` warm-start task fingerprint; gated to keep skinny +# (non-`aixi`) builds free of a digest dependency they cannot use. +sha2 = { version = "0.10", optional = true } +wide = "1.1.1" +libc = { version = "0.2.177", optional = true } + +[features] +default = ["default-backends", "aixi"] +# Grouped capability topology. +default-backends = ["capability-default"] +capability-default = [ + "capability-statistical", + "capability-neural", + "capability-archive", +] +capability-statistical = [ + "backend-rosa", + "backend-ctw", + "backend-match", + "backend-ppmd", + "backend-sequitur", + "backend-mixture", + "backend-particle", + "backend-calibrated", +] +capability-neural = [ + "backend-mamba", + "backend-rwkv", +] +capability-archive = [ + "backend-zpaq", +] +capability-vm = ["aixi-vm"] +aixi = ["dep:sha2"] +tuner = ["aixi", "dep:libc"] +aixi-gameengine = ["aixi", "dep:gameengine", "gameengine/builtin"] +aixi-gameengine-physics = ["aixi-gameengine", "gameengine/physics"] +aixi-vm = ["vm"] + +# Legacy compatibility aliases. +all-backends = ["capability-default"] +backend-rosa = [] +backend-ctw = [] +backend-match = [] +backend-ppmd = [] +backend-sequitur = [] +backend-mixture = [] +backend-particle = [] +backend-calibrated = [] +backend-mamba = [] +backend-rwkv = [] +backend-zpaq = ["dep:zpaq_rs"] +research-tooling = [] +cli = ["aixi"] +vm = ["aixi", "dep:nyx-lite"] + +[[bin]] +name = "infotheory" +path = "src/main.rs" +required-features = ["cli"] + + +[[bench]] +name = "par" +harness = false + +[[bench]] +name = "aixi" +harness = false + +[[bench]] +name = "aiqi" +harness = false +required-features = ["aixi"] + +[[bench]] +name = "mixture_backends" +harness = false + +[[bench]] +name = "neural_baseline" +harness = false + +[[bench]] +name = "simd_hotspots" +harness = false + +[[bench]] +name = "rate_backend_coders" +harness = false +required-features = ["backend-rwkv"] + +[[bench]] +name = "mamba_rate" +harness = false +required-features = ["backend-mamba"] + +[[bench]] +name = "mamba_online_train_full" +harness = false +required-features = ["backend-mamba"] + +[[bench]] +name = "rwkv_online_train_full" +harness = false +required-features = ["backend-rwkv"] + +[[bench]] +name = "exact_hotpaths" +harness = false +required-features = ["backend-rwkv", "backend-mamba"] + +[[bench]] +name = "compiled_plan_overhead" +harness = false + +[[bench]] +name = "mcts_planners" +harness = false +required-features = ["aixi"] + +[[bench]] +name = "warmstart" +harness = false +required-features = ["aixi", "backend-ctw"] + +[dev-dependencies] +serde = { version = "1.0", features = ["derive"] } +# Available to feature-light test/bench builds (e.g. `backend-zpaq` without +# `aixi`) where the optional main-crate `sha2` dependency is not enabled. +sha2 = "0.10" +criterion = { version = "0.5", default-features = false, features = ["cargo_bench_support"] } + +[lints] +workspace = true diff --git a/benches/aiqi.rs b/crates/infotheory/benches/aiqi.rs similarity index 55% rename from benches/aiqi.rs rename to crates/infotheory/benches/aiqi.rs index cc79bf96..d18f667d 100644 --- a/benches/aiqi.rs +++ b/crates/infotheory/benches/aiqi.rs @@ -1,5 +1,6 @@ use infotheory::aixi::aiqi::{AiqiAgent, AiqiConfig}; -use infotheory::{MixtureExpertSpec, MixtureKind, MixtureSpec, RateBackend}; +use infotheory::aixi::common::ActionAlphabet; +use infotheory::api::{MixtureExpertSpec, MixtureKind, MixtureSpec, RateBackend}; use std::hint::black_box; use std::sync::Arc; use std::time::Instant; @@ -12,45 +13,40 @@ fn env_usize(name: &str, default: usize) -> usize { .unwrap_or(default) } -fn base_cfg(algorithm: &str) -> AiqiConfig { - AiqiConfig { - algorithm: algorithm.to_string(), - ct_depth: 12, - observation_bits: 1, - observation_stream_len: 1, - reward_bits: 1, - agent_actions: 2, - min_reward: 0, - max_reward: 1, - reward_offset: 0, - discount_gamma: 0.99, - return_horizon: 4, - return_bins: 8, - augmentation_period: 4, - history_prune_keep_steps: None, - baseline_exploration: 1e-12, - random_seed: Some(7), - rate_backend: None, - rate_backend_max_order: 8, - rwkv_model_path: None, - rosa_max_order: Some(8), - zpaq_method: None, - } +fn base_cfg(backend: RateBackend) -> AiqiConfig { + let mut cfg = AiqiConfig::default(); + cfg.rate_backend = backend; + cfg.observation_bits = 1; + cfg.observation_stream_len = 1; + cfg.reward_bits = 1; + cfg.agent_actions = + ActionAlphabet::try_from_usize(2).expect("benchmark action alphabet must be non-zero"); + cfg.min_reward = 0; + cfg.max_reward = 1; + cfg.reward_offset = 0; + cfg.discount_gamma = 0.99; + cfg.return_horizon = 4; + cfg.return_bins = 8; + cfg.augmentation_period = 4; + cfg.history_prune_keep_steps = None; + cfg.baseline_exploration = 1e-12; + cfg.random_seed = Some(7); + cfg } fn mixture_backend() -> RateBackend { let experts = vec![ - MixtureExpertSpec { - name: Some("ctw".to_string()), - log_prior: 0.0, - max_order: -1, - backend: RateBackend::Ctw { depth: 10 }, + { + let mut expert = MixtureExpertSpec::new(RateBackend::Ctw { depth: 10 }); + expert.name = Some("ctw".to_string()); + expert.log_prior = 0.0; + expert }, - MixtureExpertSpec { - name: Some("rosa".to_string()), - log_prior: 0.0, - max_order: 8, - backend: RateBackend::RosaPlus, + { + let mut expert = MixtureExpertSpec::new(RateBackend::RosaPlus { max_order: 8 }); + expert.name = Some("rosa".to_string()); + expert.log_prior = 0.0; + expert }, ]; RateBackend::Mixture { @@ -73,17 +69,10 @@ fn main() { let iterations = env_usize("AIQI_BENCH_ITERS", 1_000); let seed_steps = env_usize("AIQI_BENCH_HISTORY", 128); - let rate_backend_cfg = |backend: RateBackend| { - let mut cfg = base_cfg("ignored-by-rate-backend"); - cfg.rate_backend = Some(backend); - cfg - }; - let benches = [ - ("ac-ctw", base_cfg("ac-ctw")), - ("rosa", base_cfg("rosa")), - ("rate-rosa", rate_backend_cfg(RateBackend::RosaPlus)), - ("mix-bayes", rate_backend_cfg(mixture_backend())), + ("ctw", base_cfg(RateBackend::Ctw { depth: 12 })), + ("rosaplus", base_cfg(RateBackend::RosaPlus { max_order: 8 })), + ("mix-bayes", base_cfg(mixture_backend())), ]; println!( diff --git a/crates/infotheory/benches/aixi.rs b/crates/infotheory/benches/aixi.rs new file mode 100644 index 00000000..2c41b6b1 --- /dev/null +++ b/crates/infotheory/benches/aixi.rs @@ -0,0 +1,203 @@ +#[cfg(not(feature = "aixi-gameengine"))] +fn main() { + eprintln!("bench 'aixi' requires feature 'aixi-gameengine'"); +} + +#[cfg(feature = "aixi-gameengine")] +mod bench_impl { + use infotheory::aixi::agent::{Agent, AgentConfig}; + use infotheory::aixi::environment::Environment; + use infotheory::aixi::gameengine::build_builtin_environment; + use infotheory::api::{ + MixtureExpertSpec, MixtureKind, MixtureScheduleMode, MixtureSpec, RateBackend, + }; + use infotheory::spec::BuiltinEnvironmentSpec; + use std::sync::Arc; + use std::time::{Duration, Instant}; + + fn env_usize(name: &str, default: usize) -> usize { + std::env::var(name) + .ok() + .and_then(|value| value.parse::().ok()) + .filter(|&value| value > 0) + .unwrap_or(default) + } + + fn bench_agent( + mut agent: Agent, + mut env: Box, + cycles: usize, + warmup: usize, + ) -> Duration { + let mut prev_action = 0u64; + let mut obs_stream = env.drain_observations(); + let mut rew = env.get_reward(); + + for _ in 0..warmup { + agent.model_update_percept_stream(&obs_stream, rew); + let action = agent.get_planned_action(&obs_stream, rew, prev_action); + agent.model_update_action_external(action); + env.perform_action(action); + obs_stream = env.drain_observations(); + rew = env.get_reward(); + prev_action = action; + } + + let now = Instant::now(); + for _ in 0..cycles { + agent.model_update_percept_stream(&obs_stream, rew); + let action = agent.get_planned_action(&obs_stream, rew, prev_action); + agent.model_update_action_external(action); + env.perform_action(action); + obs_stream = env.drain_observations(); + rew = env.get_reward(); + prev_action = action; + } + now.elapsed() + } + + pub(super) fn run() { + let cycles = env_usize("AIXI_BENCH_CYCLES", 2_000); + let warmup = env_usize("AIXI_BENCH_WARMUP", 200); + let env_name = "blackjack"; + + let base_cfg = |backend: RateBackend| { + let mut cfg = AgentConfig::default(); + cfg.rate_backend = backend; + cfg.agent_horizon = 5; + cfg.observation_bits = 64; + cfg.observation_stream_len = 4; + cfg.observation_key_mode = infotheory::aixi::common::ObservationKeyMode::FullStream; + cfg.reward_bits = 2; + cfg.agent_actions = 2; + cfg.num_simulations = 400; + cfg.exploration_exploitation_ratio = 1.4; + cfg.discount_gamma = 1.0; + cfg.min_reward = -1; + cfg.max_reward = 1; + cfg.reward_offset = 1; + cfg.random_seed = Some(1); + cfg + }; + + let make_mixture = + |kind: MixtureKind, alpha: f64, schedule: MixtureScheduleMode| RateBackend::Mixture { + spec: Arc::new( + MixtureSpec::new( + kind, + vec![ + { + let mut expert = + MixtureExpertSpec::new(RateBackend::Ctw { depth: 32 }); + expert.name = Some("ctw".to_string()); + expert.log_prior = 0.0; + expert + }, + { + let mut expert = + MixtureExpertSpec::new(RateBackend::RosaPlus { max_order: 20 }); + expert.name = Some("rosa".to_string()); + expert.log_prior = 0.0; + expert + }, + ], + ) + .with_schedule(schedule) + .with_alpha(alpha), + ), + }; + + let rate_backend_cfg = |backend: RateBackend| { + let mut cfg = base_cfg(RateBackend::Ctw { depth: 32 }); + cfg.rate_backend = backend; + cfg + }; + + let benches = [ + ( + "fac-ctw", + base_cfg(RateBackend::FacCtw { + base_depth: 32, + num_percept_bits: 258, + encoding_bits: 1, + msb_first: None, + }), + ), + ( + "rosaplus", + base_cfg(RateBackend::RosaPlus { max_order: 20 }), + ), + ("rate-ctw", rate_backend_cfg(RateBackend::Ctw { depth: 32 })), + ( + "rate-rosa", + rate_backend_cfg(RateBackend::RosaPlus { max_order: 20 }), + ), + ( + "mix-bayes", + rate_backend_cfg(make_mixture( + MixtureKind::Bayes, + 0.01, + MixtureScheduleMode::Default, + )), + ), + ( + "mix-switch", + rate_backend_cfg(make_mixture( + MixtureKind::Switching, + 0.17, + MixtureScheduleMode::Default, + )), + ), + ( + "mix-switch-thm", + rate_backend_cfg(make_mixture( + MixtureKind::Switching, + 0.99, + MixtureScheduleMode::Theorem, + )), + ), + ( + "mix-convex", + rate_backend_cfg(make_mixture( + MixtureKind::Convex, + 1.25, + MixtureScheduleMode::Default, + )), + ), + ( + "mix-convex-thm", + rate_backend_cfg(make_mixture( + MixtureKind::Convex, + 7.5, + MixtureScheduleMode::Theorem, + )), + ), + ]; + + println!( + "MC-AIXI benchmark (env={}, warmup={}, cycles={})", + env_name, warmup, cycles + ); + for (name, cfg) in benches { + let env = build_builtin_environment(BuiltinEnvironmentSpec::Blackjack) + .expect("blackjack builtin env"); + let agent = Agent::new(cfg.clone()); + let elapsed = bench_agent(agent, env, cycles, warmup); + + let ns_per_cycle = (elapsed.as_nanos() as f64) / (cycles as f64); + let cycles_per_s = (cycles as f64) / elapsed.as_secs_f64().max(1e-12); + println!( + "{:>7}: {:>8.3} ms total | {:>10.1} ns/cycle | {:>10.1} cycles/s", + name, + elapsed.as_secs_f64() * 1e3, + ns_per_cycle, + cycles_per_s + ); + } + } +} + +#[cfg(feature = "aixi-gameengine")] +fn main() { + bench_impl::run(); +} diff --git a/crates/infotheory/benches/compiled_plan_overhead.rs b/crates/infotheory/benches/compiled_plan_overhead.rs new file mode 100644 index 00000000..9946f53c --- /dev/null +++ b/crates/infotheory/benches/compiled_plan_overhead.rs @@ -0,0 +1,207 @@ +use criterion::{BenchmarkId, Criterion, Throughput, black_box, criterion_group, criterion_main}; +use infotheory::api::{ + CompressionBackend, RateBackend, RateBackendSession, try_compress_size_backend, + try_entropy_rate_backend, +}; +#[cfg(all( + feature = "backend-mixture", + feature = "backend-ctw", + feature = "backend-match" +))] +use infotheory::api::{MixtureExpertSpec, MixtureKind, MixtureSpec}; +use infotheory::coders::CoderType; +use infotheory::compression::FramingMode; +#[cfg(all( + feature = "backend-mixture", + feature = "backend-ctw", + feature = "backend-match" +))] +use std::sync::Arc; + +fn short_bench_data() -> Vec { + let seed = b"compiled-plan-overhead"; + let mut out = Vec::with_capacity(64); + while out.len() < 64 { + out.extend_from_slice(seed); + } + out.truncate(64); + out +} + +fn bench_case(c: &mut Criterion, label: &str, rate_backend: RateBackend, data: &[u8]) { + let compiled_rate = rate_backend + .compile() + .expect("compile benchmark rate backend"); + let compression_backend = CompressionBackend::Rate { + rate_backend: rate_backend.clone(), + coder: CoderType::AC, + framing: FramingMode::Framed, + }; + let compiled_compression = compression_backend + .compile() + .expect("compile benchmark compression backend"); + + let mut setup_group = c.benchmark_group("compiled_plan_overhead_setup"); + setup_group.bench_function(format!("{label}_rate_compiled_clone"), |b| { + b.iter(|| { + black_box(compiled_rate.clone()); + }); + }); + setup_group.bench_function(format!("{label}_rate_compile_each_call"), |b| { + b.iter(|| { + black_box(&rate_backend) + .clone() + .compile() + .expect("compile-each-call rate setup benchmark"); + }); + }); + setup_group.bench_function(format!("{label}_compression_compiled_clone"), |b| { + b.iter(|| { + black_box(compiled_compression.clone()); + }); + }); + setup_group.bench_function(format!("{label}_compression_compile_each_call"), |b| { + b.iter(|| { + black_box(&compression_backend) + .clone() + .compile() + .expect("compile-each-call compression setup benchmark"); + }); + }); + setup_group.finish(); + + let mut entropy_group = c.benchmark_group("compiled_plan_overhead_entropy"); + entropy_group.throughput(Throughput::Bytes(data.len() as u64)); + entropy_group.bench_with_input( + BenchmarkId::new(format!("{label}_compiled_reuse"), data.len()), + data, + |b, d| { + b.iter(|| { + try_entropy_rate_backend(black_box(d), black_box(&compiled_rate)) + .expect("compiled entropy benchmark"); + }); + }, + ); + entropy_group.bench_with_input( + BenchmarkId::new(format!("{label}_compile_each_call"), data.len()), + data, + |b, d| { + b.iter(|| { + let compiled = black_box(&rate_backend) + .compile() + .expect("compile-each-call rate backend"); + try_entropy_rate_backend(black_box(d), black_box(&compiled)) + .expect("compile-each-call entropy benchmark"); + }); + }, + ); + entropy_group.finish(); + + let mut compression_group = c.benchmark_group("compiled_plan_overhead_compression"); + compression_group.throughput(Throughput::Bytes(data.len() as u64)); + compression_group.bench_with_input( + BenchmarkId::new(format!("{label}_compiled_reuse"), data.len()), + data, + |b, d| { + b.iter(|| { + try_compress_size_backend(black_box(d), black_box(&compiled_compression)) + .expect("compiled compression benchmark"); + }); + }, + ); + compression_group.bench_with_input( + BenchmarkId::new(format!("{label}_compile_each_call"), data.len()), + data, + |b, d| { + b.iter(|| { + let compiled = black_box(&compression_backend) + .compile() + .expect("compile-each-call compression backend"); + try_compress_size_backend(black_box(d), black_box(&compiled)) + .expect("compile-each-call compression benchmark"); + }); + }, + ); + compression_group.finish(); + + let mut session_group = c.benchmark_group("compiled_plan_overhead_session"); + session_group.throughput(Throughput::Bytes(data.len() as u64)); + session_group.bench_with_input( + BenchmarkId::new(format!("{label}_compiled_reuse"), data.len()), + data, + |b, d| { + b.iter(|| { + let mut session = + RateBackendSession::from_backend(black_box(compiled_rate.clone()), None) + .expect("compiled session benchmark"); + session.observe(black_box(d)); + let mut log_probs = [0.0; 256]; + session.fill_log_probs(&mut log_probs); + black_box(log_probs); + }); + }, + ); + session_group.bench_with_input( + BenchmarkId::new(format!("{label}_compile_each_call"), data.len()), + data, + |b, d| { + b.iter(|| { + let mut session = + RateBackendSession::from_spec(black_box(rate_backend.clone()), None) + .expect("compile-each-call session benchmark"); + session.observe(black_box(d)); + let mut log_probs = [0.0; 256]; + session.fill_log_probs(&mut log_probs); + black_box(log_probs); + }); + }, + ); + session_group.finish(); +} + +#[cfg(all( + feature = "backend-mixture", + feature = "backend-ctw", + feature = "backend-match" +))] +fn nested_mixture_backend() -> RateBackend { + RateBackend::Mixture { + spec: Arc::new(MixtureSpec::new( + MixtureKind::Bayes, + vec![ + { + let mut expert = MixtureExpertSpec::new(RateBackend::Ctw { depth: 8 }); + expert.name = Some("ctw".to_string()); + expert.log_prior = 0.0; + expert + }, + { + let mut expert = MixtureExpertSpec::new(RateBackend::Match { + hash_bits: 18, + min_len: 4, + max_len: 64, + base_mix: 0.02, + confidence_scale: 1.0, + }); + expert.name = Some("match".to_string()); + expert.log_prior = -0.15; + expert + }, + ], + )), + } +} + +fn bench_compiled_plan_overhead(c: &mut Criterion) { + let data = short_bench_data(); + bench_case(c, "ctw", RateBackend::Ctw { depth: 8 }, &data); + #[cfg(all( + feature = "backend-mixture", + feature = "backend-ctw", + feature = "backend-match" + ))] + bench_case(c, "mixture", nested_mixture_backend(), &data); +} + +criterion_group!(compiled_plan_overhead, bench_compiled_plan_overhead); +criterion_main!(compiled_plan_overhead); diff --git a/benches/exact_hotpaths.rs b/crates/infotheory/benches/exact_hotpaths.rs similarity index 89% rename from benches/exact_hotpaths.rs rename to crates/infotheory/benches/exact_hotpaths.rs index 2f7a29f6..90767921 100644 --- a/benches/exact_hotpaths.rs +++ b/crates/infotheory/benches/exact_hotpaths.rs @@ -1,28 +1,33 @@ #![cfg(all(feature = "backend-rwkv", feature = "backend-mamba"))] use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; +use infotheory::api::{ + CompiledRateBackend, MixtureExpertSpec, MixtureKind, MixtureSpec, RateBackend, +}; use infotheory::backends::ctw::FacContextTree; use infotheory::backends::llm_policy::OptimizerKind; use infotheory::coders::CoderType; use infotheory::compression::{FramingMode, compress_rate_bytes}; use infotheory::mambazip::mamba1; use infotheory::rwkvzip::rwkv7; -use infotheory::{MixtureExpertSpec, MixtureKind, MixtureSpec, RateBackend}; use std::sync::Arc; use std::time::Duration; const DATA_LEN: usize = 16 * 1024; fn bench_data() -> Vec { + let repo_root = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("..") + .join(".."); let seeds = [ - "README.md", - "LICENSE-APACHE", - "Cargo.toml", - "examples/two.json", + repo_root.join("README.md"), + repo_root.join("LICENSE-APACHE"), + repo_root.join("Cargo.toml"), + repo_root.join("configs/bench/two.json"), ]; let mut out = Vec::with_capacity(DATA_LEN); while out.len() < DATA_LEN { - for path in seeds { + for path in &seeds { let chunk = std::fs::read(path).expect("failed to read bench seed file"); out.extend_from_slice(&chunk); if out.len() >= DATA_LEN { @@ -92,43 +97,32 @@ fn mamba_cfg() -> mamba1::Config { } } -fn two_json_backend() -> RateBackend { +fn two_json_backend() -> CompiledRateBackend { let spec = MixtureSpec::new( MixtureKind::Neural, vec![ - MixtureExpertSpec { - name: Some("ctw".to_string()), - log_prior: 0.0, - max_order: -1, - backend: RateBackend::Ctw { depth: 24 }, - }, - MixtureExpertSpec { - name: Some("ppmd".to_string()), - log_prior: 0.0, - max_order: -1, - backend: RateBackend::Ppmd { - order: 10, - memory_mb: 64, - }, - }, - MixtureExpertSpec { - name: Some("match".to_string()), - log_prior: 0.0, - max_order: -1, - backend: RateBackend::Match { - hash_bits: 20, - min_len: 4, - max_len: 255, - base_mix: 0.02, - confidence_scale: 1.0, - }, - }, + MixtureExpertSpec::new(RateBackend::Ctw { depth: 24 }).with_name("ctw"), + MixtureExpertSpec::new(RateBackend::Ppmd { + order: 10, + memory_mb: 64, + }) + .with_name("ppmd"), + MixtureExpertSpec::new(RateBackend::Match { + hash_bits: 20, + min_len: 4, + max_len: 255, + base_mix: 0.02, + confidence_scale: 1.0, + }) + .with_name("match"), ], ) .with_alpha(0.03); RateBackend::Mixture { spec: Arc::new(spec), } + .compile() + .expect("compile two.json benchmark backend") } fn bench_rwkv_direct(c: &mut Criterion) { @@ -328,7 +322,7 @@ fn bench_two_json_end_to_end(c: &mut Criterion) { group.throughput(Throughput::Bytes(data.len() as u64)); group.bench_with_input(BenchmarkId::new("rate_ac", data.len()), &data, |b, d| { b.iter(|| { - let encoded = compress_rate_bytes(d, &backend, -1, CoderType::AC, FramingMode::Raw) + let encoded = compress_rate_bytes(d, &backend, CoderType::AC, FramingMode::Raw) .expect("two.json compression bench failed"); criterion::black_box(encoded.len()) }); diff --git a/benches/mamba_online_train_full.rs b/crates/infotheory/benches/mamba_online_train_full.rs similarity index 75% rename from benches/mamba_online_train_full.rs rename to crates/infotheory/benches/mamba_online_train_full.rs index e650c13f..6db1ca06 100644 --- a/benches/mamba_online_train_full.rs +++ b/crates/infotheory/benches/mamba_online_train_full.rs @@ -1,7 +1,7 @@ #![cfg(feature = "backend-mamba")] use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; -use infotheory::{RateBackend, entropy_rate_backend}; +use infotheory::api::{CompiledRateBackend, RateBackend, try_entropy_rate_backend}; use std::time::Duration; const DATA_LEN: usize = 64 * 1024; @@ -18,7 +18,8 @@ fn bench_data() -> Vec { fn backend(method: &str) -> RateBackend { RateBackend::MambaMethod { - method: method.to_string(), + method: infotheory::mambazip::parse_method_spec(method) + .expect("mamba benchmark method must be valid"), } } @@ -26,10 +27,14 @@ fn bench_mamba_online_train_full(c: &mut Criterion) { let data = bench_data(); let infer = backend( "cfg:hidden=128,layers=2,intermediate=256,state=16,conv=4,dt_rank=16,seed=7,train=none,lr=0.0,stride=1;policy:schedule=0..100:infer", - ); + ) + .compile() + .expect("compile mamba infer backend"); let train_full = backend( "cfg:hidden=128,layers=2,intermediate=256,state=16,conv=4,dt_rank=16,seed=7,train=adam,lr=0.001,stride=1;policy:schedule=0..100:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=1,clip=0,momentum=0.9)", - ); + ) + .compile() + .expect("compile mamba train backend"); let mut group = c.benchmark_group("mamba_online_train_full"); group.throughput(Throughput::Bytes(data.len() as u64)); @@ -39,7 +44,7 @@ fn bench_mamba_online_train_full(c: &mut Criterion) { &infer, |b, backend| { b.iter(|| { - let h = entropy_rate_backend(&data, -1, backend); + let h = entropy_rate_backend(&data, backend); criterion::black_box(h) }); }, @@ -50,7 +55,7 @@ fn bench_mamba_online_train_full(c: &mut Criterion) { &train_full, |b, backend| { b.iter(|| { - let h = entropy_rate_backend(&data, -1, backend); + let h = entropy_rate_backend(&data, backend); criterion::black_box(h) }); }, @@ -68,3 +73,6 @@ criterion_group! { targets = bench_mamba_online_train_full } criterion_main!(mamba_online_train_full); +fn entropy_rate_backend(data: &[u8], backend: &CompiledRateBackend) -> f64 { + try_entropy_rate_backend(data, backend).expect("entropy rate") +} diff --git a/benches/mamba_rate.rs b/crates/infotheory/benches/mamba_rate.rs similarity index 73% rename from benches/mamba_rate.rs rename to crates/infotheory/benches/mamba_rate.rs index 15100b09..bc6c2fba 100644 --- a/benches/mamba_rate.rs +++ b/crates/infotheory/benches/mamba_rate.rs @@ -1,9 +1,9 @@ #![cfg(feature = "backend-mamba")] use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; +use infotheory::api::{CompiledRateBackend, RateBackend, try_entropy_rate_backend}; use infotheory::coders::CoderType; use infotheory::compression::{FramingMode, compress_rate_bytes}; -use infotheory::{RateBackend, entropy_rate_backend}; use std::time::Duration; const DATA_LEN: usize = 16 * 1024; @@ -20,13 +20,14 @@ fn bench_data() -> Vec { fn mamba_backend() -> RateBackend { RateBackend::MambaMethod { - method: "cfg:hidden=128,layers=2,intermediate=256,state=16,conv=4,dt_rank=16,train=none,seed=7;policy:schedule=0..100:infer".to_string(), + method: infotheory::mambazip::parse_method_spec("cfg:hidden=128,layers=2,intermediate=256,state=16,conv=4,dt_rank=16,train=none,seed=7;policy:schedule=0..100:infer") + .expect("mamba benchmark method must be valid"), } } fn bench_mamba(c: &mut Criterion) { let data = bench_data(); - let backend = mamba_backend(); + let backend = mamba_backend().compile().expect("compile mamba backend"); let mut h_group = c.benchmark_group("mamba_entropy"); h_group.throughput(Throughput::Bytes(data.len() as u64)); @@ -35,7 +36,7 @@ fn bench_mamba(c: &mut Criterion) { &data, |b, d| { b.iter(|| { - let h = entropy_rate_backend(d, -1, &backend); + let h = entropy_rate_backend(d, &backend); criterion::black_box(h) }); }, @@ -51,7 +52,7 @@ fn bench_mamba(c: &mut Criterion) { &data, |b, d| { b.iter(|| { - let out = compress_rate_bytes(d, &backend, -1, coder, FramingMode::Raw) + let out = compress_rate_bytes(d, &backend, coder, FramingMode::Raw) .expect("mamba rate compression benchmark failed"); criterion::black_box(out.len()) }); @@ -70,3 +71,6 @@ criterion_group! { targets = bench_mamba } criterion_main!(mamba_rate); +fn entropy_rate_backend(data: &[u8], backend: &CompiledRateBackend) -> f64 { + try_entropy_rate_backend(data, backend).expect("entropy rate") +} diff --git a/crates/infotheory/benches/mcts_planners.rs b/crates/infotheory/benches/mcts_planners.rs new file mode 100644 index 00000000..9ef40a1f --- /dev/null +++ b/crates/infotheory/benches/mcts_planners.rs @@ -0,0 +1,295 @@ +use criterion::{BenchmarkId, Criterion, Throughput, black_box, criterion_group, criterion_main}; +use infotheory::aixi::common::{Action, ActionAlphabet, Reward}; +use infotheory::aixi::mcts::{AgentSimulator, ParallelUctPlanner, RhoUctPlanner}; +use rayon::ThreadPool; +use std::num::NonZeroUsize; + +fn nz(n: usize) -> NonZeroUsize { + NonZeroUsize::new(n).expect("benchmark worker count must be non-zero") +} + +#[derive(Clone)] +struct BenchAgent { + num_actions: usize, + horizon: usize, + min_reward: Reward, + max_reward: Reward, + discount_gamma: f64, + explore_exploit_ratio: f64, + step_work: usize, + clone_work: usize, + rng_state: u64, + last_action: Action, + emit_reward: bool, +} + +impl BenchAgent { + fn planner_stub(num_actions: usize, horizon: usize, step_work: usize) -> Self { + Self { + num_actions, + horizon, + min_reward: 0, + max_reward: 1, + discount_gamma: 0.95, + explore_exploit_ratio: 1.0, + step_work, + clone_work: step_work / 2, + rng_state: 1, + last_action: 0, + emit_reward: false, + } + } + + fn burn(&mut self, rounds: usize) -> u64 { + let mut acc = self.rng_state ^ 0x9E37_79B9_7F4A_7C15; + for _ in 0..rounds { + acc ^= acc << 7; + acc ^= acc >> 9; + acc = acc.wrapping_mul(0xA24B_AED4_963E_E407); + } + self.rng_state = acc; + acc + } +} + +impl AgentSimulator for BenchAgent { + fn get_num_actions(&self) -> ActionAlphabet { + ActionAlphabet::try_from_usize(self.num_actions) + .expect("benchmark action alphabet must be non-zero") + } + + fn get_num_observation_bits(&self) -> usize { + 1 + } + + fn get_num_reward_bits(&self) -> usize { + 1 + } + + fn horizon(&self) -> usize { + self.horizon + } + + fn max_reward(&self) -> Reward { + self.max_reward + } + + fn min_reward(&self) -> Reward { + self.min_reward + } + + fn get_explore_exploit_ratio(&self) -> f64 { + self.explore_exploit_ratio + } + + fn discount_gamma(&self) -> f64 { + self.discount_gamma + } + + fn model_update_action(&mut self, action: Action) { + self.last_action = action; + self.emit_reward = false; + let _ = self.burn(self.step_work / 4); + } + + fn gen_percept_and_update(&mut self, _bits: usize) -> u64 { + let mix = self.burn(self.step_work.max(1)); + if self.emit_reward { + self.emit_reward = false; + (self.last_action ^ mix) & 1 + } else { + self.emit_reward = true; + (mix >> 5) & 1 + } + } + + fn begin_simulation(&mut self) { + self.emit_reward = false; + } + + fn begin_discardable_simulation(&mut self) { + self.emit_reward = false; + } + + fn model_revert(&mut self, _steps: usize) { + self.emit_reward = false; + } + + fn gen_range(&mut self, end: usize) -> usize { + if end <= 1 { + return 0; + } + (self.burn(1) as usize) % end + } + + fn gen_f64(&mut self) -> f64 { + const SCALE: f64 = (u64::MAX as f64) + 1.0; + (self.burn(1) as f64) / SCALE + } + + fn boxed_clone_with_seed(&self, seed: u64) -> Box { + let mut clone = self.clone(); + clone.rng_state ^= seed.wrapping_mul(0xD6E8_FD50_5D2B_9B7D); + let _ = clone.burn(clone.clone_work.max(1)); + Box::new(clone) + } +} + +fn planner_pool(threads: usize) -> ThreadPool { + rayon::ThreadPoolBuilder::new() + .num_threads(threads) + .build() + .expect("benchmark thread pool") +} + +fn bench_planner_throughput(c: &mut Criterion) { + let mut group = c.benchmark_group("mcts_planner_throughput"); + let samples = 256usize; + group.throughput(Throughput::Elements(samples as u64)); + + let seq_agent = BenchAgent::planner_stub(8, 6, 8); + group.bench_with_input( + BenchmarkId::new("rho_uct", samples), + &samples, + |b, &samples| { + b.iter(|| { + let mut planner = RhoUctPlanner::new(); + let mut agent = seq_agent.clone(); + black_box(planner.search(&mut agent, &[0], 0, 0, samples)); + }); + }, + ); + + let wu_pool = planner_pool(1); + let wu_agent = BenchAgent::planner_stub(8, 6, 8); + group.bench_with_input( + BenchmarkId::new("parallel_uct_wu_workers1", samples), + &samples, + |b, &samples| { + b.iter(|| { + wu_pool.install(|| { + let mut planner = + ParallelUctPlanner::new(nz(1), None).expect("valid parallel_uct planner"); + let mut agent = wu_agent.clone(); + black_box( + planner + .search(&mut agent, &[0], 0, 0, samples) + .expect("benchmark planner stub uses positive horizon"), + ); + }); + }); + }, + ); + + let par_pool = planner_pool(4); + let par_agent = BenchAgent::planner_stub(8, 6, 8); + group.bench_with_input( + BenchmarkId::new("parallel_uct_wu_workers4", samples), + &samples, + |b, &samples| { + b.iter(|| { + par_pool.install(|| { + let mut planner = + ParallelUctPlanner::new(nz(4), None).expect("valid parallel_uct planner"); + let mut agent = par_agent.clone(); + black_box( + planner + .search(&mut agent, &[0], 0, 0, samples) + .expect("benchmark planner stub uses positive horizon"), + ); + }); + }); + }, + ); + + let bu_pool = planner_pool(4); + let bu_agent = BenchAgent::planner_stub(8, 6, 8); + group.bench_with_input( + BenchmarkId::new("parallel_uct_bu_core_workers4", samples), + &samples, + |b, &samples| { + b.iter(|| { + bu_pool.install(|| { + let mut planner = ParallelUctPlanner::new(nz(4), Some(0.8)) + .expect("valid parallel_uct planner"); + let mut agent = bu_agent.clone(); + black_box( + planner + .search(&mut agent, &[0], 0, 0, samples) + .expect("benchmark planner stub uses positive horizon"), + ); + }); + }); + }, + ); + group.finish(); +} + +fn bench_tuner_shaped_mcts(c: &mut Criterion) { + let mut group = c.benchmark_group("mcts_tuner_shaped"); + let samples = 192usize; + group.throughput(Throughput::Elements(samples as u64)); + + let seq_agent = BenchAgent::planner_stub(64, 4, 48); + group.bench_with_input( + BenchmarkId::new("rho_uct_actions64_h4", samples), + &samples, + |b, &samples| { + b.iter(|| { + let mut planner = RhoUctPlanner::new(); + let mut agent = seq_agent.clone(); + black_box(planner.search(&mut agent, &[0], 0, 0, samples)); + }); + }, + ); + + let wu_pool = planner_pool(4); + let wu_agent = BenchAgent::planner_stub(64, 4, 48); + group.bench_with_input( + BenchmarkId::new("parallel_uct_wu_actions64_h4", samples), + &samples, + |b, &samples| { + b.iter(|| { + wu_pool.install(|| { + let mut planner = + ParallelUctPlanner::new(nz(4), None).expect("valid parallel_uct planner"); + let mut agent = wu_agent.clone(); + black_box( + planner + .search(&mut agent, &[0], 0, 0, samples) + .expect("benchmark planner stub uses positive horizon"), + ); + }); + }); + }, + ); + + let bu_pool = planner_pool(4); + let bu_agent = BenchAgent::planner_stub(64, 4, 48); + group.bench_with_input( + BenchmarkId::new("parallel_uct_bu_core_actions64_h4", samples), + &samples, + |b, &samples| { + b.iter(|| { + bu_pool.install(|| { + let mut planner = ParallelUctPlanner::new(nz(4), Some(0.8)) + .expect("valid parallel_uct planner"); + let mut agent = bu_agent.clone(); + black_box( + planner + .search(&mut agent, &[0], 0, 0, samples) + .expect("benchmark planner stub uses positive horizon"), + ); + }); + }); + }, + ); + group.finish(); +} + +criterion_group!( + mcts_planners, + bench_planner_throughput, + bench_tuner_shaped_mcts +); +criterion_main!(mcts_planners); diff --git a/benches/mixture_backends.rs b/crates/infotheory/benches/mixture_backends.rs similarity index 79% rename from benches/mixture_backends.rs rename to crates/infotheory/benches/mixture_backends.rs index 332110d6..4ea31fec 100644 --- a/benches/mixture_backends.rs +++ b/crates/infotheory/benches/mixture_backends.rs @@ -1,4 +1,7 @@ -use infotheory::{MixtureExpertSpec, MixtureKind, MixtureSpec, RateBackend, entropy_rate_backend}; +use infotheory::api::{ + CompiledRateBackend, MixtureExpertSpec, MixtureKind, MixtureSpec, RateBackend, + try_entropy_rate_backend, +}; use std::env; use std::hint::black_box; use std::sync::Arc; @@ -31,22 +34,14 @@ fn source_data(expand_factor: usize) -> Vec { fn make_experts() -> Vec { vec![ - MixtureExpertSpec { - name: Some("fac".to_string()), - log_prior: 0.0, - max_order: -1, - backend: RateBackend::FacCtw { - base_depth: 16, - encoding_bits: 8, - num_percept_bits: 8, - }, - }, - MixtureExpertSpec { - name: Some("rosa".to_string()), - log_prior: 0.0, - max_order: -1, - backend: RateBackend::RosaPlus, - }, + MixtureExpertSpec::new(RateBackend::FacCtw { + base_depth: 16, + encoding_bits: 8, + num_percept_bits: 8, + msb_first: None, + }) + .with_name("fac"), + MixtureExpertSpec::new(RateBackend::RosaPlus { max_order: -1 }).with_name("rosa"), ] } @@ -69,17 +64,19 @@ fn bench_kind( ) -> Duration { let backend = RateBackend::Mixture { spec: Arc::new(make_spec(kind)), - }; + } + .compile() + .expect("compile mixture backend"); for _ in 0..warmup_iters { - let h = entropy_rate_backend(data, -1, &backend); + let h = entropy_rate_backend(data, &backend); black_box(h); } let start = Instant::now(); let mut sink = 0.0; for _ in 0..bench_iters { - let h = entropy_rate_backend(data, -1, &backend); + let h = entropy_rate_backend(data, &backend); sink += h; } black_box(sink); @@ -122,3 +119,6 @@ fn main() { } println!("total elapsed: {:.3} s", total.as_secs_f64()); } +fn entropy_rate_backend(data: &[u8], backend: &CompiledRateBackend) -> f64 { + try_entropy_rate_backend(data, backend).expect("entropy rate") +} diff --git a/benches/neural_baseline.rs b/crates/infotheory/benches/neural_baseline.rs similarity index 61% rename from benches/neural_baseline.rs rename to crates/infotheory/benches/neural_baseline.rs index 1a6fdb91..34816796 100644 --- a/benches/neural_baseline.rs +++ b/crates/infotheory/benches/neural_baseline.rs @@ -1,4 +1,7 @@ -use infotheory::{MixtureExpertSpec, MixtureKind, MixtureSpec, RateBackend, entropy_rate_backend}; +use infotheory::api::{ + CompiledRateBackend, MixtureExpertSpec, MixtureKind, MixtureSpec, RateBackend, + try_entropy_rate_backend, +}; use std::hint::black_box; use std::sync::Arc; use std::time::Instant; @@ -13,22 +16,14 @@ fn data() -> Vec { fn experts() -> Vec { vec![ - MixtureExpertSpec { - name: Some("fac".to_string()), - log_prior: 0.0, - max_order: -1, - backend: RateBackend::FacCtw { - base_depth: 16, - encoding_bits: 8, - num_percept_bits: 8, - }, - }, - MixtureExpertSpec { - name: Some("rosa".to_string()), - log_prior: 0.0, - max_order: -1, - backend: RateBackend::RosaPlus, - }, + MixtureExpertSpec::new(RateBackend::FacCtw { + base_depth: 16, + encoding_bits: 8, + num_percept_bits: 8, + msb_first: None, + }) + .with_name("fac"), + MixtureExpertSpec::new(RateBackend::RosaPlus { max_order: -1 }).with_name("rosa"), ] } @@ -39,24 +34,24 @@ fn backend(kind: MixtureKind) -> RateBackend { } fn run_one(name: &str, kind: MixtureKind, bytes: &[u8]) { - let backend = backend(kind); + let backend = backend(kind).compile().expect("compile mixture backend"); for _ in 0..WARMUP_ITERS { - let h = entropy_rate_backend(bytes, -1, &backend); + let h = entropy_rate_backend(bytes, &backend); black_box(h); } let start = Instant::now(); let mut sum = 0.0; for _ in 0..BENCH_ITERS { - sum += entropy_rate_backend(bytes, -1, &backend); + sum += entropy_rate_backend(bytes, &backend); } black_box(sum); let elapsed = start.elapsed().as_secs_f64(); let ms = elapsed * 1e3 / (BENCH_ITERS as f64); let mib_s = ((bytes.len() * BENCH_ITERS) as f64) / elapsed / (1024.0 * 1024.0); - let h = entropy_rate_backend(bytes, -1, &backend); + let h = entropy_rate_backend(bytes, &backend); println!( "{name:>7}: {:>9.3} ms/iter | {:>8.3} MiB/s | H={:.9}", ms, mib_s, h @@ -73,3 +68,6 @@ fn main() { run_one("switch", MixtureKind::Switching, &bytes); run_one("bayes", MixtureKind::Bayes, &bytes); } +fn entropy_rate_backend(data: &[u8], backend: &CompiledRateBackend) -> f64 { + try_entropy_rate_backend(data, backend).expect("entropy rate") +} diff --git a/crates/infotheory/benches/par.rs b/crates/infotheory/benches/par.rs new file mode 100644 index 00000000..89008f62 --- /dev/null +++ b/crates/infotheory/benches/par.rs @@ -0,0 +1,68 @@ +use infotheory::api::{ + CompressionBackend, CompressionPathBatchOptions, OperationParallelism, + try_get_compressed_sizes_from_paths_backend_with_options, +}; +use infotheory::spec::CompiledCompressionBackend; +use std::hint::black_box; +use std::time::Instant; + +const RUNS: usize = 32; +const PATHS: &[&str] = &["compressme", "scompressme", "largebench"]; + +fn main() { + let backend = CompressionBackend::try_default() + .expect("default compression backend") + .compile() + .expect("compiled default compression backend"); + + benchmark_variant( + "serial", + PATHS, + &backend, + CompressionPathBatchOptions { + parallelism: OperationParallelism::Serial, + }, + ); + benchmark_variant( + "auto", + PATHS, + &backend, + CompressionPathBatchOptions { + parallelism: OperationParallelism::Auto, + }, + ); + benchmark_variant( + "threads(4)", + PATHS, + &backend, + CompressionPathBatchOptions { + parallelism: OperationParallelism::Threads(4), + }, + ); +} + +fn benchmark_variant( + label: &str, + paths: &[&str], + backend: &CompiledCompressionBackend, + options: CompressionPathBatchOptions, +) { + println!( + "{label} warmup: {:?}", + get_compressed_sizes(paths, backend, options) + ); + let now = Instant::now(); + for _ in 0..RUNS { + black_box(get_compressed_sizes(paths, backend, options)); + } + println!("{label}: elapsed time for {RUNS} runs: {:?}", now.elapsed()); +} + +fn get_compressed_sizes( + paths: &[&str], + backend: &CompiledCompressionBackend, + options: CompressionPathBatchOptions, +) -> Vec { + try_get_compressed_sizes_from_paths_backend_with_options(paths, backend, options) + .expect("compressed sizes") +} diff --git a/benches/rate_backend_coders.rs b/crates/infotheory/benches/rate_backend_coders.rs similarity index 55% rename from benches/rate_backend_coders.rs rename to crates/infotheory/benches/rate_backend_coders.rs index 6d2b4057..6e6dfac5 100644 --- a/benches/rate_backend_coders.rs +++ b/crates/infotheory/benches/rate_backend_coders.rs @@ -1,9 +1,11 @@ #![cfg(feature = "backend-rwkv")] use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; +use infotheory::api::{ + CompiledRateBackend, MixtureExpertSpec, MixtureKind, MixtureSpec, ParticleSpec, RateBackend, +}; +use infotheory::coders::CoderType; use infotheory::compression::{FramingMode, compress_rate_bytes}; -use infotheory::rwkvzip::{self, OnlineConfig, OnlineTrainMode}; -use infotheory::{MixtureExpertSpec, MixtureKind, MixtureSpec, ParticleSpec, RateBackend}; use std::sync::{Arc, OnceLock}; use std::time::Duration; @@ -11,6 +13,8 @@ const DATA_LEN: usize = 10 * 1024; const CTW_DEPTH: usize = 6; const MIX_ALPHA: f64 = 0.03; const MIX_DECAY: f64 = 0.995; +const RWKV_BENCH_METHOD: &str = + "cfg:hidden=64,intermediate=64,layers=1,train=none,lr=0.0;policy:schedule=0..100:infer"; fn bench_data() -> &'static [u8] { static DATA: OnceLock> = OnceLock::new(); @@ -29,96 +33,80 @@ fn bench_data() -> &'static [u8] { } fn particle_spec_from_example() -> ParticleSpec { - ParticleSpec { - num_particles: 8, - context_window: 64, - unroll_steps: 2, - num_cells: 6, - cell_dim: 12, - num_rules: 2, - selector_hidden: 32, - rule_hidden: 32, - noise_dim: 8, - deterministic: true, - enable_noise: true, - noise_scale: 0.08, - noise_anneal_steps: 8192, - learning_rate_readout: 0.0003, - learning_rate_selector: 0.0, - learning_rate_rule: 0.0, - bptt_depth: 1, - optimizer_momentum: 0.05, - grad_clip: 1.0, - state_clip: 8.0, - forget_lambda: 0.0, - resample_threshold: 0.5, - mutate_fraction: 0.25, - mutate_scale: 0.01, - mutate_model_params: false, - diagnostics_interval: 0, - min_prob: 2f64.powi(-24), - seed: 42, - } + let mut spec = ParticleSpec::default(); + spec.num_particles = 8; + spec.context_window = 64; + spec.unroll_steps = 2; + spec.num_cells = 6; + spec.cell_dim = 12; + spec.num_rules = 2; + spec.selector_hidden = 32; + spec.rule_hidden = 32; + spec.noise_dim = 8; + spec.deterministic = true; + spec.enable_noise = true; + spec.noise_scale = 0.08; + spec.noise_anneal_steps = 8192; + spec.learning_rate_readout = 0.0003; + spec.learning_rate_selector = 0.0; + spec.learning_rate_rule = 0.0; + spec.bptt_depth = 1; + spec.optimizer_momentum = 0.05; + spec.grad_clip = 1.0; + spec.state_clip = 8.0; + spec.forget_lambda = 0.0; + spec.resample_threshold = 0.5; + spec.mutate_fraction = 0.25; + spec.mutate_scale = 0.01; + spec.mutate_model_params = false; + spec.diagnostics_interval = 0; + spec.min_prob = 2f64.powi(-24); + spec.seed = 42; + spec } -fn rwkv_model_64x64() -> Arc { - static MODEL: OnceLock> = OnceLock::new(); - MODEL - .get_or_init(|| { - let cfg = OnlineConfig { - hidden: 64, - intermediate: 64, - layers: 1, - train_mode: OnlineTrainMode::None, - seed: 7, - ..OnlineConfig::default() - }; - let rwkv_cfg = cfg - .to_rwkv_config() - .expect("failed to build RWKV config for 64x64 benchmark model"); - Arc::new( - rwkvzip::Model::new_random(rwkv_cfg, cfg.seed) - .expect("failed to initialize random RWKV benchmark model"), - ) - }) - .clone() +fn compile_rate_backend(backend: RateBackend) -> CompiledRateBackend { + backend.compile().expect("compile benchmark rate backend") } -fn individual_backends() -> Vec<(&'static str, RateBackend)> { +fn individual_backends() -> Vec<(&'static str, CompiledRateBackend)> { vec![ - ("rosaplus-o-1", RateBackend::RosaPlus), - ("ctw-d6", RateBackend::Ctw { depth: CTW_DEPTH }), + ( + "rosaplus-o-1", + compile_rate_backend(RateBackend::RosaPlus { max_order: -1 }), + ), + ( + "ctw-d6", + compile_rate_backend(RateBackend::Ctw { depth: CTW_DEPTH }), + ), ( "rwkv64x64", - RateBackend::Rwkv7 { - model: rwkv_model_64x64(), - }, + compile_rate_backend(RateBackend::Rwkv7Method { + method: infotheory::rwkvzip::parse_method_spec(RWKV_BENCH_METHOD) + .expect("rwkv benchmark method must be valid"), + }), ), ( "particle-like-example", - RateBackend::Particle { + compile_rate_backend(RateBackend::Particle { spec: Arc::new(particle_spec_from_example()), - }, + }), ), ] } fn make_expert(name: &str, backend: RateBackend) -> MixtureExpertSpec { - MixtureExpertSpec { - name: Some(name.to_string()), - log_prior: 0.0, - max_order: -1, - backend, - } + MixtureExpertSpec::new(backend).with_name(name) } -fn mixture_backends() -> Vec<(&'static str, RateBackend)> { - let rosa = make_expert("rosa", RateBackend::RosaPlus); +fn mixture_backends() -> Vec<(&'static str, CompiledRateBackend)> { + let rosa = make_expert("rosa", RateBackend::RosaPlus { max_order: -1 }); let ctw = make_expert("ctw", RateBackend::Ctw { depth: CTW_DEPTH }); let rwkv = make_expert( "rwkv64x64", - RateBackend::Rwkv7 { - model: rwkv_model_64x64(), + RateBackend::Rwkv7Method { + method: infotheory::rwkvzip::parse_method_spec(RWKV_BENCH_METHOD) + .expect("rwkv benchmark method must be valid"), }, ); let particle = make_expert( @@ -133,9 +121,9 @@ fn mixture_backends() -> Vec<(&'static str, RateBackend)> { if matches!(kind, MixtureKind::FadingBayes) { spec = spec.with_decay(MIX_DECAY); } - RateBackend::Mixture { + compile_rate_backend(RateBackend::Mixture { spec: Arc::new(spec), - } + }) }; vec![ @@ -167,19 +155,15 @@ fn bench_matrix(c: &mut Criterion) { let mut group = c.benchmark_group("rate_coders_individual"); group.throughput(Throughput::Bytes(data.len() as u64)); - for (label, coder) in [ - ("ac", rwkvzip::CoderType::AC), - ("rans", rwkvzip::CoderType::RANS), - ] { + for (label, coder) in [("ac", CoderType::AC), ("rans", CoderType::RANS)] { for (backend_name, backend) in individual_backends() { group.bench_with_input( BenchmarkId::new(format!("{label}/{backend_name}"), data.len()), &backend, |b, rate_backend| { b.iter(|| { - let out = - compress_rate_bytes(data, rate_backend, -1, coder, FramingMode::Raw) - .expect("compression benchmark failed"); + let out = compress_rate_bytes(data, rate_backend, coder, FramingMode::Raw) + .expect("compression benchmark failed"); criterion::black_box(out.len()) }); }, @@ -191,19 +175,15 @@ fn bench_matrix(c: &mut Criterion) { let mut mix_group = c.benchmark_group("rate_coders_mixtures"); mix_group.throughput(Throughput::Bytes(data.len() as u64)); - for (label, coder) in [ - ("ac", rwkvzip::CoderType::AC), - ("rans", rwkvzip::CoderType::RANS), - ] { + for (label, coder) in [("ac", CoderType::AC), ("rans", CoderType::RANS)] { for (mix_name, backend) in mixture_backends() { mix_group.bench_with_input( BenchmarkId::new(format!("{label}/{mix_name}"), data.len()), &backend, |b, rate_backend| { b.iter(|| { - let out = - compress_rate_bytes(data, rate_backend, -1, coder, FramingMode::Raw) - .expect("mixture compression benchmark failed"); + let out = compress_rate_bytes(data, rate_backend, coder, FramingMode::Raw) + .expect("mixture compression benchmark failed"); criterion::black_box(out.len()) }); }, diff --git a/benches/rwkv_online_train_full.rs b/crates/infotheory/benches/rwkv_online_train_full.rs similarity index 76% rename from benches/rwkv_online_train_full.rs rename to crates/infotheory/benches/rwkv_online_train_full.rs index 419cf273..c26f70c6 100644 --- a/benches/rwkv_online_train_full.rs +++ b/crates/infotheory/benches/rwkv_online_train_full.rs @@ -1,7 +1,7 @@ #![cfg(feature = "backend-rwkv")] use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; -use infotheory::{RateBackend, entropy_rate_backend}; +use infotheory::api::{CompiledRateBackend, RateBackend, try_entropy_rate_backend}; use std::time::Duration; const DATA_LEN: usize = 64 * 1024; @@ -18,7 +18,8 @@ fn bench_data() -> Vec { fn backend(method: &str) -> RateBackend { RateBackend::Rwkv7Method { - method: method.to_string(), + method: infotheory::rwkvzip::parse_method_spec(method) + .expect("rwkv benchmark method must be valid"), } } @@ -26,10 +27,14 @@ fn bench_rwkv_online_train_full(c: &mut Criterion) { let data = bench_data(); let infer = backend( "cfg:hidden=128,layers=2,intermediate=256,decay_rank=16,a_rank=16,v_rank=16,g_rank=16,seed=7,train=none,lr=0.0,stride=1;policy:schedule=0..100:infer", - ); + ) + .compile() + .expect("compile rwkv infer backend"); let train_full = backend( "cfg:hidden=128,layers=2,intermediate=256,decay_rank=16,a_rank=16,v_rank=16,g_rank=16,seed=7,train=adam,lr=0.001,stride=1;policy:schedule=0..100:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=1,clip=0,momentum=0.9)", - ); + ) + .compile() + .expect("compile rwkv train backend"); let mut group = c.benchmark_group("rwkv_online_train_full"); group.throughput(Throughput::Bytes(data.len() as u64)); @@ -39,7 +44,7 @@ fn bench_rwkv_online_train_full(c: &mut Criterion) { &infer, |b, backend| { b.iter(|| { - let h = entropy_rate_backend(&data, -1, backend); + let h = entropy_rate_backend(&data, backend); criterion::black_box(h) }); }, @@ -50,7 +55,7 @@ fn bench_rwkv_online_train_full(c: &mut Criterion) { &train_full, |b, backend| { b.iter(|| { - let h = entropy_rate_backend(&data, -1, backend); + let h = entropy_rate_backend(&data, backend); criterion::black_box(h) }); }, @@ -68,3 +73,6 @@ criterion_group! { targets = bench_rwkv_online_train_full } criterion_main!(rwkv_online_train_full); +fn entropy_rate_backend(data: &[u8], backend: &CompiledRateBackend) -> f64 { + try_entropy_rate_backend(data, backend).expect("entropy rate") +} diff --git a/benches/simd_hotspots.rs b/crates/infotheory/benches/simd_hotspots.rs similarity index 80% rename from benches/simd_hotspots.rs rename to crates/infotheory/benches/simd_hotspots.rs index 1d05ead1..82a43d27 100644 --- a/benches/simd_hotspots.rs +++ b/crates/infotheory/benches/simd_hotspots.rs @@ -1,5 +1,8 @@ +use infotheory::api::{ + CompiledRateBackend, MixtureExpertSpec, MixtureKind, MixtureSpec, RateBackend, + try_entropy_rate_backend, +}; use infotheory::coders::ac::softmax_pdf_floor_inplace; -use infotheory::{MixtureExpertSpec, MixtureKind, MixtureSpec, RateBackend, entropy_rate_backend}; use std::env; use std::hint::black_box; use std::sync::Arc; @@ -31,39 +34,33 @@ fn source_data(expand_factor: usize) -> Vec { fn make_experts() -> Vec { vec![ - MixtureExpertSpec { - name: Some("fac".to_string()), - log_prior: 0.0, - max_order: -1, - backend: RateBackend::FacCtw { - base_depth: 16, - encoding_bits: 8, - num_percept_bits: 8, - }, - }, - MixtureExpertSpec { - name: Some("rosa".to_string()), - log_prior: 0.0, - max_order: -1, - backend: RateBackend::RosaPlus, - }, + MixtureExpertSpec::new(RateBackend::FacCtw { + base_depth: 16, + encoding_bits: 8, + num_percept_bits: 8, + msb_first: None, + }) + .with_name("fac"), + MixtureExpertSpec::new(RateBackend::RosaPlus { max_order: -1 }).with_name("rosa"), ] } fn bench_neural_mixture(data: &[u8], warmup_iters: usize, bench_iters: usize) { let backend = RateBackend::Mixture { spec: Arc::new(MixtureSpec::new(MixtureKind::Neural, make_experts()).with_alpha(ALPHA)), - }; + } + .compile() + .expect("compile neural mixture backend"); for _ in 0..warmup_iters { - let h = entropy_rate_backend(data, -1, &backend); + let h = entropy_rate_backend(data, &backend); black_box(h); } let start = Instant::now(); let mut sink = 0.0; for _ in 0..bench_iters { - sink += entropy_rate_backend(data, -1, &backend); + sink += entropy_rate_backend(data, &backend); } black_box(sink); let elapsed = start.elapsed().as_secs_f64(); @@ -123,3 +120,6 @@ fn main() { bench_neural_mixture(&data, warmup_iters, bench_iters); bench_ac_softmax_floor_256(warmup_iters * 128, bench_iters * 20_000); } +fn entropy_rate_backend(data: &[u8], backend: &CompiledRateBackend) -> f64 { + try_entropy_rate_backend(data, backend).expect("entropy rate") +} diff --git a/crates/infotheory/benches/warmstart.rs b/crates/infotheory/benches/warmstart.rs new file mode 100644 index 00000000..25f7dd02 --- /dev/null +++ b/crates/infotheory/benches/warmstart.rs @@ -0,0 +1,250 @@ +use criterion::{BatchSize, BenchmarkId, Criterion, black_box, criterion_group, criterion_main}; +use infotheory::aixi::warmstart::{WarmStartExactJhAgent, WarmStartExactJhTeacherDataset}; +use infotheory::aixi::warmstart_contract::{ + TaskFingerprint, WARMSTART_STANDALONE_OBSERVATION_ADAPTER_SPEC_REF, + WARMSTART_STANDALONE_SCALAR_REPRESENTATION, WARMSTART_TEACHER_CONTRACT_SCHEMA_VERSION, + standalone_exact_reward_encoding_certificate_hash, + standalone_observation_adapter_content_crc32, warmstart_exact_jh_planner_task_fingerprint, +}; +use infotheory::spec::{CompiledPlannerRunSpec, SpecDocument}; +use serde_json::json; + +#[derive(Clone, Copy)] +struct WarmstartBenchCase { + name: &'static str, + actions: usize, + observation_bits: usize, + reward_bits: usize, + return_horizon: usize, + return_bins: usize, + label_phase_period: usize, + teacher_steps: usize, + live_steps: usize, +} + +fn bench_cases() -> [WarmstartBenchCase; 2] { + [ + WarmstartBenchCase { + name: "small", + actions: 2, + observation_bits: 1, + reward_bits: 2, + return_horizon: 1, + return_bins: 4, + label_phase_period: 1, + teacher_steps: 16, + live_steps: 4, + }, + WarmstartBenchCase { + name: "medium", + actions: 4, + observation_bits: 2, + reward_bits: 3, + return_horizon: 3, + return_bins: 16, + label_phase_period: 3, + teacher_steps: 64, + live_steps: 12, + }, + ] +} + +fn compiled_planner_run(case: WarmstartBenchCase) -> CompiledPlannerRunSpec { + let document = SpecDocument::parse_json_value( + &json!({ + "schema_version": 1, + "kind": "planner_run", + "assets": [{ + "id": "teacher", + "path": "warmstart-bench-teacher.json" + }], + "environment": { + "kind": "builtin", + "name": "coin_flip" + }, + "interface": { + "observation_bits": case.observation_bits, + "observation_stream_len": 1, + "observation_key_mode": "full_stream", + "reward_bits": case.reward_bits, + "agent_actions": case.actions + }, + "controller": { + "kind": "aiqi_warmstart_exact_jh", + "predictor": { + "kind": "ctw", + "depth": 8 + }, + "bit_stream_semantics": { "kind": "binary_tokens" }, + "return_horizon": case.return_horizon, + "return_bins": case.return_bins, + "label_phase_period": case.label_phase_period, + "teacher_dataset_asset": "teacher", + "planner_simulations_per_step": 1 + }, + "runtime": { + "random_seed": 7, + "learn_cycles": 1, + "eval_cycles": 1, + "terminate_lifetime": 2, + "log_every": 1, + "perf": false, + "vm_perf_only": false, + "explore_epsilon": 0.0, + "explore_gamma": 1.0 + } + }), + std::path::Path::new("."), + ) + .expect("benchmark planner document"); + let SpecDocument::PlannerRun(spec) = document else { + panic!("benchmark document must be planner_run"); + }; + spec.compile().expect("benchmark planner run must compile") +} + +fn teacher_dataset( + case: WarmstartBenchCase, + task_fingerprint: TaskFingerprint, +) -> WarmStartExactJhTeacherDataset { + let max_reward = ((case.return_bins - 1) / case.return_horizon) as i64; + let observation_mod = 1u64 << case.observation_bits; + let action_mod = u64::try_from(case.actions).expect("benchmark action count must fit u64"); + let transitions = (0..case.teacher_steps) + .map(|step| { + json!({ + "action": (step as u64) % action_mod, + "observations": [(step as u64) % observation_mod], + "reward": (step as i64) % (max_reward + 1), + }) + }) + .collect::>(); + WarmStartExactJhTeacherDataset::from_json_value(&json!({ + "schema_version": WARMSTART_TEACHER_CONTRACT_SCHEMA_VERSION, + "contract": { + "task_fingerprint": task_fingerprint.to_string(), + "action_alphabet_size": case.actions, + "observation_bits": case.observation_bits, + "observation_stream_len": 1, + "observation_key_mode": "full_stream", + "observation_adapter_spec_ref": WARMSTART_STANDALONE_OBSERVATION_ADAPTER_SPEC_REF, + "observation_adapter_content_crc32": standalone_observation_adapter_content_crc32( + case.observation_bits, + 1, + case.reward_bits, + ) + .expect("benchmark observation adapter digest"), + "reward_bits": case.reward_bits, + "return_horizon": case.return_horizon, + "label_phase_period": case.label_phase_period, + "scalar_representation": WARMSTART_STANDALONE_SCALAR_REPRESENTATION, + "exact_reward_encoding_certificate": standalone_exact_reward_encoding_certificate_hash( + case.reward_bits, + ) + .expect("benchmark reward certificate digest"), + }, + "traces": [{ + "transitions": transitions, + }], + })) + .expect("benchmark teacher dataset") +} + +fn seeded_agent( + case: WarmstartBenchCase, + compiled: &CompiledPlannerRunSpec, + teacher: &WarmStartExactJhTeacherDataset, +) -> WarmStartExactJhAgent { + let mut agent = + WarmStartExactJhAgent::from_compiled_planner_run(compiled, teacher.clone()).expect("agent"); + let observation_mod = 1u64 << case.observation_bits; + let action_mod = u64::try_from(case.actions).expect("benchmark action count must fit u64"); + let max_reward = ((case.return_bins - 1) / case.return_horizon) as i64; + for step in 0..case.live_steps { + agent + .observe_transition( + (step as u64) % action_mod, + &[(step as u64) % observation_mod], + (step as i64) % (max_reward + 1), + ) + .expect("benchmark live transition"); + } + agent +} + +fn bench_warmstart(c: &mut Criterion) { + for case in bench_cases() { + let compiled = compiled_planner_run(case); + let task_fingerprint = + warmstart_exact_jh_planner_task_fingerprint(&compiled).expect("task fingerprint"); + let teacher = teacher_dataset(case, task_fingerprint); + + let mut values_group = c.benchmark_group("warmstart_action_values"); + values_group.bench_with_input( + BenchmarkId::new("estimate_action_values", case.name), + &case, + |b, &case| { + b.iter_batched( + || seeded_agent(case, &compiled, &teacher), + |mut agent| black_box(agent.estimate_action_values().expect("action values")), + BatchSize::SmallInput, + ); + }, + ); + values_group.finish(); + + let mut plan_group = c.benchmark_group("warmstart_action_selection"); + plan_group.bench_with_input( + BenchmarkId::new("get_planned_action", case.name), + &case, + |b, &case| { + b.iter_batched( + || seeded_agent(case, &compiled, &teacher), + |mut agent| black_box(agent.get_planned_action()), + BatchSize::SmallInput, + ); + }, + ); + plan_group.finish(); + + let mut observe_group = c.benchmark_group("warmstart_observe_transition"); + observe_group.bench_with_input( + BenchmarkId::new("observe_transition", case.name), + &case, + |b, &case| { + b.iter_batched( + || seeded_agent(case, &compiled, &teacher), + |mut agent| { + agent + .observe_transition(0, &[0], 0) + .expect("observe transition"); + black_box(agent) + }, + BatchSize::SmallInput, + ); + }, + ); + observe_group.finish(); + + let mut construct_group = c.benchmark_group("warmstart_construction"); + construct_group.bench_with_input( + BenchmarkId::new("from_teacher_dataset", case.name), + &case, + |b, _| { + b.iter(|| { + black_box( + WarmStartExactJhAgent::from_compiled_planner_run( + &compiled, + teacher.clone(), + ) + .expect("agent"), + ) + }); + }, + ); + construct_group.finish(); + } +} + +criterion_group!(warmstart, bench_warmstart); +criterion_main!(warmstart); diff --git a/crates/infotheory/build.rs b/crates/infotheory/build.rs new file mode 100644 index 00000000..551ecfcc --- /dev/null +++ b/crates/infotheory/build.rs @@ -0,0 +1,27 @@ +fn env_flag_enabled(name: &str) -> bool { + match std::env::var(name) { + Ok(value) => { + let trimmed: &str = value.trim(); + trimmed == "1" + || trimmed.eq_ignore_ascii_case("true") + || trimmed.eq_ignore_ascii_case("yes") + || trimmed.eq_ignore_ascii_case("on") + } + Err(_) => false, + } +} + +fn main() { + println!("cargo:rerun-if-env-changed=INFOTHEORY_AC_ENCODE_DEINLINE"); + println!("cargo:rerun-if-env-changed=INFOTHEORY_AC_DECODE_INLINE"); + + println!("cargo:rustc-check-cfg=cfg(infotheory_ac_encode_deinline)"); + println!("cargo:rustc-check-cfg=cfg(infotheory_ac_decode_inline)"); + + if env_flag_enabled("INFOTHEORY_AC_ENCODE_DEINLINE") { + println!("cargo:rustc-cfg=infotheory_ac_encode_deinline"); + } + if env_flag_enabled("INFOTHEORY_AC_DECODE_INLINE") { + println!("cargo:rustc-cfg=infotheory_ac_decode_inline"); + } +} diff --git a/crates/infotheory/src/aixi/agent.rs b/crates/infotheory/src/aixi/agent.rs new file mode 100644 index 00000000..d4db9ef0 --- /dev/null +++ b/crates/infotheory/src/aixi/agent.rs @@ -0,0 +1,1074 @@ +//! The core AIXI agent implementation. +//! +//! This module defines the `Agent` struct, which ties together a world model +//! (Predictor) and an explicit MCTS planner state to form a complete autonomous +//! entity. + +use crate::aixi::common::{ + Action, ActionAlphabet, MctsStrategy, ObservationKeyMode, PerceptVal, RandomGenerator, Reward, + RewardEncodingError, byte_packed_percept_bits, decode, encode, + nonnegative_reward_encoding_bounds, observation_repr_from_stream, resolve_random_seed, + validate_mc_aixi_byte_packed_alignment, validate_reward_encoding_bounds, + warn_parallel_uct_workers_one_once, +}; +use crate::aixi::mcts::{ + AgentSimulator, ParallelUctPlanner, ParallelUctPlannerInitError, RhoUctPlanner, +}; +use crate::aixi::model::{ + Predictor, PredictorBuildError, build_mc_aixi_predictor, default_aixi_bit_stream_semantics, +}; +use crate::aixi::planner_spec::{PlannerInterfaceConfig, build_default_planner_run_spec}; +use crate::api::{BitStreamSemantics, RateBackend, validate_rate_backend}; +use crate::spec::{ + CompiledPlannerController, CompiledPlannerRunSpec, ControllerSpec, McAixiControllerSpec, + PlannerRunSpec, SpecError, +}; +use std::error::Error; +use std::fmt; + +/// Error returned by MC-AIXI configuration validation and construction. +#[derive(Debug)] +#[non_exhaustive] +pub enum AgentError { + /// `agent_horizon` was zero. + AgentHorizonZero, + /// `num_simulations` was zero. + NumSimulationsZero, + /// The UCT exploration/exploitation constant was non-positive. + InvalidExplorationExploitationRatio { + /// The invalid exploration/exploitation ratio value. + value: f64, + }, + /// The configured MC-AIXI discount factor was outside `[0, 1]`. + InvalidDiscountGamma { + /// The invalid discount factor value. + value: f64, + }, + /// The configured reward range is not representable. + RewardEncoding(RewardEncodingError), + /// The configured rate backend failed validation. + InvalidRateBackend(crate::error::InfotheoryError), + /// The configured rate backend violates MC-AIXI runtime requirements. + UnsupportedRateBackend { + /// Human-readable explanation of why the backend is unsupported. + reason: &'static str, + }, + /// Planner-run spec compilation failed. + Spec(SpecError), + /// The compiled planner-run controller kind was not MC-AIXI. + ControllerKindMismatch, + /// Predictor construction failed. + Predictor(PredictorBuildError), + /// Parallel UCT planner construction failed (e.g. invalid `bu_uct_m_max`). + ParallelUctPlannerInit(ParallelUctPlannerInitError), +} + +impl fmt::Display for AgentError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::AgentHorizonZero => f.write_str("agent_horizon must be >= 1"), + Self::NumSimulationsZero => f.write_str("num_simulations must be >= 1"), + Self::InvalidExplorationExploitationRatio { value: _ } => { + f.write_str("exploration_exploitation_ratio must be > 0") + } + Self::InvalidDiscountGamma { value } => { + write!( + f, + "discount_gamma must be in [0, 1] for MC-AIXI, got {value}" + ) + } + Self::RewardEncoding(err) => write!(f, "{err}"), + Self::InvalidRateBackend(err) => write!(f, "invalid rate_backend: {err}"), + Self::UnsupportedRateBackend { reason } => f.write_str(reason), + Self::Spec(err) => write!(f, "{err}"), + Self::ControllerKindMismatch => { + f.write_str("compiled planner run does not contain an MC-AIXI controller") + } + Self::Predictor(err) => write!(f, "{err}"), + Self::ParallelUctPlannerInit(err) => { + write!(f, "parallel_uct planner construction failed: {err}") + } + } + } +} + +impl Error for AgentError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::RewardEncoding(err) => Some(err), + Self::InvalidRateBackend(err) => Some(err), + Self::Spec(err) => Some(err), + Self::Predictor(err) => Some(err), + Self::ParallelUctPlannerInit(err) => Some(err), + _ => None, + } + } +} + +impl From for AgentError { + fn from(value: RewardEncodingError) -> Self { + Self::RewardEncoding(value) + } +} + +impl From for AgentError { + fn from(value: SpecError) -> Self { + Self::Spec(value) + } +} + +impl From for AgentError { + fn from(value: ParallelUctPlannerInitError) -> Self { + Self::ParallelUctPlannerInit(value) + } +} + +/// Configuration parameters for an AIXI agent. +#[derive(Clone)] +#[non_exhaustive] +pub struct AgentConfig { + /// Predictive backend used by MC-AIXI. + pub rate_backend: RateBackend, + /// Bit-stream semantics used to adapt generic rate backends to AIXI symbols. + pub bit_stream_semantics: BitStreamSemantics, + /// Planning horizon for MCTS. + pub agent_horizon: usize, + /// Number of bits used to encode observations. + pub observation_bits: usize, + /// Number of observation symbols per action (stream length). + pub observation_stream_len: usize, + /// Strategy for mapping observation streams into search keys. + pub observation_key_mode: ObservationKeyMode, + /// Number of bits used to encode rewards. + pub reward_bits: usize, + /// Cardinality of the action alphabet. + pub agent_actions: ActionAlphabet, + /// Number of MCTS simulations per planning step. + pub num_simulations: usize, + /// Explicit MCTS strategy. + pub mcts_strategy: MctsStrategy, + /// Constant governing exploration vs exploitation in UCT. + pub exploration_exploitation_ratio: f64, + /// Discount factor for future rewards (1.0 = undiscounted). + pub discount_gamma: f64, + /// Minimum possible instantaneous reward in the environment. + pub min_reward: Reward, + /// Maximum possible instantaneous reward in the environment. + pub max_reward: Reward, + /// Reward offset applied before encoding rewards as unsigned bits. + /// + /// Paper-compatible encoding shifts rewards by an offset so all encoded values are non-negative. + pub reward_offset: Reward, + /// Optional deterministic RNG seed for planning/simulation behavior. + /// + /// When `None`, planner runtime canonicalizes this to seed `0`. + pub random_seed: Option, +} + +impl Default for AgentConfig { + fn default() -> Self { + Self { + rate_backend: RateBackend::Ctw { depth: 8 }, + bit_stream_semantics: default_aixi_bit_stream_semantics(), + agent_horizon: 5, + observation_bits: 1, + observation_stream_len: 1, + observation_key_mode: ObservationKeyMode::FullStream, + reward_bits: 1, + agent_actions: ActionAlphabet::try_from_usize(2) + .expect("default action alphabet must be non-zero"), + num_simulations: 100, + mcts_strategy: MctsStrategy::RhoUct, + exploration_exploitation_ratio: 1.0, + discount_gamma: 1.0, + min_reward: 0, + max_reward: 1, + reward_offset: 0, + random_seed: None, + } + } +} + +impl AgentConfig { + fn canonical_predictor_backend(&self) -> RateBackend { + self.rate_backend.clone() + } + + fn canonical_planner_run_spec(&self) -> PlannerRunSpec { + let predictor = self.canonical_predictor_backend(); + build_default_planner_run_spec( + PlannerInterfaceConfig { + observation_bits: self.observation_bits, + observation_stream_len: self.observation_stream_len, + observation_key_mode: self.observation_key_mode, + reward_bits: self.reward_bits, + agent_actions: self.agent_actions, + }, + ControllerSpec::McAixi(McAixiControllerSpec { + predictor, + bit_stream_semantics: self.bit_stream_semantics, + agent_horizon: self.agent_horizon, + num_simulations: self.num_simulations, + mcts_strategy: self.mcts_strategy, + exploration_exploitation_ratio: self.exploration_exploitation_ratio, + discount_gamma: self.discount_gamma, + }), + self.random_seed, + ) + } + + fn compile_planner_run_spec(&self) -> Result { + self.canonical_planner_run_spec() + .compile() + .map_err(AgentError::from) + } + + fn validate_runtime_invariants(&self) -> Result<(), AgentError> { + if self.agent_horizon == 0 { + return Err(AgentError::AgentHorizonZero); + } + if self.num_simulations == 0 { + return Err(AgentError::NumSimulationsZero); + } + match self.mcts_strategy { + MctsStrategy::RhoUct => {} + MctsStrategy::ParallelUct { + workers, + bu_uct_m_max, + } => { + // `workers` is `NonZeroUsize`, so the `>= 1` invariant is + // type-enforced and no runtime check is needed here. + if workers.get() == 1 { + warn_parallel_uct_workers_one_once(); + } + if bu_uct_m_max.is_some_and(|m_max| !(0.0 < m_max && m_max < 1.0)) { + return Err(AgentError::Spec(SpecError::new( + "controller.mcts_strategy.bu_uct_m_max must be in (0, 1)", + ))); + } + } + } + if self.exploration_exploitation_ratio <= 0.0 { + return Err(AgentError::InvalidExplorationExploitationRatio { + value: self.exploration_exploitation_ratio, + }); + } + if !(0.0..=1.0).contains(&self.discount_gamma) { + return Err(AgentError::InvalidDiscountGamma { + value: self.discount_gamma, + }); + } + validate_reward_encoding_bounds( + self.min_reward, + self.max_reward, + self.reward_offset, + self.reward_bits, + )?; + if matches!( + self.bit_stream_semantics, + BitStreamSemantics::BytePacked { .. } + ) { + let action_bits = self.agent_actions.action_bits(); + let percept_bits = byte_packed_percept_bits( + self.observation_bits, + self.observation_stream_len, + self.reward_bits, + ); + validate_mc_aixi_byte_packed_alignment(action_bits, percept_bits) + .map_err(|reason| AgentError::UnsupportedRateBackend { reason })?; + } + + validate_rate_backend(&self.rate_backend).map_err(AgentError::InvalidRateBackend)?; + let compiled = self.rate_backend.compile().map_err(AgentError::from)?; + if compiled.contains_zpaq() { + return Err(AgentError::UnsupportedRateBackend { + reason: "MC-AIXI strict generic rate_backend support requires reversible action conditioning; configured rate_backend contains zpaq which does not provide the reversible action conditioning required by \"A Monte-Carlo AIXI Approximation\"", + }); + } + + Ok(()) + } + + /// Validate configuration constraints for MC-AIXI. + pub fn validate(&self) -> Result<(), AgentError> { + self.validate_runtime_invariants()?; + self.compile_planner_run_spec().map(|_| ()) + } +} + +#[derive(Clone)] +struct AgentRuntimeConfig { + agent_horizon: usize, + observation_bits: usize, + observation_stream_len: usize, + observation_key_mode: ObservationKeyMode, + reward_bits: usize, + agent_actions: ActionAlphabet, + num_simulations: usize, + mcts_strategy: MctsStrategy, + exploration_exploitation_ratio: f64, + discount_gamma: f64, + min_reward: Reward, + max_reward: Reward, + reward_offset: Reward, + random_seed: u64, + bit_stream_semantics: BitStreamSemantics, +} + +impl AgentRuntimeConfig { + fn from_config(config: &AgentConfig) -> Self { + Self { + agent_horizon: config.agent_horizon, + observation_bits: config.observation_bits, + observation_stream_len: config.observation_stream_len.max(1), + observation_key_mode: config.observation_key_mode, + reward_bits: config.reward_bits, + agent_actions: config.agent_actions, + num_simulations: config.num_simulations, + mcts_strategy: config.mcts_strategy, + exploration_exploitation_ratio: config.exploration_exploitation_ratio, + discount_gamma: config.discount_gamma, + min_reward: config.min_reward, + max_reward: config.max_reward, + reward_offset: config.reward_offset, + random_seed: resolve_random_seed(config.random_seed), + bit_stream_semantics: config.bit_stream_semantics, + } + } + + fn from_compiled(compiled: &CompiledPlannerRunSpec) -> Result { + let interface = compiled.interface(); + let runtime = compiled.runtime(); + let ( + agent_horizon, + num_simulations, + mcts_strategy, + exploration_exploitation_ratio, + discount_gamma, + bit_stream_semantics, + ) = match compiled.controller() { + CompiledPlannerController::McAixi { + agent_horizon, + num_simulations, + mcts_strategy, + exploration_exploitation_ratio, + discount_gamma, + bit_stream_semantics, + .. + } => ( + *agent_horizon, + *num_simulations, + *mcts_strategy, + *exploration_exploitation_ratio, + *discount_gamma, + *bit_stream_semantics, + ), + _ => return Err(AgentError::ControllerKindMismatch), + }; + + let (min_reward, max_reward, reward_offset) = + nonnegative_reward_encoding_bounds(interface.reward_bits); + Ok(Self { + agent_horizon, + observation_bits: interface.observation_bits, + observation_stream_len: interface.observation_stream_len.max(1), + observation_key_mode: interface.observation_key_mode, + reward_bits: interface.reward_bits, + agent_actions: interface.agent_actions, + num_simulations, + mcts_strategy, + exploration_exploitation_ratio, + discount_gamma, + min_reward, + max_reward, + reward_offset, + random_seed: resolve_random_seed(runtime.random_seed), + bit_stream_semantics, + }) + } +} + +enum PlannerState { + RhoUct(RhoUctPlanner), + ParallelUct(ParallelUctPlanner), +} + +impl PlannerState { + /// Construct the planner backend selected by `strategy`. + /// + /// `workers == 0` is type-prevented by [`MctsStrategy::ParallelUct`]. The + /// only remaining failure mode is an out-of-range `bu_uct_m_max`, which + /// is surfaced via [`AgentError::ParallelUctPlannerInit`]. Callers must + /// chain this through `?` (typically from `Agent::from_compiled_config`) + /// rather than panicking at construction time. + fn new(strategy: MctsStrategy) -> Result { + match strategy { + MctsStrategy::RhoUct => Ok(Self::RhoUct(RhoUctPlanner::new())), + MctsStrategy::ParallelUct { + workers, + bu_uct_m_max, + } => Ok(Self::ParallelUct(ParallelUctPlanner::new( + workers, + bu_uct_m_max, + )?)), + } + } + + fn search( + &mut self, + agent: &mut dyn AgentSimulator, + prev_obs_stream: &[PerceptVal], + prev_rew: Reward, + prev_act: Action, + samples: usize, + ) -> Action { + match self { + Self::RhoUct(planner) => { + planner.search(agent, prev_obs_stream, prev_rew, prev_act, samples) + } + Self::ParallelUct(planner) => { + planner.search_validated(agent, prev_obs_stream, prev_rew, prev_act, samples) + } + } + } +} + +/// A complete MC-AIXI agent. +/// +/// The agent maintains an internal world model and a planning tree. It can +/// be used for both live interaction with an environment and for +/// "imaginary" simulations during planning. +pub struct Agent { + /// The world model used for prediction. + model: Box, + /// The MCTS planner, temporarily taken during search. + planner: Option, + /// Configuration settings. + config: AgentRuntimeConfig, + + /// Total number of interaction cycles. + age: u64, + /// Accumulated reward. + total_reward: f64, + + /// Pre-calculated bit depth for actions based on `agent_actions`. + action_bits: usize, + + /// Internal PRNG for simulations. + rng: RandomGenerator, + + /// Recycled buffer for observation generation during planning. + obs_buffer: Vec, + /// Recycled buffer for symbol processing. + sym_buffer: Vec, +} + +impl Agent { + /// Creates a new `Agent` with the given configuration. + pub fn new(config: AgentConfig) -> Self { + Self::try_new(config).unwrap_or_else(|err| panic!("Invalid MC-AIXI config: {err}")) + } + + /// Creates a new `Agent` with the given configuration, returning a validation error on failure. + pub fn try_new(config: AgentConfig) -> Result { + config.validate_runtime_invariants()?; + let compiled = config.compile_planner_run_spec()?; + let runtime = AgentRuntimeConfig::from_config(&config); + Self::from_compiled_config(runtime, &compiled) + } + + /// Creates a new `Agent` from a compiled planner-run spec. + pub fn from_compiled_planner_run( + compiled: &CompiledPlannerRunSpec, + ) -> Result { + let config = AgentRuntimeConfig::from_compiled(compiled)?; + Self::from_compiled_config(config, compiled) + } + + fn from_compiled_config( + config: AgentRuntimeConfig, + compiled: &CompiledPlannerRunSpec, + ) -> Result { + let predictor = match compiled.controller() { + CompiledPlannerController::McAixi { predictor, .. } => predictor, + _ => return Err(AgentError::ControllerKindMismatch), + }; + let percept_bits = (compiled.interface().observation_bits + * compiled.interface().observation_stream_len.max(1)) + + compiled.interface().reward_bits; + let model = build_mc_aixi_predictor(predictor, percept_bits, config.bit_stream_semantics) + .map_err(AgentError::Predictor)?; + + let rng = RandomGenerator::from_seed(config.random_seed); + + let planner = PlannerState::new(config.mcts_strategy)?; + Ok(Self { + model, + planner: Some(planner), + config, + age: 0, + total_reward: 0.0, + action_bits: compiled.action_bits(), + rng, + obs_buffer: Vec::with_capacity(128), + sym_buffer: Vec::with_capacity(64), + }) + } + + fn clone_for_simulation(&self, seed: u64) -> Self { + Self { + model: self.model.boxed_clone(), + planner: None, + config: self.config.clone(), + age: self.age, + total_reward: self.total_reward, + action_bits: self.action_bits, + rng: self.rng.fork_with(seed), + obs_buffer: Vec::with_capacity(128), + sym_buffer: Vec::with_capacity(64), + } + } + + /// Resets the agent's interaction statistics. + pub fn reset(&mut self) { + self.age = 0; + self.total_reward = 0.0; + } + + /// Returns the resolved deterministic seed used by this agent. + pub fn resolved_random_seed(&self) -> u64 { + self.config.random_seed + } + + pub(crate) fn reseed_random(&mut self, seed: u64) { + self.config.random_seed = seed; + self.rng = RandomGenerator::from_seed(seed); + } + + pub(crate) fn reset_planner_state(&mut self) { + self.planner = Some( + PlannerState::new(self.config.mcts_strategy) + .expect("validated MC-AIXI strategy must rebuild planner state"), + ); + } + + /// Primary interface for decision making. + /// + /// Uses MCTS to find the action that maximizes expected future reward. + pub fn get_planned_action( + &mut self, + prev_obs_stream: &[PerceptVal], + prev_rew: Reward, + prev_act: Action, + ) -> Action { + let mut planner = self.planner.take().expect("Planner missing"); + let num_sim = self.config.num_simulations; + let action = planner.search(self, prev_obs_stream, prev_rew, prev_act, num_sim); + self.planner = Some(planner); + action + } + + /// Updates the world model with real-world percepts. + pub fn model_update_percept(&mut self, observation: PerceptVal, reward: Reward) { + self.model_update_percept_stream(&[observation], reward); + } + + /// Updates the world model with an observation stream and a terminal reward. + pub fn model_update_percept_stream(&mut self, observations: &[PerceptVal], reward: Reward) { + debug_assert!( + !observations.is_empty() || self.config.observation_bits == 0, + "percept update missing observation stream" + ); + let mut percept_syms = Vec::new(); + for &obs in observations { + encode(&mut percept_syms, obs, self.config.observation_bits); + } + crate::aixi::common::encode_reward_offset( + &mut percept_syms, + reward, + self.config.reward_bits, + self.config.reward_offset, + ); + + for &sym in &percept_syms { + self.model.commit_update(sym); + } + + self.total_reward += reward as f64; + } + + /// Computes the observation key used for search-tree branching. + pub fn observation_repr_from_stream(&self, observations: &[PerceptVal]) -> Vec { + observation_repr_from_stream( + self.config.observation_key_mode, + observations, + self.config.observation_bits, + ) + } + + /// Explicitly updates the world model with an action. + pub fn model_update_action_external(&mut self, action: Action) { + self.sym_buffer.clear(); + encode(&mut self.sym_buffer, action, self.action_bits); + + for &sym in &self.sym_buffer { + self.model.commit_update_history(sym); + } + } +} + +impl AgentSimulator for Agent { + fn get_num_actions(&self) -> ActionAlphabet { + self.config.agent_actions + } + + fn get_num_observation_bits(&self) -> usize { + self.config.observation_bits + } + + fn observation_stream_len(&self) -> usize { + self.config.observation_stream_len.max(1) + } + + fn observation_key_mode(&self) -> ObservationKeyMode { + self.config.observation_key_mode + } + + fn get_num_reward_bits(&self) -> usize { + self.config.reward_bits + } + + fn horizon(&self) -> usize { + self.config.agent_horizon + } + + fn max_reward(&self) -> Reward { + self.config.max_reward + } + + fn min_reward(&self) -> Reward { + self.config.min_reward + } + + fn reward_offset(&self) -> i64 { + self.config.reward_offset + } + + fn get_explore_exploit_ratio(&self) -> f64 { + self.config.exploration_exploitation_ratio + } + + fn discount_gamma(&self) -> f64 { + self.config.discount_gamma + } + + fn model_update_action(&mut self, action: Action) { + self.sym_buffer.clear(); + encode(&mut self.sym_buffer, action, self.action_bits); + + for &sym in &self.sym_buffer { + self.model.update_history(sym); + } + } + + fn gen_percept_and_update(&mut self, bits: usize) -> u64 { + self.sym_buffer.clear(); + for _ in 0..bits { + let prob_1 = self.model.predict_one(); + let sym = self.rng.gen_bool(prob_1); + self.model.update(sym); + self.sym_buffer.push(sym); + } + decode(&self.sym_buffer, bits) + } + + fn begin_simulation(&mut self) { + self.model.begin_rollback_scope(); + } + + fn begin_discardable_simulation(&mut self) { + self.model.begin_discardable_scope(); + } + + fn gen_percepts_and_update(&mut self) -> (Vec, Reward) { + let obs_bits = self.config.observation_bits; + let obs_len = self.config.observation_stream_len.max(1); + + self.obs_buffer.clear(); + for _ in 0..obs_len { + let p = self.gen_percept_and_update(obs_bits); + self.obs_buffer.push(p); + } + + let obs_repr = observation_repr_from_stream( + self.config.observation_key_mode, + &self.obs_buffer, + obs_bits, + ); + let rew_bits = self.config.reward_bits; + let rew_u = self.gen_percept_and_update(rew_bits); + let rew = (rew_u as i64) - self.config.reward_offset; + + // Mark that we've completed a percept cycle (ready for next action) + + (obs_repr, rew) + } + + fn gen_range(&mut self, end: usize) -> usize { + self.rng.gen_range(end) + } + + fn gen_f64(&mut self) -> f64 { + self.rng.gen_f64() + } + + fn model_revert(&mut self, steps: usize) { + if self.model.rollback_scope() { + return; + } + let obs_bits = self.config.observation_bits * self.config.observation_stream_len.max(1); + let percept_bits = obs_bits + self.config.reward_bits; + + for _ in 0..steps { + for _ in 0..percept_bits { + self.model.revert(); + } + for _ in 0..self.action_bits { + self.model.pop_history(); + } + } + } + + fn boxed_clone_with_seed(&self, seed: u64) -> Box { + Box::new(self.clone_for_simulation(seed)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[cfg(feature = "all-backends")] + use crate::aixi::environment::Environment; + #[cfg(feature = "all-backends")] + use crate::aixi::test_envs::DeterministicBinaryEnv; + #[cfg(feature = "all-backends")] + use crate::api::{MixtureExpertSpec, MixtureKind, MixtureSpec, RateBackend}; + use std::sync::{Arc, Mutex}; + + #[derive(Clone, Default)] + struct CallCounts { + update: usize, + commit_update: usize, + update_history: usize, + commit_update_history: usize, + begin_scope: usize, + begin_discardable_scope: usize, + rollback_scope: usize, + revert: usize, + pop_history: usize, + } + + #[derive(Clone)] + struct InstrumentedPredictor { + counts: Arc>, + } + + impl InstrumentedPredictor { + fn new(counts: Arc>) -> Self { + Self { counts } + } + } + + impl Predictor for InstrumentedPredictor { + fn update(&mut self, _sym: bool) { + self.counts.lock().unwrap().update += 1; + } + + fn commit_update(&mut self, _sym: bool) { + self.counts.lock().unwrap().commit_update += 1; + } + + fn update_history(&mut self, _sym: bool) { + self.counts.lock().unwrap().update_history += 1; + } + + fn commit_update_history(&mut self, _sym: bool) { + self.counts.lock().unwrap().commit_update_history += 1; + } + + fn revert(&mut self) { + self.counts.lock().unwrap().revert += 1; + } + + fn pop_history(&mut self) { + self.counts.lock().unwrap().pop_history += 1; + } + + fn begin_rollback_scope(&mut self) { + self.counts.lock().unwrap().begin_scope += 1; + } + + fn begin_discardable_scope(&mut self) { + self.counts.lock().unwrap().begin_discardable_scope += 1; + } + + fn rollback_scope(&mut self) -> bool { + self.counts.lock().unwrap().rollback_scope += 1; + true + } + + fn predict_prob(&mut self, sym: bool) -> f64 { + if sym { 0.75 } else { 0.25 } + } + + fn model_name(&self) -> String { + "InstrumentedPredictor".to_string() + } + + fn boxed_clone(&self) -> Box { + Box::new(self.clone()) + } + } + + fn basic_runtime_config() -> AgentRuntimeConfig { + AgentRuntimeConfig { + agent_horizon: 2, + observation_bits: 2, + observation_stream_len: 2, + observation_key_mode: ObservationKeyMode::FullStream, + reward_bits: 3, + agent_actions: ActionAlphabet::try_from_usize(4) + .expect("test fixture action alphabet must be non-zero"), + num_simulations: 2, + mcts_strategy: MctsStrategy::RhoUct, + exploration_exploitation_ratio: 1.0, + discount_gamma: 0.95, + min_reward: -2, + max_reward: 3, + reward_offset: 2, + random_seed: 7, + bit_stream_semantics: crate::api::BitStreamSemantics::BinaryTokens, + } + } + + fn test_agent(model: Box) -> Agent { + let config = basic_runtime_config(); + let action_bits = config.agent_actions.action_bits(); + Agent { + action_bits, + model, + planner: Some( + PlannerState::new(config.mcts_strategy) + .expect("test fixture mcts_strategy must be valid"), + ), + config, + age: 0, + total_reward: 0.0, + rng: RandomGenerator::from_seed(7), + obs_buffer: Vec::with_capacity(128), + sym_buffer: Vec::with_capacity(64), + } + } + + #[cfg(feature = "all-backends")] + fn generic_mixture_config() -> AgentConfig { + AgentConfig { + rate_backend: RateBackend::Mixture { + spec: Arc::new( + MixtureSpec::new( + MixtureKind::Convex, + vec![ + MixtureExpertSpec { + name: Some("ctw".to_string()), + log_prior: 0.0, + backend: RateBackend::Ctw { depth: 8 }, + }, + MixtureExpertSpec { + name: Some("rosa".to_string()), + log_prior: 0.0, + backend: RateBackend::RosaPlus { max_order: 8 }, + }, + ], + ) + .with_alpha(1.25), + ), + }, + bit_stream_semantics: crate::api::BitStreamSemantics::BinaryTokens, + agent_horizon: 5, + observation_bits: 1, + observation_stream_len: 1, + observation_key_mode: ObservationKeyMode::FullStream, + reward_bits: 1, + agent_actions: ActionAlphabet::try_from_usize(2) + .expect("test fixture action alphabet must be non-zero"), + num_simulations: 60, + mcts_strategy: MctsStrategy::RhoUct, + exploration_exploitation_ratio: 1.4, + discount_gamma: 1.0, + min_reward: 0, + max_reward: 1, + reward_offset: 0, + random_seed: Some(2026), + } + } + + #[cfg(feature = "all-backends")] + fn run_ctw_trace(agent: &mut Agent, cycles: usize) -> (Vec, i64) { + let mut env = DeterministicBinaryEnv::default(); + let mut actions = Vec::with_capacity(cycles); + let mut total_reward = 0i64; + let mut obs_stream = env.drain_observations(); + let mut prev_rew = env.get_reward(); + let mut prev_act = 0; + + for _ in 0..cycles { + agent.model_update_percept_stream(&obs_stream, prev_rew); + let action = agent.get_planned_action(&obs_stream, prev_rew, prev_act); + actions.push(action); + agent.model_update_action_external(action); + + env.perform_action(action); + obs_stream = env.drain_observations(); + let rew = env.get_reward(); + agent.model_update_percept_stream(&obs_stream, rew); + total_reward += rew; + prev_rew = rew; + prev_act = action; + } + + (actions, total_reward) + } + + #[test] + fn external_history_updates_use_committed_predictor_paths() { + let counts = Arc::new(Mutex::new(CallCounts::default())); + let mut agent = test_agent(Box::new(InstrumentedPredictor::new(counts.clone()))); + + agent.model_update_percept_stream(&[1, 2], 1); + agent.model_update_action_external(3); + + let snapshot = counts.lock().unwrap().clone(); + assert_eq!(snapshot.commit_update, 7); + assert_eq!(snapshot.commit_update_history, 2); + assert_eq!(snapshot.update, 0); + assert_eq!(snapshot.update_history, 0); + } + + #[test] + fn simulation_revert_prefers_predictor_scope_when_available() { + let counts = Arc::new(Mutex::new(CallCounts::default())); + let mut agent = test_agent(Box::new(InstrumentedPredictor::new(counts.clone()))); + + AgentSimulator::begin_simulation(&mut agent); + agent.model_revert(3); + + let snapshot = counts.lock().unwrap().clone(); + assert_eq!(snapshot.begin_scope, 1); + assert_eq!(snapshot.rollback_scope, 1); + assert_eq!(snapshot.revert, 0); + assert_eq!(snapshot.pop_history, 0); + } + + #[test] + fn discardable_simulation_uses_predictor_discardable_scope() { + let counts = Arc::new(Mutex::new(CallCounts::default())); + let mut agent = test_agent(Box::new(InstrumentedPredictor::new(counts.clone()))); + + AgentSimulator::begin_discardable_simulation(&mut agent); + + let snapshot = counts.lock().unwrap().clone(); + assert_eq!(snapshot.begin_discardable_scope, 1); + assert_eq!(snapshot.begin_scope, 0); + assert_eq!(snapshot.rollback_scope, 0); + } + + /// Stable shape-only descriptor for a `RateBackend` variant, used for + /// alias-equivalence assertions without requiring `Debug` on the enum. + fn backend_shape(backend: &crate::api::RateBackend) -> &'static str { + use crate::api::RateBackend; + match backend { + RateBackend::Ctw { .. } => "ctw", + RateBackend::FacCtw { .. } => "fac-ctw", + RateBackend::RosaPlus { .. } => "rosaplus", + _ => "other", + } + } + + #[test] + fn explicit_rate_backend_semantics_are_symmetric_between_mc_aixi_and_aiqi() { + let mut agent_cfg = AgentConfig { + rate_backend: crate::api::RateBackend::Ctw { depth: 8 }, + ..AgentConfig::default() + }; + let agent_ctw = agent_cfg.canonical_predictor_backend(); + + let mut aiqi_cfg = crate::aixi::aiqi::AiqiConfig { + rate_backend: crate::api::RateBackend::Ctw { depth: 8 }, + ..crate::aixi::aiqi::AiqiConfig::default() + }; + let aiqi_ctw = aiqi_cfg.canonical_predictor_backend_for_test(); + + assert_eq!( + backend_shape(&agent_ctw), + "ctw", + "MC-AIXI: 'ctw' must produce a single-tree CTW backend" + ); + assert_eq!( + backend_shape(&aiqi_ctw), + "ctw", + "AIQI: 'ctw' must produce a single-tree CTW backend" + ); + + agent_cfg.rate_backend = crate::api::RateBackend::RosaPlus { max_order: -1 }; + aiqi_cfg.rate_backend = crate::api::RateBackend::RosaPlus { max_order: -1 }; + let agent_rosa = agent_cfg.canonical_predictor_backend(); + let aiqi_rosa = aiqi_cfg.canonical_predictor_backend_for_test(); + assert_eq!( + backend_shape(&agent_rosa), + "rosaplus", + "MC-AIXI: 'rosa' must produce ROSA+" + ); + assert_eq!( + backend_shape(&aiqi_rosa), + "rosaplus", + "AIQI: 'rosa' must produce ROSA+" + ); + } + + #[cfg(feature = "backend-ctw")] + #[test] + fn programmatic_mcaixi_preserves_explicit_signed_reward_contract() { + let config = AgentConfig { + reward_bits: 3, + min_reward: -2, + max_reward: 3, + reward_offset: 2, + num_simulations: 1, + ..AgentConfig::default() + }; + + let mut agent = Agent::try_new(config).expect("signed reward config should be valid"); + assert_eq!(agent.config.min_reward, -2); + assert_eq!(agent.config.max_reward, 3); + assert_eq!(agent.config.reward_offset, 2); + + agent.model_update_percept_stream(&[0], -2); + assert_eq!(agent.total_reward, -2.0); + } + + #[cfg(feature = "all-backends")] + #[test] + fn compiled_mcaixi_runtime_matches_legacy_config_for_generic_mixture_backend() { + let config = generic_mixture_config(); + let compiled = config + .compile_planner_run_spec() + .expect("generic planner run should compile"); + let mut legacy = Agent::try_new(config).expect("legacy config agent"); + let mut canonical = + Agent::from_compiled_planner_run(&compiled).expect("compiled planner-run agent"); + + let legacy_trace = run_ctw_trace(&mut legacy, 32); + let canonical_trace = run_ctw_trace(&mut canonical, 32); + assert_eq!(canonical_trace, legacy_trace); + } +} diff --git a/crates/infotheory/src/aixi/aiqi.rs b/crates/infotheory/src/aixi/aiqi.rs new file mode 100644 index 00000000..e417afff --- /dev/null +++ b/crates/infotheory/src/aixi/aiqi.rs @@ -0,0 +1,2109 @@ +//! AIQI implementation from "A Model-Free Universal AI". +//! +//! This module implements a model-free universal agent that predicts +//! discretized H-step returns directly from augmented interaction history. +//! The implementation follows the phase-indexed periodic augmentation in +//! "A Model-Free Universal AI": +//! for return horizon `H` and period `N >= H`, each phase model only inserts +//! returns at indices `i % N == phase`. + +use crate::aixi::common::{ + Action, ActionAlphabet, PerceptVal, RandomGenerator, Reward, RewardEncodingError, + bits_for_cardinality, byte_packed_percept_bits, nonnegative_reward_encoding_bounds, + resolve_random_seed, validate_aiqi_byte_packed_alignment, validate_reward_encoding_bounds, +}; +use crate::aixi::model::{ + Predictor, PredictorBuildError, build_aiqi_predictor, default_aixi_bit_stream_semantics, +}; +use crate::aixi::planner_spec::{PlannerInterfaceConfig, build_default_planner_run_spec}; +use crate::aixi::return_law::{ + ReturnLabelCodec, ReturnLawEvaluator, ReturnPrefixUpdate, predict_expected_label, +}; +use crate::api::{BitStreamSemantics, RateBackend, validate_rate_backend}; +use crate::spec::{ + AiqiDiscountedControllerSpec, CompiledPlannerController, CompiledPlannerRunSpec, + ControllerSpec, PlannerRunSpec, SpecError, +}; +use std::error::Error; +use std::fmt; + +/// Error returned by AIQI configuration validation, construction, and transition ingestion. +#[derive(Debug)] +#[non_exhaustive] +pub enum AiqiError { + /// `return_horizon` was zero. + ReturnHorizonZero, + /// `return_bins` was zero. + ReturnBinsZero, + /// The augmentation period was smaller than the return horizon. + AugmentationPeriodTooShort { + /// The configured augmentation period. + augmentation_period: usize, + /// The configured return horizon. + return_horizon: usize, + }, + /// The discount factor was outside `(0, 1)`. + InvalidDiscountGamma { + /// The invalid discount factor value. + value: f64, + }, + /// The baseline exploration probability was outside `(0, 1]`. + InvalidBaselineExploration { + /// The invalid baseline exploration value. + value: f64, + }, + /// The configured reward range is not representable. + RewardEncoding(RewardEncodingError), + /// The configured rate backend failed validation. + InvalidRateBackend(crate::error::InfotheoryError), + /// The configured rate backend violates AIQI runtime requirements. + UnsupportedRateBackend { + /// Human-readable explanation of why the backend is unsupported. + reason: &'static str, + }, + /// Planner-run spec compilation failed. + Spec(SpecError), + /// The compiled planner-run controller kind was not discounted AIQI. + ControllerKindMismatch, + /// Predictor construction failed. + Predictor(PredictorBuildError), + /// An observed action was outside the configured action alphabet. + ActionOutOfRange { + /// The out-of-range action token. + action: Action, + /// The configured action alphabet cardinality. + agent_actions: ActionAlphabet, + }, + /// The observation stream length did not match the configured interface. + ObservationStreamLengthMismatch { + /// Expected observation stream length. + expected: usize, + /// Actual observation stream length. + actual: usize, + }, + /// An observed reward was outside the configured reward range. + RewardOutOfRange { + /// Out-of-range reward value. + reward: Reward, + /// Minimum configured reward. + min_reward: Reward, + /// Maximum configured reward. + max_reward: Reward, + }, + /// An observation value exceeded the configured observation bit width. + ObservationValueOutOfRange { + /// Out-of-range observation value. + observation: PerceptVal, + /// Configured observation bit width. + observation_bits: usize, + /// Maximum representable observation value for `observation_bits`. + maximum: PerceptVal, + }, + /// A shifted observed reward became negative. + NegativeEncodedReward { + /// Original reward value. + reward: Reward, + /// Configured reward offset. + reward_offset: Reward, + }, + /// A shifted observed reward exceeded the configured reward bit capacity. + EncodedRewardTooLarge { + /// Shifted reward value after applying offset. + shifted_reward: i128, + /// Configured reward bit width. + reward_bits: usize, + /// Maximum representable encoded reward for `reward_bits`. + maximum_encoded: u128, + }, + /// A requested global step is no longer present in retained history. + HistoryIndexOutOfRange { + /// Requested global step index. + global_step: usize, + /// First retained global step index. + history_base_step: usize, + /// Last observed global step index. + total_steps_observed: usize, + }, + /// A phase-model update required a return bin that has not been computed. + MissingReturnBin { + /// Global step whose return bin was missing. + step: usize, + /// Augmentation phase that required the return bin. + phase: usize, + }, +} + +impl fmt::Display for AiqiError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::ReturnHorizonZero => f.write_str("return_horizon must be >= 1"), + Self::ReturnBinsZero => f.write_str("return_bins must be >= 1"), + Self::AugmentationPeriodTooShort { + augmentation_period, + return_horizon, + } => write!( + f, + "augmentation_period must be >= return_horizon (got N={augmentation_period}, H={return_horizon})" + ), + Self::InvalidDiscountGamma { value } => write!( + f, + "discount_gamma must be in (0, 1) for AIQI as defined in \"A Model-Free Universal AI\", got {value}" + ), + Self::InvalidBaselineExploration { value } => write!( + f, + "baseline_exploration (tau) must be in (0, 1] for AIQI as defined in \"A Model-Free Universal AI\", got {value}" + ), + Self::RewardEncoding(err) => write!(f, "{err}"), + Self::InvalidRateBackend(err) => write!(f, "invalid rate_backend: {err}"), + Self::UnsupportedRateBackend { reason } => f.write_str(reason), + Self::Spec(err) => write!(f, "{err}"), + Self::ControllerKindMismatch => { + f.write_str("compiled planner run does not contain a discounted AIQI controller") + } + Self::Predictor(err) => write!(f, "{err}"), + Self::ActionOutOfRange { + action, + agent_actions, + } => write!( + f, + "action out of range: action={action} but agent_actions={agent_actions}" + ), + Self::ObservationStreamLengthMismatch { expected, actual } => write!( + f, + "observation stream length mismatch: expected {expected}, got {actual}" + ), + Self::RewardOutOfRange { + reward, + min_reward, + max_reward, + } => write!( + f, + "reward out of configured range: reward={reward} not in [{min_reward}, {max_reward}]" + ), + Self::ObservationValueOutOfRange { + observation, + observation_bits, + maximum, + } => write!( + f, + "observation value {observation} does not fit observation_bits={observation_bits} (max={maximum})" + ), + Self::NegativeEncodedReward { + reward, + reward_offset, + } => write!( + f, + "encoded reward became negative after offset: reward={reward} offset={reward_offset}" + ), + Self::EncodedRewardTooLarge { + shifted_reward, + reward_bits, + maximum_encoded, + } => write!( + f, + "encoded reward {shifted_reward} exceeds reward_bits={reward_bits} capacity {maximum_encoded}" + ), + Self::HistoryIndexOutOfRange { + global_step, + history_base_step, + total_steps_observed, + } => write!( + f, + "global step {global_step} out of retained history range [{history_base_step}, {total_steps_observed}]" + ), + Self::MissingReturnBin { step, phase } => write!( + f, + "missing return bin for step {step} in phase {phase} while pushing augmented history" + ), + } + } +} + +impl Error for AiqiError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::RewardEncoding(err) => Some(err), + Self::InvalidRateBackend(err) => Some(err), + Self::Spec(err) => Some(err), + Self::Predictor(err) => Some(err), + _ => None, + } + } +} + +impl From for AiqiError { + fn from(value: RewardEncodingError) -> Self { + Self::RewardEncoding(value) + } +} + +impl From for AiqiError { + fn from(value: SpecError) -> Self { + Self::Spec(value) + } +} + +/// Configuration parameters for an AIQI agent. +#[derive(Clone)] +#[non_exhaustive] +pub struct AiqiConfig { + /// Predictive backend. + pub rate_backend: RateBackend, + /// Bit-stream semantics used to adapt generic rate backends to AIQI symbols. + pub bit_stream_semantics: BitStreamSemantics, + /// Number of bits used to encode observations. + pub observation_bits: usize, + /// Number of observation symbols per environment step. + pub observation_stream_len: usize, + /// Number of bits used to encode rewards. + pub reward_bits: usize, + /// Number of valid actions. + pub agent_actions: ActionAlphabet, + /// Minimum possible environment reward. + pub min_reward: Reward, + /// Maximum possible environment reward. + pub max_reward: Reward, + /// Offset applied before encoding reward bits. + pub reward_offset: Reward, + /// Discount factor used when constructing H-step returns. + pub discount_gamma: f64, + /// Return horizon `H`. + pub return_horizon: usize, + /// Number of discretization bins `M` for returns. + /// + /// Non-power-of-two alphabets are represented by fixed-width binary labels + /// with invalid leaves excluded from the exact return law. + pub return_bins: usize, + /// Augmentation period `N` (must satisfy `N >= H`). + pub augmentation_period: usize, + /// Optional history retention knob for bounded memory growth. + /// + /// - `None`: keep full history (default behavior, no pruning). + /// - `Some(k)`: keep at least the most recent `k` steps, while also + /// preserving all steps still required for exact return construction and + /// deferred phase-model advancement. + pub history_prune_keep_steps: Option, + /// Baseline epsilon-greedy exploration probability `tau`. + pub baseline_exploration: f64, + /// Optional deterministic RNG seed for action selection/exploration. + /// + /// When `None`, planner runtime canonicalizes this to seed `0`. + pub random_seed: Option, +} + +impl Default for AiqiConfig { + fn default() -> Self { + Self { + rate_backend: RateBackend::Ctw { depth: 8 }, + bit_stream_semantics: default_aixi_bit_stream_semantics(), + observation_bits: 1, + observation_stream_len: 1, + reward_bits: 1, + agent_actions: ActionAlphabet::try_from_usize(2) + .expect("default action alphabet must be non-zero"), + min_reward: 0, + max_reward: 1, + reward_offset: 0, + discount_gamma: 0.99, + return_horizon: 4, + return_bins: 8, + augmentation_period: 4, + history_prune_keep_steps: None, + baseline_exploration: 0.01, + random_seed: None, + } + } +} + +impl AiqiConfig { + fn canonical_predictor_backend(&self) -> RateBackend { + self.rate_backend.clone() + } + + fn canonical_planner_run_spec(&self) -> PlannerRunSpec { + let predictor = self.canonical_predictor_backend(); + build_default_planner_run_spec( + PlannerInterfaceConfig { + observation_bits: self.observation_bits, + observation_stream_len: self.observation_stream_len, + observation_key_mode: crate::aixi::common::ObservationKeyMode::FullStream, + reward_bits: self.reward_bits, + agent_actions: self.agent_actions, + }, + ControllerSpec::AiqiDiscounted(AiqiDiscountedControllerSpec { + predictor, + bit_stream_semantics: self.bit_stream_semantics, + discount_gamma: self.discount_gamma, + return_horizon: self.return_horizon, + return_bins: self.return_bins, + augmentation_period: self.augmentation_period, + history_prune_keep_steps: self.history_prune_keep_steps, + baseline_exploration: self.baseline_exploration, + }), + self.random_seed, + ) + } + + fn compile_planner_run_spec(&self) -> Result { + self.canonical_planner_run_spec() + .compile() + .map_err(AiqiError::from) + } + + fn validate_runtime_invariants(&self) -> Result<(), AiqiError> { + if self.return_horizon == 0 { + return Err(AiqiError::ReturnHorizonZero); + } + if self.return_bins == 0 { + return Err(AiqiError::ReturnBinsZero); + } + if self.augmentation_period < self.return_horizon { + return Err(AiqiError::AugmentationPeriodTooShort { + augmentation_period: self.augmentation_period, + return_horizon: self.return_horizon, + }); + } + if !(0.0 < self.discount_gamma && self.discount_gamma < 1.0) { + return Err(AiqiError::InvalidDiscountGamma { + value: self.discount_gamma, + }); + } + if !(0.0 < self.baseline_exploration && self.baseline_exploration <= 1.0) { + return Err(AiqiError::InvalidBaselineExploration { + value: self.baseline_exploration, + }); + } + validate_reward_encoding_bounds( + self.min_reward, + self.max_reward, + self.reward_offset, + self.reward_bits, + )?; + if matches!( + self.bit_stream_semantics, + BitStreamSemantics::BytePacked { .. } + ) { + let action_bits = self.agent_actions.action_bits(); + let percept_bits = byte_packed_percept_bits( + self.observation_bits, + self.observation_stream_len, + self.reward_bits, + ); + let return_bits = bits_for_cardinality(self.return_bins); + validate_aiqi_byte_packed_alignment(action_bits, percept_bits, return_bits) + .map_err(|reason| AiqiError::UnsupportedRateBackend { reason })?; + } + + validate_rate_backend(&self.rate_backend).map_err(AiqiError::InvalidRateBackend)?; + if !rate_backend_supports_aiqi_frozen_conditioning(&self.rate_backend) { + return Err(AiqiError::UnsupportedRateBackend { + reason: "AIQI strict mode requires frozen context updates; configured rate_backend contains zpaq which does not provide strict frozen conditioning", + }); + } + Ok(()) + } + + /// Validate configuration constraints. + pub fn validate(&self) -> Result<(), AiqiError> { + self.validate_runtime_invariants()?; + self.compile_planner_run_spec().map(|_| ()) + } + + /// Test-only accessor for the private `canonical_predictor_backend` mapping. + /// + /// Used by cross-config alias-symmetry tests in [`crate::aixi::agent`]. + #[doc(hidden)] + #[cfg(test)] + pub(crate) fn canonical_predictor_backend_for_test(&self) -> RateBackend { + self.canonical_predictor_backend() + } +} + +#[derive(Clone)] +struct AiqiRuntimeConfig { + observation_bits: usize, + observation_stream_len: usize, + reward_bits: usize, + agent_actions: ActionAlphabet, + min_reward: Reward, + max_reward: Reward, + reward_offset: Reward, + discount_gamma: f64, + return_horizon: usize, + return_bins: usize, + augmentation_period: usize, + history_prune_keep_steps: Option, + baseline_exploration: f64, + random_seed: u64, +} + +impl AiqiRuntimeConfig { + fn from_config(config: &AiqiConfig) -> Self { + Self { + observation_bits: config.observation_bits, + observation_stream_len: config.observation_stream_len.max(1), + reward_bits: config.reward_bits, + agent_actions: config.agent_actions, + min_reward: config.min_reward, + max_reward: config.max_reward, + reward_offset: config.reward_offset, + discount_gamma: config.discount_gamma, + return_horizon: config.return_horizon, + return_bins: config.return_bins, + augmentation_period: config.augmentation_period, + history_prune_keep_steps: config.history_prune_keep_steps, + baseline_exploration: config.baseline_exploration, + random_seed: resolve_random_seed(config.random_seed), + } + } + + fn from_compiled(compiled: &CompiledPlannerRunSpec) -> Result { + let interface = compiled.interface(); + let runtime = compiled.runtime(); + let ( + discount_gamma, + return_horizon, + return_bins, + augmentation_period, + history_prune_keep_steps, + baseline_exploration, + ) = match compiled.controller() { + CompiledPlannerController::AiqiDiscounted { + discount_gamma, + return_horizon, + return_bins, + augmentation_period, + history_prune_keep_steps, + baseline_exploration, + .. + } => ( + *discount_gamma, + *return_horizon, + *return_bins, + *augmentation_period, + *history_prune_keep_steps, + *baseline_exploration, + ), + _ => return Err(AiqiError::ControllerKindMismatch), + }; + + let (min_reward, max_reward, reward_offset) = + nonnegative_reward_encoding_bounds(interface.reward_bits); + Ok(Self { + observation_bits: interface.observation_bits, + observation_stream_len: interface.observation_stream_len.max(1), + reward_bits: interface.reward_bits, + agent_actions: interface.agent_actions, + min_reward, + max_reward, + reward_offset, + discount_gamma, + return_horizon, + return_bins, + augmentation_period, + history_prune_keep_steps, + baseline_exploration, + random_seed: resolve_random_seed(runtime.random_seed), + }) + } +} + +#[derive(Clone, Debug)] +struct StepRecord { + action: Action, + observations: Vec, + reward: Reward, +} + +struct PhaseModel { + predictor: Box, + // Largest step index for which this phase model has consumed + // the augmented stream up to and including that step's percept. + last_augmented_step: usize, +} + +/// AIQI agent with phase-indexed augmented return predictors. +pub struct AiqiAgent { + config: AiqiRuntimeConfig, + phases: Vec, + steps: Vec, + return_bins_by_step: Vec>, + // Global 1-based index of steps[0] / return_bins_by_step[0]. + history_base_step: usize, + // Total number of transitions observed so far (global 1-based max step index). + total_steps_observed: usize, + action_bits: usize, + return_label_codec: ReturnLabelCodec, + use_generic_planner: bool, + rng: RandomGenerator, +} + +impl AiqiAgent { + /// Construct a new AIQI agent. + pub fn new(config: AiqiConfig) -> Result { + config.validate_runtime_invariants()?; + let compiled = config.compile_planner_run_spec()?; + let runtime = AiqiRuntimeConfig::from_config(&config); + Self::from_compiled_config(runtime, &compiled) + } + + /// Construct a new AIQI agent directly from a compiled planner-run spec. + pub fn from_compiled_planner_run(compiled: &CompiledPlannerRunSpec) -> Result { + let config = AiqiRuntimeConfig::from_compiled(compiled)?; + Self::from_compiled_config(config, compiled) + } + + fn from_compiled_config( + config: AiqiRuntimeConfig, + compiled: &CompiledPlannerRunSpec, + ) -> Result { + let (predictor, augmentation_period, return_bins, bit_stream_semantics) = + match compiled.controller() { + CompiledPlannerController::AiqiDiscounted { + predictor, + augmentation_period, + return_bins, + bit_stream_semantics, + .. + } => ( + predictor, + *augmentation_period, + *return_bins, + *bit_stream_semantics, + ), + _ => return Err(AiqiError::ControllerKindMismatch), + }; + let action_bits = compiled.action_bits(); + let return_label_codec = ReturnLabelCodec::value_monotone(return_bins); + let return_bits = return_label_codec.bits(); + let uses_native_reversible_binary_predictor = bit_stream_semantics + == BitStreamSemantics::BinaryTokens + && predictor.supports_native_bit_prediction() + && predictor.supports_reversible_bit_updates(); + let use_generic_planner = !uses_native_reversible_binary_predictor; + + let mut phases = Vec::with_capacity(augmentation_period); + for _ in 0..augmentation_period { + phases.push(PhaseModel { + predictor: build_aiqi_predictor(predictor, return_bits, bit_stream_semantics) + .map_err(AiqiError::Predictor)?, + last_augmented_step: 0, + }); + } + + let rng = RandomGenerator::from_seed(config.random_seed); + + Ok(Self { + action_bits, + return_label_codec, + use_generic_planner, + config, + phases, + steps: Vec::new(), + return_bins_by_step: Vec::new(), + history_base_step: 1, + total_steps_observed: 0, + rng, + }) + } + + /// Number of transitions incorporated so far. + pub fn steps_observed(&self) -> usize { + self.total_steps_observed + } + + /// Returns the configured action alphabet cardinality. + pub fn num_actions(&self) -> ActionAlphabet { + self.config.agent_actions + } + + /// Returns the resolved deterministic seed used by this AIQI agent. + pub fn resolved_random_seed(&self) -> u64 { + self.config.random_seed + } + + pub(crate) fn reseed_random(&mut self, seed: u64) { + self.config.random_seed = seed; + self.rng = RandomGenerator::from_seed(seed); + } + + /// Select the next action from the current history. + pub fn get_planned_action(&mut self) -> Action { + self.get_planned_action_with_extra_exploration_flag(0.0).0 + } + + /// Select the next action and report whether it was sampled for exploration. + pub fn get_planned_action_with_extra_exploration_flag( + &mut self, + extra_exploration: f64, + ) -> (Action, bool) { + let extra = extra_exploration.clamp(0.0, 1.0); + let tau = self.config.baseline_exploration.clamp(0.0, 1.0); + let effective = 1.0 - (1.0 - tau) * (1.0 - extra); + if effective > 0.0 && self.rng.gen_bool(effective) { + ( + self.rng.gen_range(self.config.agent_actions.get()) as u64, + true, + ) + } else { + let q_values = self.estimate_q_values(); + let greedy_action = argmax_with_fixed_tie_break(&q_values) as u64; + (greedy_action, false) + } + } + + /// Select the next action, adding optional extra exploration. + /// + /// The extra exploration probability is combined as + /// `p = 1 - (1 - tau) * (1 - extra)`, where `tau` is the baseline + /// exploration in [`AiqiConfig`]. + pub fn get_planned_action_with_extra_exploration(&mut self, extra_exploration: f64) -> Action { + self.get_planned_action_with_extra_exploration_flag(extra_exploration) + .0 + } + + /// Record one environment transition `(action, observations, reward)`. + /// + /// This appends to history and, when enough future rewards are known, + /// computes and learns one newly available discretized return. + pub fn observe_transition( + &mut self, + action: Action, + observations: &[PerceptVal], + reward: Reward, + ) -> Result<(), AiqiError> { + if action as usize >= self.config.agent_actions.get() { + return Err(AiqiError::ActionOutOfRange { + action, + agent_actions: self.config.agent_actions, + }); + } + + let expected_obs = self.config.observation_stream_len.max(1); + if observations.len() != expected_obs { + return Err(AiqiError::ObservationStreamLengthMismatch { + expected: expected_obs, + actual: observations.len(), + }); + } + + if reward < self.config.min_reward || reward > self.config.max_reward { + return Err(AiqiError::RewardOutOfRange { + reward, + min_reward: self.config.min_reward, + max_reward: self.config.max_reward, + }); + } + + let obs_max = max_value_for_bits(self.config.observation_bits); + for &obs in observations { + if obs > obs_max { + return Err(AiqiError::ObservationValueOutOfRange { + observation: obs, + observation_bits: self.config.observation_bits, + maximum: obs_max, + }); + } + } + + let rew_shifted = (reward as i128) + (self.config.reward_offset as i128); + if rew_shifted < 0 { + return Err(AiqiError::NegativeEncodedReward { + reward, + reward_offset: self.config.reward_offset, + }); + } + if self.config.reward_bits < 64 { + let max_enc = (1u128 << self.config.reward_bits) - 1; + if (rew_shifted as u128) > max_enc { + return Err(AiqiError::EncodedRewardTooLarge { + shifted_reward: rew_shifted, + reward_bits: self.config.reward_bits, + maximum_encoded: max_enc, + }); + } + } + + self.steps.push(StepRecord { + action, + observations: observations.to_vec(), + reward, + }); + self.total_steps_observed += 1; + self.return_bins_by_step.push(None); + + self.maybe_learn_new_return()?; + self.maybe_prune_history(); + Ok(()) + } + + fn maybe_learn_new_return(&mut self) -> Result<(), AiqiError> { + let t = self.total_steps_observed; + let h = self.config.return_horizon; + if t < h { + return Ok(()); + } + + // Newly available return index (1-based): i = t - H + 1. + let i = t + 1 - h; + let bin = self.compute_return_bin(i); + let local_idx = self.local_index(i)?; + self.return_bins_by_step[local_idx] = Some(bin); + + let phase = i % self.config.augmentation_period; + self.advance_phase_model_to_step(phase, i) + } + + fn estimate_q_values(&mut self) -> Vec { + if self.use_generic_planner { + return self.estimate_q_values_generic(); + } + + let step = self.total_steps_observed + 1; + let phase = step % self.config.augmentation_period; + let config = &self.config; + let steps = &self.steps; + let return_bins_by_step = &self.return_bins_by_step; + let history_base_step = self.history_base_step; + let action_bits = self.action_bits; + let return_label_codec = self.return_label_codec; + let token_ctx = AiqiAugmentedTokenContext { + config, + history_base_step, + steps, + return_bins_by_step, + action_bits, + return_label_codec, + phase, + }; + + let mut q_values = vec![0.0; self.config.agent_actions.get()]; + let mut pushed_fast_forward = 0usize; + + { + let model = &mut self.phases[phase]; + let start = (model.last_augmented_step + 1).max(history_base_step); + let end = step.saturating_sub(1); + if start <= end { + for idx in start..=end { + pushed_fast_forward += + push_step_tokens_history(&token_ctx, model.predictor.as_mut(), idx); + } + } + + for (action, q_value) in q_values + .iter_mut() + .enumerate() + .take(self.config.agent_actions.get()) + { + let pushed_action = push_encoded_bits_history( + model.predictor.as_mut(), + action as u64, + self.action_bits, + ); + let expected_label = predict_expected_label( + model.predictor.as_mut(), + self.return_label_codec, + ReturnPrefixUpdate::Training, + ReturnLawEvaluator::SharedPrefix, + ); + *q_value = expected_label / self.config.return_bins as f64; + pop_history_bits(model.predictor.as_mut(), pushed_action); + } + + pop_history_bits(model.predictor.as_mut(), pushed_fast_forward); + } + + q_values + } + + fn estimate_q_values_generic(&mut self) -> Vec { + let step = self.total_steps_observed + 1; + let phase = step % self.config.augmentation_period; + + let model = &self.phases[phase]; + let mut context_predictor = model.predictor.boxed_clone(); + let token_ctx = AiqiAugmentedTokenContext { + config: &self.config, + history_base_step: self.history_base_step, + steps: &self.steps, + return_bins_by_step: &self.return_bins_by_step, + action_bits: self.action_bits, + return_label_codec: self.return_label_codec, + phase, + }; + + let start = (model.last_augmented_step + 1).max(self.history_base_step); + let end = step.saturating_sub(1); + if start <= end { + for idx in start..=end { + push_augmented_step_tokens_commit(&token_ctx, context_predictor.as_mut(), idx) + .expect( + "generic planner retained history must contain required augmented return", + ); + } + } + + let mut q_values = vec![0.0; self.config.agent_actions.get()]; + for (action, q_value) in q_values + .iter_mut() + .enumerate() + .take(self.config.agent_actions.get()) + { + let mut action_predictor = context_predictor.boxed_clone(); + let _ = push_encoded_bits_commit_history( + action_predictor.as_mut(), + action as u64, + self.action_bits, + ); + let expected_label = predict_expected_label( + action_predictor.as_mut(), + self.return_label_codec, + ReturnPrefixUpdate::Training, + ReturnLawEvaluator::SharedPrefix, + ); + *q_value = expected_label / self.config.return_bins as f64; + } + + q_values + } + + fn advance_phase_model_to_step( + &mut self, + phase: usize, + target_step: usize, + ) -> Result<(), AiqiError> { + let token_ctx = AiqiAugmentedTokenContext { + config: &self.config, + history_base_step: self.history_base_step, + steps: &self.steps, + return_bins_by_step: &self.return_bins_by_step, + action_bits: self.action_bits, + return_label_codec: self.return_label_codec, + phase, + }; + let model = &mut self.phases[phase]; + if target_step <= model.last_augmented_step { + return Ok(()); + } + + let start = (model.last_augmented_step + 1).max(token_ctx.history_base_step); + for idx in start..=target_step { + push_augmented_step_tokens_commit(&token_ctx, model.predictor.as_mut(), idx)?; + } + + model.last_augmented_step = target_step; + Ok(()) + } + + fn compute_return_bin(&self, start_step: usize) -> u64 { + let h = self.config.return_horizon; + let gamma = self.config.discount_gamma; + + debug_assert!(0.0 < gamma && gamma < 1.0); + let reward_range = (self.config.max_reward - self.config.min_reward) as f64; + + // Paper definition: R_{t,H} = (1-gamma) * sum_{k=0}^{H-1} gamma^k r_{t+k}. + let mut total = 0.0f64; + let mut gk = 1.0f64; + for k in 0..h { + let idx = start_step + k; + let local_idx = self + .local_index(idx) + .expect("return computation requires in-range history"); + let r = self.steps[local_idx].reward; + let rn = if reward_range <= 0.0 { + 0.0 + } else { + ((r - self.config.min_reward) as f64 / reward_range).clamp(0.0, 1.0) + }; + total += gk * rn; + gk *= gamma; + } + let ret = ((1.0 - gamma) * total).clamp(0.0, 1.0); + + let mut bin = (ret * (self.config.return_bins as f64)).floor() as u64; + let max_bin = (self.config.return_bins as u64).saturating_sub(1); + if bin > max_bin { + bin = max_bin; + } + bin + } + + fn local_index(&self, global_step: usize) -> Result { + if global_step < self.history_base_step || global_step > self.total_steps_observed { + return Err(AiqiError::HistoryIndexOutOfRange { + global_step, + history_base_step: self.history_base_step, + total_steps_observed: self.total_steps_observed, + }); + } + Ok(global_step - self.history_base_step) + } + + fn maybe_prune_history(&mut self) { + let Some(keep_steps) = self.config.history_prune_keep_steps else { + return; + }; + if self.steps.is_empty() { + return; + } + + let min_phase_committed = self + .phases + .iter() + .map(|phase| phase.last_augmented_step) + .min() + .unwrap_or(0); + + // For the next return update, we must retain steps from + // (t+2-H) onward (1-based indexing). Everything before that is no + // longer needed for exact H-step return construction. + let next_start_needed = self + .total_steps_observed + .saturating_add(2) + .saturating_sub(self.config.return_horizon); + let returns_safe_drop_upto = next_start_needed.saturating_sub(1); + + let mut safe_drop_upto = min_phase_committed.min(returns_safe_drop_upto); + + // Optional retention floor: keep at least `keep_steps` most recent + // transitions in memory for diagnostics/debugging. + let keep_floor_drop_upto = self.total_steps_observed.saturating_sub(keep_steps); + safe_drop_upto = safe_drop_upto.min(keep_floor_drop_upto); + + if safe_drop_upto < self.history_base_step { + return; + } + + let drain_count = safe_drop_upto - self.history_base_step + 1; + if drain_count == 0 || drain_count > self.steps.len() { + return; + } + + self.steps.drain(0..drain_count); + self.return_bins_by_step.drain(0..drain_count); + self.history_base_step += drain_count; + } +} + +struct AiqiAugmentedTokenContext<'a> { + config: &'a AiqiRuntimeConfig, + history_base_step: usize, + steps: &'a [StepRecord], + return_bins_by_step: &'a [Option], + action_bits: usize, + return_label_codec: ReturnLabelCodec, + phase: usize, +} + +fn push_step_tokens_history( + ctx: &AiqiAugmentedTokenContext<'_>, + predictor: &mut dyn Predictor, + idx: usize, +) -> usize { + let mut pushed = 0usize; + pushed += push_action_tokens_history( + ctx.history_base_step, + ctx.steps, + ctx.action_bits, + predictor, + idx, + ); + + if idx % ctx.config.augmentation_period == ctx.phase { + let local_idx = idx - ctx.history_base_step; + if let Some(bin) = ctx.return_bins_by_step[local_idx] { + pushed += ctx.return_label_codec.push_label_history(predictor, bin); + } + } + + pushed + + push_percept_tokens_history(ctx.config, ctx.history_base_step, ctx.steps, predictor, idx) +} + +fn push_augmented_step_tokens_commit( + ctx: &AiqiAugmentedTokenContext<'_>, + predictor: &mut dyn Predictor, + idx: usize, +) -> Result { + let mut pushed = 0usize; + pushed += push_action_tokens_commit_history( + ctx.history_base_step, + ctx.steps, + ctx.action_bits, + predictor, + idx, + ); + + if idx % ctx.config.augmentation_period == ctx.phase { + let local_idx = idx - ctx.history_base_step; + let bin = ctx.return_bins_by_step[local_idx].ok_or(AiqiError::MissingReturnBin { + step: idx, + phase: ctx.phase, + })?; + pushed += ctx.return_label_codec.push_label_commit(predictor, bin); + } + + Ok(pushed + + push_percept_tokens_commit_history( + ctx.config, + ctx.history_base_step, + ctx.steps, + predictor, + idx, + )) +} + +fn push_action_tokens_history( + history_base_step: usize, + steps: &[StepRecord], + action_bits: usize, + predictor: &mut dyn Predictor, + idx: usize, +) -> usize { + let action = steps[idx - history_base_step].action; + push_encoded_bits_history(predictor, action, action_bits) +} + +fn push_action_tokens_commit_history( + history_base_step: usize, + steps: &[StepRecord], + action_bits: usize, + predictor: &mut dyn Predictor, + idx: usize, +) -> usize { + let action = steps[idx - history_base_step].action; + push_encoded_bits_commit_history(predictor, action, action_bits) +} + +fn push_percept_tokens_history( + config: &AiqiRuntimeConfig, + history_base_step: usize, + steps: &[StepRecord], + predictor: &mut dyn Predictor, + idx: usize, +) -> usize { + let step = &steps[idx - history_base_step]; + let mut pushed = 0usize; + for &obs in &step.observations { + pushed += push_encoded_bits_history(predictor, obs, config.observation_bits); + } + pushed + + push_encoded_reward_history( + predictor, + step.reward, + config.reward_bits, + config.reward_offset, + ) +} + +fn push_percept_tokens_commit_history( + config: &AiqiRuntimeConfig, + history_base_step: usize, + steps: &[StepRecord], + predictor: &mut dyn Predictor, + idx: usize, +) -> usize { + let step = &steps[idx - history_base_step]; + let mut pushed = 0usize; + for &obs in &step.observations { + pushed += push_encoded_bits_commit_history(predictor, obs, config.observation_bits); + } + pushed + + push_encoded_reward_commit_history( + predictor, + step.reward, + config.reward_bits, + config.reward_offset, + ) +} + +fn rate_backend_supports_aiqi_frozen_conditioning(backend: &RateBackend) -> bool { + backend + .compile() + .map(|compiled| compiled.supports_frozen_conditioning()) + .unwrap_or(false) +} + +fn max_value_for_bits(bits: usize) -> u64 { + if bits >= 64 { + u64::MAX + } else if bits == 0 { + 0 + } else { + (1u64 << bits) - 1 + } +} + +fn push_encoded_bits_history(predictor: &mut dyn Predictor, value: u64, bits: usize) -> usize { + let mut v = value; + for _ in 0..bits { + predictor.update_history((v & 1) == 1); + v >>= 1; + } + bits +} + +fn push_encoded_bits_commit_history( + predictor: &mut dyn Predictor, + value: u64, + bits: usize, +) -> usize { + let mut v = value; + for _ in 0..bits { + predictor.commit_update_history((v & 1) == 1); + v >>= 1; + } + bits +} + +fn push_encoded_reward_history( + predictor: &mut dyn Predictor, + reward: Reward, + bits: usize, + offset: Reward, +) -> usize { + let shifted = (reward as i128) + (offset as i128); + let as_u64 = if shifted <= 0 { + 0 + } else if shifted > (u64::MAX as i128) { + u64::MAX + } else { + shifted as u64 + }; + push_encoded_bits_history(predictor, as_u64, bits) +} + +fn push_encoded_reward_commit_history( + predictor: &mut dyn Predictor, + reward: Reward, + bits: usize, + offset: Reward, +) -> usize { + let shifted = (reward as i128) + (offset as i128); + let as_u64 = if shifted <= 0 { + 0 + } else if shifted > (u64::MAX as i128) { + u64::MAX + } else { + shifted as u64 + }; + push_encoded_bits_commit_history(predictor, as_u64, bits) +} + +fn pop_history_bits(predictor: &mut dyn Predictor, bits: usize) { + for _ in 0..bits { + predictor.pop_history(); + } +} + +fn argmax_with_fixed_tie_break(values: &[f64]) -> usize { + let mut best_value = f64::NEG_INFINITY; + let mut best_idx = 0usize; + for (i, &v) in values.iter().enumerate() { + if v > best_value { + best_value = v; + best_idx = i; + } + } + best_idx +} + +#[cfg(all(test, feature = "all-backends"))] +mod tests { + use super::*; + use crate::aixi::environment::Environment; + use crate::aixi::return_law::{ + ReturnLabelBitOrder, ReturnLabelCodec, ReturnLawEvaluator, ReturnPrefixUpdate, + predict_return_law, + }; + use crate::aixi::test_envs::DeterministicBinaryEnv; + use crate::api::{MixtureKind, MixtureSpec}; + use std::sync::{Arc, Mutex}; + + fn basic_config() -> AiqiConfig { + AiqiConfig { + rate_backend: RateBackend::Ctw { depth: 8 }, + bit_stream_semantics: crate::api::BitStreamSemantics::BinaryTokens, + observation_bits: 1, + observation_stream_len: 1, + reward_bits: 1, + agent_actions: ActionAlphabet::try_from_usize(2) + .expect("test fixture action alphabet must be non-zero"), + min_reward: 0, + max_reward: 1, + reward_offset: 0, + discount_gamma: 0.99, + return_horizon: 2, + return_bins: 8, + augmentation_period: 2, + history_prune_keep_steps: None, + baseline_exploration: 0.01, + random_seed: Some(7), + } + } + + fn generic_mixture_config() -> AiqiConfig { + AiqiConfig { + rate_backend: RateBackend::Mixture { + spec: Arc::new( + MixtureSpec::new( + MixtureKind::Bayes, + vec![ + crate::api::MixtureExpertSpec { + name: Some("ctw".to_string()), + log_prior: 0.0, + backend: RateBackend::Ctw { depth: 8 }, + }, + crate::api::MixtureExpertSpec { + name: Some("match".to_string()), + log_prior: 0.0, + backend: RateBackend::Match { + hash_bits: 16, + min_len: 2, + max_len: 16, + base_mix: 0.05, + confidence_scale: 1.0, + }, + }, + ], + ) + .with_alpha(0.03), + ), + }, + random_seed: Some(11), + baseline_exploration: 0.01, + ..basic_config() + } + } + + fn native_reversible_mixture_config() -> AiqiConfig { + AiqiConfig { + rate_backend: RateBackend::Mixture { + spec: Arc::new( + MixtureSpec::new( + MixtureKind::Bayes, + vec![ + crate::api::MixtureExpertSpec { + name: Some("ctw".to_string()), + log_prior: 0.0, + backend: RateBackend::Ctw { depth: 8 }, + }, + crate::api::MixtureExpertSpec { + name: Some("fac-ctw".to_string()), + log_prior: 0.0, + backend: RateBackend::FacCtw { + base_depth: 8, + num_percept_bits: 1, + encoding_bits: 1, + msb_first: None, + }, + }, + ], + ) + .with_alpha(0.03), + ), + }, + random_seed: Some(13), + baseline_exploration: 0.01, + ..basic_config() + } + } + + fn run_ctw_trace(agent: &mut AiqiAgent, cycles: usize) -> (Vec, i64) { + let mut env = DeterministicBinaryEnv::default(); + let mut actions = Vec::with_capacity(cycles); + let mut total_reward = 0i64; + + for _ in 0..cycles { + let action = agent.get_planned_action(); + actions.push(action); + env.perform_action(action); + let obs_stream = env.drain_observations(); + let reward = env.get_reward(); + agent + .observe_transition(action, &obs_stream, reward) + .expect("transition should be accepted"); + total_reward += reward; + } + + (actions, total_reward) + } + + #[test] + fn programmatic_aiqi_preserves_explicit_signed_reward_contract() { + let mut config = basic_config(); + config.reward_bits = 3; + config.min_reward = -2; + config.max_reward = 3; + config.reward_offset = 2; + + let mut agent = AiqiAgent::new(config).expect("signed reward config should be valid"); + assert_eq!(agent.config.min_reward, -2); + assert_eq!(agent.config.max_reward, 3); + assert_eq!(agent.config.reward_offset, 2); + + agent + .observe_transition(0, &[0], -2) + .expect("signed reward within explicit config bounds should be accepted"); + } + + #[derive(Clone, Default)] + struct CountingPredictor { + update_calls: usize, + commit_update_calls: usize, + update_history_calls: usize, + commit_update_history_calls: usize, + revert_calls: usize, + pop_history_calls: usize, + } + + impl Predictor for CountingPredictor { + fn update(&mut self, _sym: bool) { + self.update_calls += 1; + } + + fn commit_update(&mut self, _sym: bool) { + self.commit_update_calls += 1; + } + + fn update_history(&mut self, _sym: bool) { + self.update_history_calls += 1; + } + + fn commit_update_history(&mut self, _sym: bool) { + self.commit_update_history_calls += 1; + } + + fn revert(&mut self) { + self.revert_calls += 1; + } + + fn pop_history(&mut self) { + self.pop_history_calls += 1; + } + + fn predict_prob(&mut self, sym: bool) -> f64 { + if sym { 0.75 } else { 0.25 } + } + + fn model_name(&self) -> String { + "CountingPredictor".to_string() + } + + fn boxed_clone(&self) -> Box { + Box::new(self.clone()) + } + } + + #[derive(Clone, Default)] + struct SharedCallCounts { + update: usize, + commit_update: usize, + update_history: usize, + commit_update_history: usize, + } + + #[derive(Clone)] + struct SharedCountingPredictor { + counts: Arc>, + } + + impl SharedCountingPredictor { + fn new(counts: Arc>) -> Self { + Self { counts } + } + } + + impl Predictor for SharedCountingPredictor { + fn update(&mut self, _sym: bool) { + self.counts.lock().unwrap().update += 1; + } + + fn commit_update(&mut self, _sym: bool) { + self.counts.lock().unwrap().commit_update += 1; + } + + fn update_history(&mut self, _sym: bool) { + self.counts.lock().unwrap().update_history += 1; + } + + fn commit_update_history(&mut self, _sym: bool) { + self.counts.lock().unwrap().commit_update_history += 1; + } + + fn revert(&mut self) {} + + fn pop_history(&mut self) {} + + fn predict_prob(&mut self, sym: bool) -> f64 { + if sym { 0.75 } else { 0.25 } + } + + fn model_name(&self) -> String { + "SharedCountingPredictor".to_string() + } + + fn boxed_clone(&self) -> Box { + Box::new(self.clone()) + } + } + + #[derive(Clone, Default)] + struct ReturnLearningPredictor { + saw_training_one: bool, + rollback: Vec, + } + + impl Predictor for ReturnLearningPredictor { + fn update(&mut self, sym: bool) { + self.rollback.push(self.saw_training_one); + if sym { + self.saw_training_one = true; + } + } + + fn commit_update(&mut self, sym: bool) { + if sym { + self.saw_training_one = true; + } + } + + fn update_history(&mut self, _sym: bool) {} + + fn commit_update_history(&mut self, _sym: bool) {} + + fn revert(&mut self) { + self.saw_training_one = self + .rollback + .pop() + .expect("test predictor rollback underflow"); + } + + fn pop_history(&mut self) {} + + fn predict_prob(&mut self, sym: bool) -> f64 { + let p1 = if self.saw_training_one { 0.75 } else { 0.25 }; + if sym { p1 } else { 1.0 - p1 } + } + + fn model_name(&self) -> String { + "ReturnLearningPredictor".to_string() + } + + fn boxed_clone(&self) -> Box { + Box::new(self.clone()) + } + } + + #[derive(Clone, Default)] + struct ActionConditionedBernoulliPredictor { + history: Vec, + } + + impl ActionConditionedBernoulliPredictor { + fn p_one_after_action(&self) -> f64 { + if self.history.first().copied().unwrap_or(false) { + 0.75 + } else { + 0.25 + } + } + } + + impl Predictor for ActionConditionedBernoulliPredictor { + fn update(&mut self, sym: bool) { + self.history.push(sym); + } + + fn commit_update(&mut self, sym: bool) { + self.history.push(sym); + } + + fn update_history(&mut self, sym: bool) { + self.history.push(sym); + } + + fn commit_update_history(&mut self, sym: bool) { + self.history.push(sym); + } + + fn revert(&mut self) { + self.history + .pop() + .expect("test predictor rollback underflow"); + } + + fn pop_history(&mut self) { + self.history + .pop() + .expect("test predictor history underflow"); + } + + fn predict_prob(&mut self, sym: bool) -> f64 { + let p_one = self.p_one_after_action(); + if sym { p_one } else { 1.0 - p_one } + } + + fn model_name(&self) -> String { + "ActionConditionedBernoulliPredictor".to_string() + } + + fn boxed_clone(&self) -> Box { + Box::new(self.clone()) + } + } + + #[derive(Clone)] + struct ScopedReturnLearningPredictor { + saw_training_one: bool, + rollback: Vec, + clone_count: Arc>, + } + + impl ScopedReturnLearningPredictor { + fn new(clone_count: Arc>) -> Self { + Self { + saw_training_one: false, + rollback: Vec::new(), + clone_count, + } + } + } + + impl Predictor for ScopedReturnLearningPredictor { + fn update(&mut self, sym: bool) { + self.rollback.push(self.saw_training_one); + if sym { + self.saw_training_one = true; + } + } + + fn commit_update(&mut self, sym: bool) { + if sym { + self.saw_training_one = true; + } + } + + fn revert(&mut self) { + self.saw_training_one = self + .rollback + .pop() + .expect("test predictor rollback underflow"); + } + + fn predict_prob(&mut self, sym: bool) -> f64 { + let p1 = if self.saw_training_one { 0.75 } else { 0.25 }; + if sym { p1 } else { 1.0 - p1 } + } + + fn model_name(&self) -> String { + "ScopedReturnLearningPredictor".to_string() + } + + fn boxed_clone(&self) -> Box { + *self.clone_count.lock().unwrap() += 1; + Box::new(self.clone()) + } + } + + #[test] + fn config_rejects_invalid_period() { + let mut cfg = basic_config(); + cfg.augmentation_period = 1; + cfg.return_horizon = 2; + let err = cfg + .validate() + .expect_err("N < H must be rejected to match \"A Model-Free Universal AI\""); + assert!(matches!( + err, + AiqiError::AugmentationPeriodTooShort { + augmentation_period: 1, + return_horizon: 2 + } + )); + } + + #[test] + fn config_accepts_non_power_of_two_return_bins() { + let mut cfg = basic_config(); + cfg.return_bins = 3; + cfg.validate() + .expect("non-power-of-two return_bins are valid AIQI discretization levels"); + } + + #[test] + fn non_power_of_two_aiqi_runtime_normalizes_invalid_return_leaf_and_plans() { + let mut cfg = basic_config(); + cfg.return_bins = 3; + cfg.return_horizon = 1; + cfg.augmentation_period = 1; + cfg.baseline_exploration = f64::MIN_POSITIVE; + cfg.random_seed = Some(3); + + let mut agent = AiqiAgent::new(cfg).expect("non-power-of-two AIQI config should build"); + agent.phases[0].predictor = Box::new(ActionConditionedBernoulliPredictor::default()); + + let mut action_one_predictor = ActionConditionedBernoulliPredictor::default(); + action_one_predictor.update_history(true); + let law = predict_return_law( + &mut action_one_predictor, + agent.return_label_codec, + ReturnPrefixUpdate::Training, + ReturnLawEvaluator::SharedPrefix, + ); + assert_eq!(law.probabilities.len(), 3); + assert_eq!(law.stats.invalid_leaves, 1); + assert!( + (law.probabilities.iter().sum::() - 1.0).abs() < 1e-12, + "non-power-of-two AIQI law must normalize only valid labels: {:?}", + law.probabilities + ); + let expected_action_one_law = [1.0 / 7.0, 3.0 / 7.0, 3.0 / 7.0]; + for (actual, expected) in law.probabilities.iter().zip(expected_action_one_law.iter()) { + assert!( + (actual - expected).abs() < 1e-12, + "expected action-1 law {:?}, got {:?}", + expected_action_one_law, + law.probabilities + ); + } + + let q_values = agent.estimate_q_values(); + assert_eq!(q_values.len(), 2); + assert!( + (q_values[0] - 0.2).abs() < 1e-12, + "action 0 should decode the normalized [0.6, 0.2, 0.2] law, got {q_values:?}" + ); + assert!( + (q_values[1] - (3.0 / 7.0)).abs() < 1e-12, + "action 1 should decode the normalized [1/7, 3/7, 3/7] law, got {q_values:?}" + ); + + agent.reseed_random(3); + let (action, explored) = agent.get_planned_action_with_extra_exploration_flag(0.0); + assert_eq!(action, 1); + assert!(!explored); + } + + #[test] + fn forced_aiqi_exploration_skips_value_descent() { + let mut agent = AiqiAgent::new(basic_config()).expect("valid aiqi config"); + let counts = Arc::new(Mutex::new(SharedCallCounts::default())); + let decision_phase = (agent.total_steps_observed + 1) % agent.config.augmentation_period; + agent.phases[decision_phase].predictor = + Box::new(SharedCountingPredictor::new(counts.clone())); + + let (_action, explored) = agent.get_planned_action_with_extra_exploration_flag(1.0); + + assert!(explored); + let snapshot = counts.lock().unwrap().clone(); + assert_eq!(snapshot.update, 0); + assert_eq!(snapshot.update_history, 0); + assert_eq!(snapshot.commit_update, 0); + assert_eq!(snapshot.commit_update_history, 0); + } + + #[test] + fn config_rejects_zpaq_rate_backend_in_strict_mode() { + let mut cfg = basic_config(); + cfg.rate_backend = RateBackend::Zpaq { + method: crate::api::ZpaqMethodSpec::literal("1"), + }; + let err = cfg + .validate() + .expect_err("strict AIQI must reject zpaq rate backend"); + assert!(matches!(err, AiqiError::UnsupportedRateBackend { .. })); + } + + #[test] + fn config_rejects_nonpaper_gamma_or_tau() { + let mut cfg = basic_config(); + cfg.discount_gamma = 1.0; + let err = cfg + .validate() + .expect_err("gamma=1 must be rejected for strict paper AIQI"); + assert!(matches!( + err, + AiqiError::InvalidDiscountGamma { value: 1.0 } + )); + + cfg = basic_config(); + cfg.baseline_exploration = 0.0; + let err = cfg + .validate() + .expect_err("tau=0 must be rejected for strict paper AIQI"); + assert!(matches!( + err, + AiqiError::InvalidBaselineExploration { value: 0.0 } + )); + } + + #[test] + fn byte_packed_config_allows_observation_and_reward_to_share_a_byte() { + let mut cfg = basic_config(); + cfg.bit_stream_semantics = BitStreamSemantics::BytePacked { + order: crate::prediction::BitOrder::MsbFirst, + }; + cfg.agent_actions = ActionAlphabet::try_from_usize(256) + .expect("test fixture action alphabet must be byte-aligned"); + cfg.observation_bits = 3; + cfg.reward_bits = 5; + cfg.return_bins = 256; + + cfg.validate().expect( + "byte-packed AIQI should allow observations and reward to share one percept byte", + ); + } + + #[test] + fn aiqi_estimates_action_values_after_observations() { + let mut agent = AiqiAgent::new(basic_config()).expect("valid aiqi config"); + for _ in 0..8 { + agent + .observe_transition(1, &[1], 1) + .expect("transition should be accepted"); + } + + let action = agent.get_planned_action(); + assert!(action <= 1); + } + + #[test] + fn fac_ctw_predictor_uses_return_bit_width() { + let mut cfg = basic_config(); + cfg.return_bins = 8; // return_bits=3 + cfg.rate_backend = RateBackend::FacCtw { + base_depth: 8, + num_percept_bits: bits_for_cardinality(cfg.return_bins), + encoding_bits: 1, + msb_first: None, + }; + + let agent = AiqiAgent::new(cfg).expect("valid aiqi config"); + let name = agent.phases[0].predictor.model_name(); + assert!( + name.contains("k=3"), + "FAC-CTW should factorize over return bits only, model_name={name}" + ); + } + + #[test] + fn ac_ctw_path_uses_single_tree_predictor() { + let agent = AiqiAgent::new(basic_config()).expect("valid aiqi config"); + let name = agent.phases[0].predictor.model_name(); + assert!( + name.starts_with("AC-CTW"), + "ac-ctw should map to the single-tree CTW predictor, model_name={name}" + ); + } + + #[test] + fn distribution_rollout_uses_update_and_revert_when_requested() { + let mut predictor = CountingPredictor::default(); + let law = predict_return_law( + &mut predictor, + ReturnLabelCodec::value_monotone(4), + ReturnPrefixUpdate::Training, + ReturnLawEvaluator::SharedPrefix, + ); + + assert_eq!(law.probabilities.len(), 4); + assert_eq!(law.stats.logical_queries, 3); + assert_eq!(predictor.update_calls, 6); + assert_eq!(predictor.revert_calls, 6); + assert_eq!(predictor.update_history_calls, 0); + assert_eq!(predictor.pop_history_calls, 0); + } + + #[test] + fn distribution_rollout_uses_history_path_when_not_requested() { + let mut predictor = CountingPredictor::default(); + let law = predict_return_law( + &mut predictor, + ReturnLabelCodec::value_monotone(4), + ReturnPrefixUpdate::FrozenHistory, + ReturnLawEvaluator::SharedPrefix, + ); + + assert_eq!(law.probabilities.len(), 4); + assert_eq!(law.stats.logical_queries, 3); + assert_eq!(predictor.update_calls, 0); + assert_eq!(predictor.revert_calls, 0); + assert_eq!(predictor.update_history_calls, 6); + assert_eq!(predictor.pop_history_calls, 6); + } + + #[test] + fn generic_distribution_rollout_trains_on_return_symbols() { + let mut predictor = ReturnLearningPredictor::default(); + let law = predict_return_law( + &mut predictor, + ReturnLabelCodec::value_monotone(4), + ReturnPrefixUpdate::Training, + ReturnLawEvaluator::SharedPrefix, + ); + let probs = law.probabilities; + + assert_eq!(probs.len(), 4); + assert!((probs.iter().sum::() - 1.0).abs() < 1e-12); + assert!( + probs[3] > probs[2], + "training on the first return bit should make bin 11 likelier than 10; got {:?}", + probs + ); + assert!( + (probs[0] - 0.5625).abs() < 1e-12, + "expected exact normalized mass for 00, got {:?}", + probs + ); + assert!( + !predictor.saw_training_one, + "shared-prefix rollout must restore the caller's predictor state" + ); + } + + #[test] + fn generic_distribution_rollout_does_not_clone_per_label() { + let clone_count = Arc::new(Mutex::new(0usize)); + let mut predictor = ScopedReturnLearningPredictor::new(clone_count.clone()); + let law = predict_return_law( + &mut predictor, + ReturnLabelCodec::value_monotone(4), + ReturnPrefixUpdate::Training, + ReturnLawEvaluator::SharedPrefix, + ); + let probs = law.probabilities; + + assert_eq!(probs.len(), 4); + assert!( + probs[3] > probs[2], + "shared-prefix training on return bits should preserve autoregressive semantics" + ); + assert_eq!( + *clone_count.lock().unwrap(), + 0, + "shared-prefix evaluation should not clone once per return bin" + ); + assert!( + !predictor.saw_training_one, + "shared-prefix rollout must restore the caller's predictor state" + ); + } + + #[test] + fn aiqi_return_label_codec_is_value_monotone() { + let agent = AiqiAgent::new(basic_config()).expect("valid aiqi config"); + assert_eq!( + agent.return_label_codec.order(), + ReturnLabelBitOrder::MsbFirst + ); + assert_eq!( + agent.return_label_codec.label_range_for_prefix(0, 1), + Some((0, 3)) + ); + assert_eq!( + agent.return_label_codec.label_range_for_prefix(1, 1), + Some((4, 7)) + ); + } + + #[test] + fn return_bin_for_gamma_less_than_one_matches_paper_h_step_return() { + let mut cfg = basic_config(); + cfg.discount_gamma = 0.5; + cfg.return_bins = 8; + + let mut agent = AiqiAgent::new(cfg).expect("valid aiqi config"); + agent + .observe_transition(0, &[0], 1) + .expect("first transition stored"); + agent + .observe_transition(0, &[0], 0) + .expect("second transition should produce first return"); + + let bin = agent.return_bins_by_step[0].expect("first return should be available"); + // Paper target: R_{t,H} = (1-gamma) * sum_{k=0}^{H-1} gamma^k r_{t+k}. + // For rewards [1, 0], gamma=0.5, H=2 this equals 0.5. + // With M=8 bins this maps to floor(8 * 0.5) = 4. + assert_eq!(bin, 4); + } + + #[test] + fn optional_history_pruning_bounds_retained_state_without_losing_progress() { + let mut cfg = basic_config(); + cfg.return_horizon = 3; + cfg.augmentation_period = 4; + cfg.history_prune_keep_steps = Some(8); + + let mut agent = AiqiAgent::new(cfg).expect("valid aiqi config"); + for i in 0..256usize { + let action = (i % 2) as u64; + let obs = [(i % 2) as u64]; + let rew = (i % 2) as i64; + agent + .observe_transition(action, &obs, rew) + .expect("transition should be accepted"); + } + + // Global progress should be preserved even when retained history is bounded. + assert_eq!(agent.steps_observed(), 256); + assert!( + agent.history_base_step > 1, + "history should have been pruned" + ); + assert!( + agent.steps.len() < agent.steps_observed(), + "retained history should be smaller than total observed" + ); + + let action = agent.get_planned_action(); + assert!(action <= 1); + } + + #[test] + fn committed_phase_advancement_uses_commit_predictor_paths() { + let mut agent = AiqiAgent::new(basic_config()).expect("valid aiqi config"); + let counts = Arc::new(Mutex::new(SharedCallCounts::default())); + agent.phases[1].predictor = Box::new(SharedCountingPredictor::new(counts.clone())); + agent.phases[1].last_augmented_step = 0; + agent.history_base_step = 1; + agent.total_steps_observed = 1; + agent.steps = vec![StepRecord { + action: 1, + observations: vec![1], + reward: 1, + }]; + agent.return_bins_by_step = vec![Some(3)]; + + agent + .advance_phase_model_to_step(1, 1) + .expect("phase advancement should succeed"); + + let snapshot = counts.lock().unwrap().clone(); + assert_eq!(snapshot.commit_update, 3); + assert_eq!(snapshot.commit_update_history, 3); + assert_eq!(snapshot.update, 0); + assert_eq!(snapshot.update_history, 0); + } + + #[test] + fn generic_planner_trains_on_returns_and_freezes_conditioning_tokens() { + let mut cfg = basic_config(); + cfg.rate_backend = RateBackend::Match { + hash_bits: 16, + min_len: 2, + max_len: 16, + base_mix: 0.05, + confidence_scale: 1.0, + }; + + let mut agent = AiqiAgent::new(cfg).expect("valid aiqi config"); + let counts = Arc::new(Mutex::new(SharedCallCounts::default())); + agent.phases[1].predictor = Box::new(SharedCountingPredictor::new(counts.clone())); + agent.phases[1].last_augmented_step = 0; + agent.history_base_step = 1; + agent.total_steps_observed = 2; + agent.steps = vec![ + StepRecord { + action: 1, + observations: vec![1], + reward: 1, + }, + StepRecord { + action: 0, + observations: vec![0], + reward: 0, + }, + ]; + agent.return_bins_by_step = vec![Some(3), None]; + + let q_values = agent.estimate_q_values_generic(); + + assert_eq!(q_values.len(), agent.config.agent_actions.get()); + let snapshot = counts.lock().unwrap().clone(); + assert!( + snapshot.update > 0, + "generic planner should train on hypothetical return-prefix symbols" + ); + assert_eq!(snapshot.update_history, 0); + assert!( + snapshot.commit_update > 0, + "generic planner should train on committed augmented return symbols" + ); + assert!( + snapshot.commit_update_history > 0, + "generic planner should keep action/percept conditioning frozen" + ); + } + + #[test] + fn native_reversible_mixture_uses_reversible_aiqi_planner() { + let config = native_reversible_mixture_config(); + let agent = AiqiAgent::new(config).expect("native reversible mixture should build"); + assert!( + !agent.use_generic_planner, + "mixtures composed of native reversible bit predictors should keep the reversible planner" + ); + assert_eq!( + agent.return_label_codec.order(), + ReturnLabelBitOrder::MsbFirst + ); + } + + #[test] + fn compiled_aiqi_runtime_matches_legacy_config_for_generic_mixture_backend() { + let config = generic_mixture_config(); + let compiled = config + .compile_planner_run_spec() + .expect("generic planner run should compile"); + let mut legacy = AiqiAgent::new(config).expect("legacy aiqi config"); + let mut canonical = + AiqiAgent::from_compiled_planner_run(&compiled).expect("compiled aiqi config"); + + let legacy_trace = run_ctw_trace(&mut legacy, 32); + let canonical_trace = run_ctw_trace(&mut canonical, 32); + assert_eq!(canonical_trace, legacy_trace); + } +} + +#[cfg(all(test, feature = "backend-ctw"))] +mod signed_reward_contract_tests { + use super::*; + + fn action_alphabet(n: usize) -> ActionAlphabet { + ActionAlphabet::try_from_usize(n).expect("test action alphabet must be non-zero") + } + + #[test] + fn programmatic_aiqi_preserves_explicit_signed_reward_contract_under_ctw() { + let config = AiqiConfig { + rate_backend: RateBackend::Ctw { depth: 8 }, + bit_stream_semantics: crate::api::BitStreamSemantics::BinaryTokens, + observation_bits: 1, + observation_stream_len: 1, + reward_bits: 3, + agent_actions: action_alphabet(2), + min_reward: -2, + max_reward: 3, + reward_offset: 2, + discount_gamma: 0.99, + return_horizon: 2, + return_bins: 8, + augmentation_period: 2, + history_prune_keep_steps: None, + baseline_exploration: 0.01, + random_seed: Some(7), + }; + + let mut agent = AiqiAgent::new(config).expect("signed reward config should be valid"); + assert_eq!(agent.config.min_reward, -2); + assert_eq!(agent.config.max_reward, 3); + assert_eq!(agent.config.reward_offset, 2); + + agent + .observe_transition(0, &[0], -2) + .expect("signed reward within explicit config bounds should be accepted"); + } +} diff --git a/crates/infotheory/src/aixi/common.rs b/crates/infotheory/src/aixi/common.rs new file mode 100644 index 00000000..4cf25e1b --- /dev/null +++ b/crates/infotheory/src/aixi/common.rs @@ -0,0 +1,774 @@ +//! Common types and utilities for the AIXI implementation. + +use std::error::Error; +use std::fmt; +use std::num::NonZeroUsize; +#[cfg(test)] +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::{AtomicBool, Ordering}; + +/// Represents a single bit (0 or 1) in the agent's interaction history. +pub type Symbol = bool; + +/// A list of symbols, used to represent encoded observations, rewards, or actions. +pub type SymbolList = Vec; + +/// Cardinality of an agent/environment action alphabet `|A|`. +/// +/// This domain type makes the core AIXI/AIQI invariant explicit: planners, +/// environments, and simulator shims operate over a non-empty action alphabet. +/// The value `0` is therefore unrepresentable once a value has crossed an API +/// boundary into the validated internal domain. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub struct ActionAlphabet(NonZeroUsize); + +impl ActionAlphabet { + /// Construct an action alphabet cardinality from a known non-zero value. + pub const fn new(value: NonZeroUsize) -> Self { + Self(value) + } + + /// Validate and construct an action alphabet cardinality from a raw `usize`. + pub fn try_from_usize(value: usize) -> Result { + NonZeroUsize::new(value) + .map(Self) + .ok_or(ZeroActionAlphabetError) + } + + /// Return the underlying cardinality. + pub const fn get(self) -> usize { + self.0.get() + } + + /// Return the minimum bit width required to encode the action alphabet. + pub fn action_bits(self) -> usize { + bits_for_cardinality(self.get()) + } +} + +impl TryFrom for ActionAlphabet { + type Error = ZeroActionAlphabetError; + + fn try_from(value: usize) -> Result { + Self::try_from_usize(value) + } +} + +impl fmt::Display for ActionAlphabet { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.get()) + } +} + +/// Error returned when attempting to construct an empty action alphabet. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct ZeroActionAlphabetError; + +impl fmt::Display for ZeroActionAlphabetError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("action alphabet cardinality must be >= 1") + } +} + +impl Error for ZeroActionAlphabetError {} + +/// Represents an action that the agent can perform. +pub type Action = u64; + +/// Represents a reward received by the agent from the environment. +pub type Reward = i64; + +/// Shared default seed for deterministic AIXI/AIQI planner-runtime behavior. +pub const DEFAULT_RANDOM_SEED: u64 = 0; + +/// Salt used to derive exploration RNG streams from the planner seed. +pub const EXPLORE_RANDOM_SALT: u64 = 0x4558_504c_4f52_455f; + +/// Resolve an optional planner/runtime seed to the canonical deterministic seed. +#[inline] +pub fn resolve_random_seed(seed: Option) -> u64 { + seed.unwrap_or(DEFAULT_RANDOM_SEED) +} + +/// A generic value for a percept component (either an observation or a reward). +pub type PerceptVal = u64; + +/// Strategy for mapping an observation stream into a single percept key for tree search. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub enum ObservationKeyMode { + /// Use the full observation stream as the key (paper-accurate expectimax). + FullStream, + /// Use the first observation symbol as the key. + First, + /// Use the last observation symbol as the key. + Last, + /// Hash the entire observation stream into a single key. + StreamHash, +} + +/// Explicit MC-AIXI Monte Carlo Tree Search strategy. +/// +/// `rho_uct` is the default sequential planner described in +/// "A Monte-Carlo AIXI Approximation". `parallel_uct` enables an explicit +/// parallel planner family whose exact behavior is controlled by the worker +/// count and optional BU-UCT threshold. +/// +/// The `#[non_exhaustive]` attribute reserves room for additional strategy +/// variants (e.g. the supplementary BU-UCT scheduler) without breaking +/// downstream `match` arms. +#[derive(Clone, Copy, Debug, Default, PartialEq)] +#[non_exhaustive] +pub enum MctsStrategy { + /// Sequential \rhoUCT / UCT planning. + #[default] + RhoUct, + /// Explicit parallel UCT planning. + ParallelUct { + /// Number of logical rollout workers (type-enforced non-zero). + workers: NonZeroUsize, + /// Optional BU-UCT threshold parameter `m_max`. + /// + /// `None` selects WU-UCT behavior. `Some(x)` enables BU-UCT + /// thresholding with `x in (0, 1)`. + bu_uct_m_max: Option, + }, +} + +impl MctsStrategy { + /// Stable canonical document/API kind string for this planner strategy. + pub const fn kind_str(self) -> &'static str { + match self { + Self::RhoUct => "rho_uct", + Self::ParallelUct { .. } => "parallel_uct", + } + } +} + +static PARALLEL_UCT_WORKERS_ONE_WARNING_EMITTED: AtomicBool = AtomicBool::new(false); +#[cfg(test)] +static PARALLEL_UCT_WORKERS_ONE_WARNING_COUNT: AtomicUsize = AtomicUsize::new(0); + +/// Emit the `parallel_uct(workers = 1)` warning at most once per process. +pub(crate) fn warn_parallel_uct_workers_one_once() -> bool { + let emitted = PARALLEL_UCT_WORKERS_ONE_WARNING_EMITTED + .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst) + .is_ok(); + if emitted { + #[cfg(test)] + PARALLEL_UCT_WORKERS_ONE_WARNING_COUNT.fetch_add(1, Ordering::SeqCst); + eprintln!( + "Warning: MC-AIXI parallel_uct configured with workers=1; prefer workers >= 2 or switch to rho_uct." + ); + } + emitted +} + +#[cfg(test)] +pub(crate) fn reset_parallel_uct_workers_one_warning_for_tests() { + PARALLEL_UCT_WORKERS_ONE_WARNING_EMITTED.store(false, Ordering::SeqCst); + PARALLEL_UCT_WORKERS_ONE_WARNING_COUNT.store(0, Ordering::SeqCst); +} + +#[cfg(test)] +pub(crate) fn parallel_uct_workers_one_warning_count_for_tests() -> usize { + PARALLEL_UCT_WORKERS_ONE_WARNING_COUNT.load(Ordering::SeqCst) +} + +/// Compute the minimum number of bits required to encode a finite cardinality. +pub(crate) fn bits_for_cardinality(cardinality: usize) -> usize { + let n = cardinality.max(1); + let bits = (usize::BITS - (n - 1).leading_zeros()) as usize; + bits.max(1) +} + +/// Return the total bit width of one percept segment in byte-packed planners. +/// +/// Observation symbols and the reward token share one contiguous conditioning +/// segment in the AIXI/AIQI token pipeline, so byte-packed validation must +/// align their combined width rather than each field independently. +pub(crate) fn byte_packed_percept_bits( + observation_bits: usize, + observation_stream_len: usize, + reward_bits: usize, +) -> usize { + observation_bits + .saturating_mul(observation_stream_len.max(1)) + .saturating_add(reward_bits) +} + +/// User-facing error when [`crate::api::BitStreamSemantics::BytePacked`] segments are +/// not byte-aligned for MC-AIXI planners. +pub(crate) const MC_AIXI_BYTE_PACKED_ALIGNMENT_MSG: &str = "BitStreamSemantics::BytePacked requires action and percept segments to end on byte boundaries; use BitStreamSemantics::BinaryTokens for arbitrary bit-width AIXI interfaces"; + +/// User-facing error when [`crate::api::BitStreamSemantics::BytePacked`] segments are +/// not byte-aligned for AIQI planners. +pub(crate) const AIQI_BYTE_PACKED_ALIGNMENT_MSG: &str = "BitStreamSemantics::BytePacked requires action, return, and percept segments to end on byte boundaries; the percept segment combines observations and reward, so use BitStreamSemantics::BinaryTokens for arbitrary bit-width AIQI interfaces"; + +/// Validate byte alignment required under `BitStreamSemantics::BytePacked` for MC-AIXI. +pub(crate) fn validate_mc_aixi_byte_packed_alignment( + action_bits: usize, + percept_bits: usize, +) -> Result<(), &'static str> { + if action_bits.is_multiple_of(8) && percept_bits.is_multiple_of(8) { + Ok(()) + } else { + Err(MC_AIXI_BYTE_PACKED_ALIGNMENT_MSG) + } +} + +/// Validate byte alignment required under `BitStreamSemantics::BytePacked` for AIQI. +pub(crate) fn validate_aiqi_byte_packed_alignment( + action_bits: usize, + percept_bits: usize, + return_bits: usize, +) -> Result<(), &'static str> { + if action_bits.is_multiple_of(8) + && return_bits.is_multiple_of(8) + && percept_bits.is_multiple_of(8) + { + Ok(()) + } else { + Err(AIQI_BYTE_PACKED_ALIGNMENT_MSG) + } +} + +#[cfg(feature = "aixi")] +pub(crate) fn action_alphabet_from_action_bits(action_bits: usize) -> ActionAlphabet { + let shift = u32::try_from(action_bits).expect("action bit width must fit within u32"); + let cardinality = 1usize + .checked_shl(shift) + .expect("environment action bit width must be < usize::BITS"); + ActionAlphabet::try_from_usize(cardinality) + .expect("1 << action_bits must always produce a non-zero action alphabet") +} + +/// Error returned when the configured reward range cannot be encoded. +#[cfg(feature = "aixi")] +#[derive(Clone, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub enum RewardEncodingError { + /// The configured maximum reward is below the configured minimum reward. + MaxBelowMin { + /// Configured maximum reward. + max_reward: i64, + /// Configured minimum reward. + min_reward: i64, + }, + /// The configured reward offset makes the minimum encoded reward negative. + NegativeShiftedMinimum { + /// Shifted minimum reward (`min_reward + reward_offset`). + shifted_minimum: i128, + }, + /// The configured reward bit width cannot represent the shifted maximum reward. + RewardBitsTooSmall { + /// Shifted maximum reward (`max_reward + reward_offset`). + shifted_maximum: i128, + /// Configured reward bit width. + reward_bits: usize, + /// Maximum representable encoded reward for `reward_bits`. + maximum_encoded: u128, + }, +} + +#[cfg(feature = "aixi")] +impl fmt::Display for RewardEncodingError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::MaxBelowMin { + max_reward, + min_reward, + } => write!( + f, + "max_reward must be >= min_reward (got {max_reward} < {min_reward})" + ), + Self::NegativeShiftedMinimum { shifted_minimum } => write!( + f, + "reward_offset too small: min_reward + reward_offset must be >= 0 (got {shifted_minimum})" + ), + Self::RewardBitsTooSmall { + shifted_maximum, + reward_bits: _, + maximum_encoded, + } => write!( + f, + "reward_bits too small for configured reward range: max shifted reward {shifted_maximum} exceeds {maximum_encoded}" + ), + } + } +} + +#[cfg(feature = "aixi")] +impl Error for RewardEncodingError {} + +/// Validate that shifted rewards are representable in the configured bit width. +#[cfg(feature = "aixi")] +pub(crate) fn validate_reward_encoding_bounds( + min_reward: i64, + max_reward: i64, + reward_offset: i64, + reward_bits: usize, +) -> Result<(), RewardEncodingError> { + if max_reward < min_reward { + return Err(RewardEncodingError::MaxBelowMin { + max_reward, + min_reward, + }); + } + + let min_shifted = (min_reward as i128) + (reward_offset as i128); + let max_shifted = (max_reward as i128) + (reward_offset as i128); + if min_shifted < 0 { + return Err(RewardEncodingError::NegativeShiftedMinimum { + shifted_minimum: min_shifted, + }); + } + if reward_bits < 64 { + let max_enc = (1u128 << reward_bits) - 1; + if (max_shifted as u128) > max_enc { + return Err(RewardEncodingError::RewardBitsTooSmall { + shifted_maximum: max_shifted, + reward_bits, + maximum_encoded: max_enc, + }); + } + } + + Ok(()) +} + +/// Maximum nonnegative instantaneous reward representable in `reward_bits` channel bits. +/// +/// For `reward_bits >= 63` this returns [`i64::MAX`] (closed interval policy shared with +/// [`validate_reward_encoding_bounds`] for wide channels). +/// +/// Callers that require a strict positive width must use [`max_nonnegative_reward_for_bits`]. +fn max_channel_reward_for_bits(reward_bits: usize) -> Reward { + if reward_bits >= 63 { + i64::MAX + } else { + ((1u64 << reward_bits) - 1) as i64 + } +} + +/// Maximum nonnegative instantaneous reward for a validated planner reward channel. +/// +/// Returns an error when `reward_bits == 0` (degenerate channel); planner specs require +/// `reward_bits >= 1` after canonicalization. +/// +/// Used by the `tuner` feature (reward certificates); default builds omit callers. +#[cfg_attr(not(feature = "tuner"), allow(dead_code))] +pub(crate) fn max_nonnegative_reward_for_bits(reward_bits: usize) -> Result { + if reward_bits == 0 { + return Err("reward_bits must be >= 1"); + } + Ok(max_channel_reward_for_bits(reward_bits)) +} + +/// Derive canonical nonnegative reward-encoding bounds from reward bit width. +/// +/// This helper is used by programmatic AIXI/AIQI/warmstart constructors now +/// that planner-interface v1 no longer carries explicit `min/max/offset`. +/// +/// For `reward_bits == 0`, the degenerate channel encodes only the value `0`. +#[cfg(feature = "aixi")] +pub(crate) fn nonnegative_reward_encoding_bounds(reward_bits: usize) -> (Reward, Reward, Reward) { + (0, max_channel_reward_for_bits(reward_bits), 0) +} + +/// Compute a percept key from an observation stream. +pub fn observation_key_from_stream( + mode: ObservationKeyMode, + observations: &[PerceptVal], + observation_bits: usize, +) -> PerceptVal { + match mode { + ObservationKeyMode::FullStream => { + debug_assert!( + false, + "observation_key_from_stream called with FullStream; use observation_repr_from_stream" + ); + // Fallback to hash in release builds to avoid panics. + observation_key_from_stream( + ObservationKeyMode::StreamHash, + observations, + observation_bits, + ) + } + ObservationKeyMode::First => observations.first().copied().unwrap_or(0), + ObservationKeyMode::Last => observations.last().copied().unwrap_or(0), + ObservationKeyMode::StreamHash => { + let mask = if observation_bits >= 64 { + u64::MAX + } else if observation_bits == 0 { + 0 + } else { + (1u64 << observation_bits) - 1 + }; + let mut h = 0u64; + for &obs in observations { + let v = obs & mask; + h = h.rotate_left(7) ^ v; + } + h + } + } +} + +/// Compute the observation representation used for tree branching. +/// +/// - `FullStream` returns the full stream (paper-accurate expectimax). +/// - Other modes collapse to a single-key vector. +pub fn observation_repr_from_stream( + mode: ObservationKeyMode, + observations: &[PerceptVal], + observation_bits: usize, +) -> Vec { + match mode { + ObservationKeyMode::FullStream => observations.to_vec(), + _ => vec![observation_key_from_stream( + mode, + observations, + observation_bits, + )], + } +} + +/// A high-performance random number generator using the XorShift64* algorithm. +#[derive(Clone, Copy)] +pub struct RandomGenerator { + state: u64, +} + +impl RandomGenerator { + #[inline] + fn initial_seed() -> u64 { + #[cfg(feature = "backend-zpaq")] + { + if let Ok(bytes) = zpaq_rs::random_bytes(8) { + let mut seed_arr = [0u8; 8]; + seed_arr.copy_from_slice(&bytes); + return u64::from_le_bytes(seed_arr); + } + } + + #[cfg(target_arch = "wasm32")] + { + // `SystemTime::now()` is unavailable on `wasm32-unknown-unknown` without WASI. + return 0xCAFEBABEDEADBEEF ^ 0x9E3779B97F4A7C15; + } + + #[cfg(not(target_arch = "wasm32"))] + #[allow(clippy::cast_possible_truncation)] + { + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_nanos() as u64) + .unwrap_or(0xCAFEBABEDEADBEEF); + return nanos ^ 0x9E3779B97F4A7C15; + } + + #[allow(unreachable_code)] + 0xCAFEBABEDEADBEEF + } + + /// Creates a new `RandomGenerator` with the canonical deterministic seed. + pub fn new() -> Self { + Self::from_seed(DEFAULT_RANDOM_SEED) + } + + /// Creates a new `RandomGenerator` from runtime entropy. + /// + /// This is an explicit opt-in escape hatch for callers that need + /// non-deterministic sampling. + pub fn from_entropy() -> Self { + Self::from_seed(Self::initial_seed()) + } + + /// Creates a new `RandomGenerator` from an explicit seed. + /// + /// A zero seed is remapped to a fixed non-zero constant to avoid the + /// xorshift zero-state trap. + pub fn from_seed(seed: u64) -> Self { + let state = if seed == 0 { 0xCAFEBABEDEADBEEF } else { seed }; + Self { state } + } + + /// Generates the next pseudo-random `u64`. + pub fn next_u64(&mut self) -> u64 { + // xorshift64* + let mut x = self.state; + x ^= x >> 12; + x ^= x << 25; + x ^= x >> 27; + self.state = x; + x.wrapping_mul(0x2545F4914F6CDD1D) + } + + /// Generates a pseudo-random `usize` in the range `[0, end)`. + pub fn gen_range(&mut self, end: usize) -> usize { + if end == 0 { + return 0; + } + (self.next_u64() % (end as u64)) as usize + } + + /// Generates a boolean value with probability `p` of being `true`. + pub fn gen_bool(&mut self, p: f64) -> bool { + self.gen_f64() < p + } + + /// Generates a pseudo-random `f64` in the range `[0, 1)`. + pub fn gen_f64(&mut self) -> f64 { + // 53 bits + let v = self.next_u64() >> 11; + (v as f64) * (1.0 / 9007199254740992.0) + } + + /// Forks the RNG state with a salt, returning an independent generator. + pub fn fork_with(&self, salt: u64) -> Self { + let mixed = Self::splitmix64(self.state ^ salt ^ 0x9E3779B97F4A7C15); + let state = if mixed == 0 { + 0xCAFEBABEDEADBEEF + } else { + mixed + }; + Self { state } + } + + fn splitmix64(mut x: u64) -> u64 { + x = x.wrapping_add(0x9E3779B97F4A7C15); + let mut z = x; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58476D1CE4E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D049BB133111EB); + z ^ (z >> 31) + } +} + +impl Default for RandomGenerator { + fn default() -> Self { + Self::new() + } +} + +/// Encodes a numeric value into its bit representation and appends it to a `SymbolList`. +/// +/// Bits are appended in least-significant-bit first order. +pub fn encode(symlist: &mut SymbolList, value: u64, bits: usize) { + let mut v = value; + for _ in 0..bits { + symlist.push((v & 1) == 1); + v >>= 1; + } +} + +/// Encodes a signed reward value into its bit representation. +pub fn encode_reward(symlist: &mut SymbolList, value: i64, bits: usize) { + let mut v = value as u64; + for _ in 0..bits { + symlist.push((v & 1) == 1); + v >>= 1; + } +} + +/// Encodes a reward after applying an additive `offset`. +pub fn encode_reward_offset(symlist: &mut SymbolList, value: i64, bits: usize, offset: i64) { + let shifted = (value + offset) as u64; + encode(symlist, shifted, bits); +} + +/// Decodes a numeric value from its bit representation. +pub fn decode(symlist: &[Symbol], bits: usize) -> u64 { + if bits == 0 { + return 0; + } + assert!(bits <= symlist.len()); + let mut value = 0u64; + for i in 0..bits { + let sym = symlist[symlist.len() - 1 - i]; + value = (value << 1) + (if sym { 1 } else { 0 }); + } + value +} + +/// Decodes a signed reward value from its bit representation. +pub fn decode_reward(symlist: &[Symbol], bits: usize) -> i64 { + if bits == 0 { + return 0; + } + let v = decode(symlist, bits); + if bits < 64 && (v & (1 << (bits - 1))) != 0 { + // Sign bit set, perform two's complement sign extension + (v | (!0u64 << bits)) as i64 + } else { + v as i64 + } +} + +/// Decodes a reward encoded with [`encode_reward_offset`]. +pub fn decode_reward_offset(symlist: &[Symbol], bits: usize, offset: i64) -> i64 { + if bits == 0 { + return 0; + } + let v = decode(symlist, bits) as i64; + v - offset +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn observation_repr_full_stream_is_identity() { + let obs = vec![1u64, 2u64, 3u64]; + let repr = observation_repr_from_stream(ObservationKeyMode::FullStream, &obs, 8); + assert_eq!(repr, obs); + } + + #[test] + fn observation_key_first_last() { + let obs = vec![10u64, 20u64, 30u64]; + assert_eq!( + observation_key_from_stream(ObservationKeyMode::First, &obs, 8), + 10 + ); + assert_eq!( + observation_key_from_stream(ObservationKeyMode::Last, &obs, 8), + 30 + ); + + let empty: Vec = vec![]; + assert_eq!( + observation_key_from_stream(ObservationKeyMode::First, &empty, 8), + 0 + ); + assert_eq!( + observation_key_from_stream(ObservationKeyMode::Last, &empty, 8), + 0 + ); + } + + #[test] + fn observation_key_stream_hash_masks_and_mix() { + // observation_bits=3 => mask=0b111 + // obs[0]=9 -> 1; h=0.rotate_left(7)^1 = 1 + // obs[1]=2 -> 2; h=1.rotate_left(7)^2 = 128^2 = 130 + let obs = vec![9u64, 2u64]; + let h = observation_key_from_stream(ObservationKeyMode::StreamHash, &obs, 3); + assert_eq!(h, 130); + } + + #[test] + fn observation_key_stream_hash_observation_bits_zero_is_zero() { + let obs = vec![123u64, 456u64, 789u64]; + let h = observation_key_from_stream(ObservationKeyMode::StreamHash, &obs, 0); + assert_eq!(h, 0); + } + + #[test] + fn observation_key_stream_hash_observation_bits_ge_64_uses_full_u64() { + let obs = vec![u64::MAX, 0x0123_4567_89ab_cdef]; + let h1 = observation_key_from_stream(ObservationKeyMode::StreamHash, &obs, 64); + let h2 = observation_key_from_stream(ObservationKeyMode::StreamHash, &obs, 128); + assert_eq!(h1, h2); + } + + #[test] + fn bits_for_cardinality_covers_extreme_sizes_without_overflow() { + assert_eq!(bits_for_cardinality(0), 1); + assert_eq!(bits_for_cardinality(1), 1); + assert_eq!(bits_for_cardinality(2), 1); + assert_eq!(bits_for_cardinality(3), 2); + assert_eq!(bits_for_cardinality(usize::MAX), usize::BITS as usize); + } + + #[test] + fn action_alphabet_rejects_zero_cardinality() { + assert_eq!( + ActionAlphabet::try_from_usize(0).expect_err("zero actions must be rejected"), + ZeroActionAlphabetError + ); + } + + #[test] + #[cfg(feature = "aixi")] + fn validate_reward_encoding_bounds_rejects_unrepresentable_ranges() { + let err = validate_reward_encoding_bounds(0, 100, 0, 1).expect_err("must fail"); + assert!(matches!( + err, + RewardEncodingError::RewardBitsTooSmall { .. } + )); + } + + #[test] + fn max_nonnegative_reward_for_bits_rejects_zero_width() { + assert_eq!( + max_nonnegative_reward_for_bits(0).expect_err("zero bits"), + "reward_bits must be >= 1" + ); + } + + #[test] + #[cfg(feature = "aixi")] + fn max_nonnegative_reward_for_bits_agrees_with_nonnegative_reward_encoding_bounds() { + for reward_bits in [1usize, 8, 62, 63] { + let max_ch = max_nonnegative_reward_for_bits(reward_bits).expect("valid bits"); + let (_min, max_r, _off) = nonnegative_reward_encoding_bounds(reward_bits); + assert_eq!(max_ch, max_r, "reward_bits={reward_bits}"); + } + let wide = max_nonnegative_reward_for_bits(64).expect("wide channel"); + assert_eq!(wide, i64::MAX); + assert_eq!(nonnegative_reward_encoding_bounds(64).1, i64::MAX); + } + + #[test] + fn max_nonnegative_reward_for_bits_boundary_powers() { + assert_eq!(max_nonnegative_reward_for_bits(1).unwrap(), 1); + assert_eq!(max_nonnegative_reward_for_bits(2).unwrap(), 3); + assert_eq!( + max_nonnegative_reward_for_bits(62).unwrap(), + (1i64 << 62) - 1 + ); + assert_eq!(max_nonnegative_reward_for_bits(63).unwrap(), i64::MAX); + } + + #[test] + fn random_generator_default_is_deterministic_seed_zero() { + let mut via_new = RandomGenerator::new(); + let mut via_seed = RandomGenerator::from_seed(DEFAULT_RANDOM_SEED); + for _ in 0..32 { + assert_eq!(via_new.next_u64(), via_seed.next_u64()); + } + } + + #[test] + fn parallel_uct_workers_one_warning_emits_once() { + reset_parallel_uct_workers_one_warning_for_tests(); + assert!(warn_parallel_uct_workers_one_once()); + assert!(!warn_parallel_uct_workers_one_once()); + assert_eq!(parallel_uct_workers_one_warning_count_for_tests(), 1); + } + + #[test] + fn random_generator_fork_with_is_stable() { + let base = RandomGenerator::from_seed(17); + let mut a = base.fork_with(99); + let mut b = base.fork_with(99); + for _ in 0..32 { + assert_eq!(a.next_u64(), b.next_u64()); + } + } + + #[test] + fn random_generator_entropy_is_explicit_opt_in() { + let mut deterministic = RandomGenerator::new(); + let mut explicit = RandomGenerator::from_seed(DEFAULT_RANDOM_SEED); + for _ in 0..16 { + assert_eq!(deterministic.next_u64(), explicit.next_u64()); + } + + // Entropy path should be callable explicitly and produce a valid stream. + let mut entropy_rng = RandomGenerator::from_entropy(); + let _ = entropy_rng.next_u64(); + } +} diff --git a/crates/infotheory/src/aixi/environment.rs b/crates/infotheory/src/aixi/environment.rs new file mode 100644 index 00000000..58c9157f --- /dev/null +++ b/crates/infotheory/src/aixi/environment.rs @@ -0,0 +1,161 @@ +//! Environment contract for AIXI/AIQI planners. +//! +//! Core AIXI is intentionally environment-agnostic. Concrete environments are +//! provided through optional integrations (for example `aixi-gameengine` and +//! `aixi-vm`) or by user-defined implementations. + +use crate::aixi::common::{ + Action, ActionAlphabet, PerceptVal, Reward, action_alphabet_from_action_bits, +}; + +/// Interface for an agent's environment. +pub trait Environment { + /// Executes an action in the environment and updates internal state. + fn perform_action(&mut self, action: Action); + + /// Returns the current observation produced by the environment. + fn get_observation(&self) -> PerceptVal; + + /// Returns the observation stream emitted by the last action. + /// + /// Default behavior is a single-symbol stream. + fn drain_observations(&mut self) -> Vec { + vec![self.get_observation()] + } + + /// Returns the current reward produced by the environment. + fn get_reward(&self) -> Reward; + + /// Returns true if the environment has reached a terminal state. + fn is_finished(&self) -> bool; + + /// Returns the number of bits used to encode observations. + fn get_observation_bits(&self) -> usize; + + /// Returns the number of bits used to encode rewards. + fn get_reward_bits(&self) -> usize; + + /// Returns the number of bits required to represent all valid actions. + fn get_action_bits(&self) -> usize; + + /// Reseeds stochastic state for deterministic runs. + fn set_random_seed(&mut self, _seed: u64) {} + + /// Returns the total number of valid actions available. + fn get_num_actions(&self) -> ActionAlphabet { + action_alphabet_from_action_bits(self.get_action_bits()) + } + + /// Returns the maximum possible reward value in this environment. + fn max_reward(&self) -> Reward { + let bits: usize = self.get_reward_bits(); + if bits == 0 { + return 0; + } + if bits >= 64 { + i64::MAX + } else { + (1i64 << (bits - 1)) - 1 + } + } + + /// Returns the minimum possible reward value in this environment. + fn min_reward(&self) -> Reward { + let bits: usize = self.get_reward_bits(); + if bits == 0 { + return 0; + } + if bits >= 64 { + i64::MIN + } else { + -(1i64 << (bits - 1)) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[derive(Clone, Copy)] + struct DummyEnv { + observation: PerceptVal, + reward: Reward, + observation_bits: usize, + reward_bits: usize, + action_bits: usize, + finished: bool, + } + + impl Environment for DummyEnv { + fn perform_action(&mut self, _action: Action) {} + + fn get_observation(&self) -> PerceptVal { + self.observation + } + + fn get_reward(&self) -> Reward { + self.reward + } + + fn is_finished(&self) -> bool { + self.finished + } + + fn get_observation_bits(&self) -> usize { + self.observation_bits + } + + fn get_reward_bits(&self) -> usize { + self.reward_bits + } + + fn get_action_bits(&self) -> usize { + self.action_bits + } + } + + #[test] + fn default_environment_helpers_are_consistent() { + let mut env = DummyEnv { + observation: 7, + reward: -2, + observation_bits: 3, + reward_bits: 4, + action_bits: 2, + finished: false, + }; + + env.set_random_seed(1234); + env.perform_action(1); + + assert_eq!(env.drain_observations(), vec![7]); + assert_eq!(env.get_num_actions().get(), 4); + assert_eq!(env.max_reward(), 7); + assert_eq!(env.min_reward(), -8); + assert_eq!(env.get_reward(), -2); + assert!(!env.is_finished()); + assert_eq!(env.get_observation_bits(), 3); + } + + #[test] + fn reward_bound_helpers_cover_zero_and_wide_bit_ranges() { + let zero_bits = DummyEnv { + observation: 0, + reward: 0, + observation_bits: 1, + reward_bits: 0, + action_bits: 1, + finished: false, + }; + assert_eq!(zero_bits.min_reward(), 0); + assert_eq!(zero_bits.max_reward(), 0); + + let wide_bits = DummyEnv { + reward_bits: 64, + ..zero_bits + }; + assert_eq!(wide_bits.min_reward(), i64::MIN); + assert_eq!(wide_bits.max_reward(), i64::MAX); + } +} diff --git a/crates/infotheory/src/aixi/gameengine.rs b/crates/infotheory/src/aixi/gameengine.rs new file mode 100644 index 00000000..f8c109a1 --- /dev/null +++ b/crates/infotheory/src/aixi/gameengine.rs @@ -0,0 +1,411 @@ +//! GameEngine integration for AIXI environments. +//! +//! This adapter is behind the `aixi-gameengine` feature and keeps core AIXI +//! independent from bundled environments. + +use crate::aixi::common::DEFAULT_RANDOM_SEED; +use crate::aixi::common::{Action, ActionAlphabet, PerceptVal, Reward}; +use crate::aixi::environment::Environment; +use crate::spec::BuiltinEnvironmentSpec; +use gameengine::GameAuthoring; +#[cfg(feature = "aixi-gameengine-physics")] +use gameengine::builtin::Platformer; +use gameengine::builtin::{ + BiasedCoinFlip, BiasedCoinFlipConfig, BiasedRockPaperScissor, Blackjack, ExtendedTiger, + KuhnPoker, TicTacToe, +}; +use gameengine::{ActionToken, AixiEnvironment as GameEngineAixiEnvironment, DefaultEnvironment}; + +/// Errors surfaced by GameEngine-backed AIXI environment construction/runtime reset. +#[derive(Debug)] +#[non_exhaustive] +pub enum GameEngineEnvironmentError { + /// Underlying GameEngine reset failed. + ResetFailed(String), + /// Environment produced an observation stream that violates compact spec shape. + PerceptStreamLengthMismatch { + /// Actual observation stream length emitted by the environment. + actual: usize, + /// Expected observation stream length from compact spec. + expected: usize, + }, + /// Coin-flip bias violates the domain invariant. + InvalidCoinFlipBias { + /// Configured coin-flip head numerator. + head_numerator: u64, + /// Configured coin-flip head denominator. + head_denominator: u64, + }, + /// The internal tuner bridge was requested as a standalone runtime environment. + TunerBridgeOnly, + /// Requested builtin requires an optional feature that is disabled. + MissingFeature { + /// Builtin environment that was requested. + builtin: BuiltinEnvironmentSpec, + /// Missing feature gate required for `builtin`. + feature: &'static str, + }, +} + +impl std::fmt::Display for GameEngineEnvironmentError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::ResetFailed(err) => { + write!(f, "failed to reset GameEngine environment: {err}") + } + Self::PerceptStreamLengthMismatch { actual, expected } => write!( + f, + "GameEngine percept stream length {actual} does not match compact spec length {expected}" + ), + Self::InvalidCoinFlipBias { + head_numerator, + head_denominator, + } => write!( + f, + "invalid coin-flip bias: expected 0 <= numerator <= denominator (got {head_numerator}/{head_denominator})" + ), + Self::TunerBridgeOnly => f.write_str( + "builtin environment 'tuner_bridge' is an internal tuner planner bridge and cannot be run as a standalone GameEngine environment", + ), + Self::MissingFeature { builtin, feature } => write!( + f, + "builtin environment '{}' requires feature '{}'", + builtin.canonical_name(), + feature + ), + } + } +} + +impl std::error::Error for GameEngineEnvironmentError {} + +/// Generic adapter from a GameEngine AIXI environment to Infotheory's AIXI trait. +pub struct GameEngineEnvironment +where + E: GameEngineAixiEnvironment, +{ + env: E, + spec: gameengine::CompactSpec, + observation_stream: Vec, + observation: PerceptVal, + reward: Reward, + finished: bool, +} + +impl GameEngineEnvironment +where + E: GameEngineAixiEnvironment, +{ + /// Builds an adapter and resets the underlying environment with `seed`. + pub fn from_environment( + mut env: E, + spec: gameengine::CompactSpec, + seed: u64, + ) -> Result { + let initial = env + .reset_seed(seed) + .map_err(|err| GameEngineEnvironmentError::ResetFailed(err.to_string()))?; + let mut adapter = Self { + env, + spec, + observation_stream: Vec::with_capacity(spec.observation_stream_len), + observation: 0, + reward: 0, + finished: false, + }; + adapter.apply_percept(initial)?; + Ok(adapter) + } + + fn apply_percept( + &mut self, + percept: gameengine::Percept, + ) -> Result<(), GameEngineEnvironmentError> { + let words = percept.observation_bits.words(); + if words.len() != self.spec.observation_stream_len { + return Err(GameEngineEnvironmentError::PerceptStreamLengthMismatch { + actual: words.len(), + expected: self.spec.observation_stream_len, + }); + } + + self.observation_stream.clear(); + self.observation_stream.extend(words.iter().copied()); + self.observation = self.observation_stream.first().copied().unwrap_or(0); + self.reward = percept.reward.raw; + self.finished = percept.terminated; + Ok(()) + } + + fn reset_with_seed(&mut self, seed: u64) -> Result<(), GameEngineEnvironmentError> { + let percept = self + .env + .reset_seed(seed) + .map_err(|err| GameEngineEnvironmentError::ResetFailed(err.to_string()))?; + self.apply_percept(percept) + } +} + +impl Environment for GameEngineEnvironment +where + E: GameEngineAixiEnvironment, +{ + fn perform_action(&mut self, action: Action) { + if self.finished { + return; + } + + let token = match ActionToken::try_new(action, self.spec.action_count) { + Ok(token) => token, + Err(_) => { + self.reward = self.spec.min_reward; + return; + } + }; + + match self.env.step(token) { + Ok(percept) => { + if self.apply_percept(percept).is_err() { + self.finished = true; + self.reward = self.spec.min_reward; + } + } + Err(_) => { + self.finished = true; + self.reward = self.spec.min_reward; + } + } + } + + fn get_observation(&self) -> PerceptVal { + self.observation + } + + fn drain_observations(&mut self) -> Vec { + self.observation_stream.clone() + } + + fn get_reward(&self) -> Reward { + self.reward + } + + fn is_finished(&self) -> bool { + self.finished + } + + fn get_observation_bits(&self) -> usize { + self.spec.observation_bits as usize + } + + fn get_reward_bits(&self) -> usize { + self.spec.reward_bits as usize + } + + fn get_action_bits(&self) -> usize { + self.spec.action_bits() as usize + } + + fn set_random_seed(&mut self, seed: u64) { + if self.reset_with_seed(seed).is_err() { + self.finished = true; + self.reward = self.spec.min_reward; + } + } + + fn get_num_actions(&self) -> ActionAlphabet { + ActionAlphabet::try_from_usize(self.spec.action_count as usize) + .expect("gameengine environments must expose a non-empty action alphabet") + } + + fn max_reward(&self) -> Reward { + self.spec.max_reward + } + + fn min_reward(&self) -> Reward { + self.spec.min_reward + } +} + +type TicTacToeEnvironment = GameEngineEnvironment, 1>; +type BlackjackEnvironment = GameEngineEnvironment, 4>; +type CoinFlipEnvironment = GameEngineEnvironment, 1>; +type BiasedRpsEnvironment = GameEngineEnvironment, 1>; +type KuhnPokerEnvironment = GameEngineEnvironment, 1>; +type ExtendedTigerEnvironment = GameEngineEnvironment, 1>; +#[cfg(feature = "aixi-gameengine-physics")] +type PlatformerEnvironment = GameEngineEnvironment, 1>; + +fn build_coin_flip_environment_from_config( + config: BiasedCoinFlipConfig, + seed: u64, +) -> Result, GameEngineEnvironmentError> { + if !config.invariant() { + return Err(GameEngineEnvironmentError::InvalidCoinFlipBias { + head_numerator: config.head_numerator, + head_denominator: config.head_denominator, + }); + } + let game = BiasedCoinFlip::new(config); + let spec = game.compact_spec(); + let env = DefaultEnvironment::::new_for_agent(game, seed, 0); + Ok(Box::new(CoinFlipEnvironment::from_environment( + env, spec, seed, + )?)) +} + +/// Builds a biased coin-flip environment from Bernoulli head probability parts. +pub fn build_coin_flip_environment( + head_numerator: u64, + head_denominator: u64, + seed: u64, +) -> Result, GameEngineEnvironmentError> { + build_coin_flip_environment_from_config( + BiasedCoinFlipConfig { + head_numerator, + head_denominator, + }, + seed, + ) +} + +/// Builds a boxed AIXI environment from the canonical builtin enum. +pub fn build_builtin_environment( + builtin: BuiltinEnvironmentSpec, +) -> Result, GameEngineEnvironmentError> { + build_builtin_environment_with_seed(builtin, DEFAULT_RANDOM_SEED) +} + +/// Builds a boxed AIXI environment from the canonical builtin enum and explicit seed. +pub fn build_builtin_environment_with_seed( + builtin: BuiltinEnvironmentSpec, + seed: u64, +) -> Result, GameEngineEnvironmentError> { + match builtin { + BuiltinEnvironmentSpec::TunerBridge => Err(GameEngineEnvironmentError::TunerBridgeOnly), + BuiltinEnvironmentSpec::CoinFlip => { + build_coin_flip_environment_from_config(BiasedCoinFlipConfig::default(), seed) + } + BuiltinEnvironmentSpec::BiasedRockPaperScissor => { + let game = BiasedRockPaperScissor; + let spec = game.compact_spec(); + let env = DefaultEnvironment::::new_for_agent(game, seed, 0); + Ok(Box::new(BiasedRpsEnvironment::from_environment( + env, spec, seed, + )?)) + } + BuiltinEnvironmentSpec::KuhnPoker => { + let game = KuhnPoker; + let spec = game.compact_spec(); + let env = DefaultEnvironment::::new_for_agent(game, seed, 0); + Ok(Box::new(KuhnPokerEnvironment::from_environment( + env, spec, seed, + )?)) + } + BuiltinEnvironmentSpec::ExtendedTiger => { + let game = ExtendedTiger; + let spec = game.compact_spec(); + let env = DefaultEnvironment::::new_for_agent(game, seed, 0); + Ok(Box::new(ExtendedTigerEnvironment::from_environment( + env, spec, seed, + )?)) + } + BuiltinEnvironmentSpec::TicTacToe => { + let game = TicTacToe; + let spec = game.compact_spec(); + let env = DefaultEnvironment::::new_for_agent(game, seed, 0); + Ok(Box::new(TicTacToeEnvironment::from_environment( + env, spec, seed, + )?)) + } + BuiltinEnvironmentSpec::Blackjack => { + let game = Blackjack; + let spec = game.compact_spec(); + let env = DefaultEnvironment::::new_for_agent(game, seed, 0); + Ok(Box::new(BlackjackEnvironment::from_environment( + env, spec, seed, + )?)) + } + #[cfg(feature = "aixi-gameengine-physics")] + BuiltinEnvironmentSpec::Platformer => { + let game = Platformer::default(); + let spec = game.compact_spec(); + let env = DefaultEnvironment::::new_for_agent(game, seed, 0); + Ok(Box::new(PlatformerEnvironment::from_environment( + env, spec, seed, + )?)) + } + #[cfg(not(feature = "aixi-gameengine-physics"))] + BuiltinEnvironmentSpec::Platformer => Err(GameEngineEnvironmentError::MissingFeature { + builtin: BuiltinEnvironmentSpec::Platformer, + feature: "aixi-gameengine-physics", + }), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn configurable_coin_flip_bias_controls_rewards_and_reseed() { + let mut heads = build_coin_flip_environment(1, 1, 7).expect("valid heads-only coin flip"); + heads.perform_action(1); + assert_eq!(heads.get_observation(), 1); + assert_eq!(heads.get_reward(), 1); + heads.set_random_seed(99); + heads.perform_action(0); + assert_eq!(heads.get_observation(), 1); + assert_eq!(heads.get_reward(), 0); + + let mut tails = build_coin_flip_environment(0, 1, 7).expect("valid tails-only coin flip"); + tails.perform_action(0); + assert_eq!(tails.get_observation(), 0); + assert_eq!(tails.get_reward(), 1); + } + + #[test] + fn invalid_coin_flip_bias_is_rejected() { + let err = match build_coin_flip_environment(2, 1, 0) { + Ok(_) => panic!("invalid ratio must fail"), + Err(err) => err, + }; + assert!(matches!( + err, + GameEngineEnvironmentError::InvalidCoinFlipBias { + head_numerator: 2, + head_denominator: 1, + } + )); + + let err = match build_coin_flip_environment(0, 0, 0) { + Ok(_) => panic!("zero denominator must fail"), + Err(err) => err, + }; + assert!(matches!( + err, + GameEngineEnvironmentError::InvalidCoinFlipBias { + head_numerator: 0, + head_denominator: 0, + } + )); + } + + #[test] + fn default_builtin_environment_matches_explicit_default_seed() { + let mut implicit = + build_builtin_environment(BuiltinEnvironmentSpec::CoinFlip).expect("builtin env"); + let mut explicit = build_builtin_environment_with_seed(BuiltinEnvironmentSpec::CoinFlip, 0) + .expect("seeded builtin env"); + + let mut implicit_trace = Vec::new(); + let mut explicit_trace = Vec::new(); + for &action in &[0u64, 1, 1, 0, 1, 0, 0, 1] { + implicit.perform_action(action); + explicit.perform_action(action); + implicit_trace.push((implicit.get_observation(), implicit.get_reward())); + explicit_trace.push((explicit.get_observation(), explicit.get_reward())); + } + + assert_eq!(implicit_trace, explicit_trace); + } +} diff --git a/crates/infotheory/src/aixi/mcts.rs b/crates/infotheory/src/aixi/mcts.rs new file mode 100644 index 00000000..1bc767e7 --- /dev/null +++ b/crates/infotheory/src/aixi/mcts.rs @@ -0,0 +1,292 @@ +//! Monte Carlo Tree Search (MCTS) for AIXI. +//! +//! The sequential `rho_uct` planner follows "A Monte-Carlo AIXI +//! Approximation". Parallel planners are explicit and live in a separate +//! backend rather than being inferred from the simulation count. + +mod parallel_uct; +mod rho_uct; + +use crate::aixi::common::{ + Action, ActionAlphabet, ObservationKeyMode, PerceptVal, Reward, observation_repr_from_stream, +}; + +pub use parallel_uct::{ParallelUctPlanner, ParallelUctPlannerInitError, ParallelUctSearchError}; +pub use rho_uct::RhoUctPlanner; + +use std::collections::HashMap; + +/// Hash key for a sampled percept outcome at a chance node. +/// +/// Both the observation representation and the immediate reward are required +/// to identify the correct continuation subtree for generic environments. +/// Some environments can emit the same observation alongside different rewards, +/// so observation-only keys would incorrectly merge distinct successor states +/// during search-tree reuse. +#[derive(Clone, Debug, Eq, PartialEq, Hash)] +pub(crate) struct PerceptOutcome { + /// Observation symbols used for chance-node branching. + observations: Box<[PerceptVal]>, + /// Immediate reward observed on the sampled edge. + reward: Reward, +} + +impl PerceptOutcome { + /// Creates a compact percept key from an observation stream and reward. + pub(crate) fn new(observations: Vec, reward: Reward) -> Self { + Self { + observations: observations.into_boxed_slice(), + reward, + } + } + + pub(crate) fn reward(&self) -> Reward { + self.reward + } +} + +pub(crate) fn prune_key( + agent: &dyn AgentSimulator, + prev_obs_stream: &[PerceptVal], + prev_rew: Reward, +) -> PerceptOutcome { + let obs_repr = agent.observation_repr_from_stream(prev_obs_stream); + PerceptOutcome::new(obs_repr, prev_rew) +} + +/// Interface for an agent that can be simulated during MCTS. +/// +/// This trait allows the MCTS algorithm to interact with an agent +/// (like `Agent` in `agent.rs`) to perform imagined actions and receive +/// imagined percepts during planning. +pub trait AgentSimulator: Send { + /// Returns the number of possible actions the agent can perform. + fn get_num_actions(&self) -> ActionAlphabet; + + /// Returns the bit-width used to encode observations. + fn get_num_observation_bits(&self) -> usize; + + /// Returns the number of observation symbols per action. + fn observation_stream_len(&self) -> usize { + 1 + } + + /// Returns the observation key mode for search-tree branching. + fn observation_key_mode(&self) -> ObservationKeyMode { + ObservationKeyMode::FullStream + } + + /// Returns the observation representation used for tree branching. + fn observation_repr_from_stream(&self, observations: &[PerceptVal]) -> Vec { + observation_repr_from_stream( + self.observation_key_mode(), + observations, + self.get_num_observation_bits(), + ) + } + + /// Returns the bit-width used to encode rewards. + fn get_num_reward_bits(&self) -> usize; + + /// Returns the planning horizon (depth of simulations). + fn horizon(&self) -> usize; + + /// Returns the maximum possible reward value. + fn max_reward(&self) -> Reward; + + /// Returns the minimum possible reward value. + fn min_reward(&self) -> Reward; + + /// Returns the reward offset used to ensure encoded rewards are non-negative. + /// + /// Paper-compatible encoding uses unsigned reward bits and shifts rewards by + /// an offset. + fn reward_offset(&self) -> i64 { + 0 + } + + /// Returns the exploration-exploitation constant. + fn get_explore_exploit_ratio(&self) -> f64 { + 1.0 + } + + /// Returns the discount factor for future rewards. + fn discount_gamma(&self) -> f64 { + 1.0 + } + + /// Updates the internal model state with a simulated action. + fn model_update_action(&mut self, action: Action); + + /// Generates a simulated percept and updates the model state. + fn gen_percept_and_update(&mut self, bits: usize) -> u64; + + /// Marks the start of a new simulation rollout. + fn begin_simulation(&mut self) {} + + /// Marks the start of a rollout whose simulator state will be discarded. + /// + /// Parallel planners run rollouts on cloned simulators and drop each clone + /// after its sampled tail reward has been computed. Implementations may use + /// this hook to suppress rollback bookkeeping that would only be useful if + /// the same simulator were restored and reused. The default delegates to + /// [`Self::begin_simulation`] so external simulator implementations keep the + /// older reversible-rollout behavior unless they opt into a cheaper + /// discardable path. + /// + /// # Safety & Invariants + /// Calling [`Self::model_revert`] within a discardable simulation scope is + /// unsupported and will result in a panic, as rollback checkpoints are not retained. + fn begin_discardable_simulation(&mut self) { + self.begin_simulation(); + } + + /// Reverts the model state to a previous point in the simulation. + /// + /// # Panics + /// Panics if called within an active discardable simulation scope opened by + /// [`Self::begin_discardable_simulation`]. + fn model_revert(&mut self, steps: usize); + + /// Generates a random value in `[0, end)`. + fn gen_range(&mut self, end: usize) -> usize; + + /// Generates a random `f64` in `[0, 1)`. + fn gen_f64(&mut self) -> f64; + + /// Creates a boxed clone of this simulator for parallel search. + fn boxed_clone(&self) -> Box { + self.boxed_clone_with_seed(0) + } + + /// Creates a boxed clone of this simulator, re-seeding any RNG state. + fn boxed_clone_with_seed(&self, seed: u64) -> Box; + + /// Returns the discounted cumulative reward bounds for the given horizon. + fn cumulative_reward_bounds(&self, horizon: usize) -> (f64, f64) { + let min = self.min_reward() as f64; + let max = self.max_reward() as f64; + let gamma = self.discount_gamma().clamp(0.0, 1.0); + let sum = discounted_horizon_sum(gamma, horizon); + (min * sum, max * sum) + } + + /// Normalizes a reward value to `[0, 1]` for a particular remaining horizon. + /// + /// For `gamma == 1` this is action-equivalent to the `aixictwx` + /// finite-horizon normalization. For `gamma < 1` this is the discounted + /// finite-horizon analogue, matching the discounted return actually backed + /// up by the planner. + fn norm_reward_for_horizon(&self, reward: f64, horizon: usize) -> f64 { + let (min_cumulative, max_cumulative) = self.cumulative_reward_bounds(horizon); + let range = max_cumulative - min_cumulative; + if range.abs() < 1e-12 { + 0.5 + } else { + ((reward - min_cumulative) / range).clamp(0.0, 1.0) + } + } + + /// Backward-compatible normalization using the full planning horizon. + fn norm_reward(&self, reward: f64) -> f64 { + self.norm_reward_for_horizon(reward, self.horizon()) + } + + /// Helper to generate a percept stream, update the model, and return a + /// search key plus reward. + fn gen_percepts_and_update(&mut self) -> (Vec, Reward) { + let obs_bits = self.get_num_observation_bits(); + let obs_len = self.observation_stream_len().max(1); + let mut observations = Vec::with_capacity(obs_len); + for _ in 0..obs_len { + observations.push(self.gen_percept_and_update(obs_bits)); + } + + let obs_key = self.observation_repr_from_stream(&observations); + let rew_bits = self.get_num_reward_bits(); + let rew_u = self.gen_percept_and_update(rew_bits); + let rew = (rew_u as i64) - self.reward_offset(); + (obs_key, rew) + } +} + +pub(crate) fn discounted_horizon_sum(gamma: f64, horizon: usize) -> f64 { + if horizon == 0 { + return 0.0; + } + if (gamma - 1.0).abs() < 1e-12 { + horizon as f64 + } else { + (1.0 - gamma.powi(horizon as i32)) / (1.0 - gamma) + } +} + +pub(crate) fn random_rollout(agent: &mut dyn AgentSimulator, horizon: usize) -> f64 { + let num_actions = agent.get_num_actions().get(); + let gamma = agent.discount_gamma().clamp(0.0, 1.0); + let mut total_reward = 0.0; + let mut discount = 1.0; + + for _ in 0..horizon { + let action = agent.gen_range(num_actions) as Action; + agent.model_update_action(action); + let (_obs, reward) = agent.gen_percepts_and_update(); + total_reward += discount * (reward as f64); + discount *= gamma; + } + + total_reward +} + +pub(crate) fn ensure_action_slots(slots: &mut Vec>, num_actions: usize) { + if slots.len() < num_actions { + slots.resize_with(num_actions, || None); + } +} + +pub(crate) fn choose_uniform_unvisited( + agent: &mut dyn AgentSimulator, + slots: &[Option], + num_actions: usize, +) -> Option { + let mut unvisited = Vec::new(); + for action_idx in 0..num_actions { + if slots.get(action_idx).and_then(Option::as_ref).is_none() { + unvisited.push(action_idx); + } + } + if unvisited.is_empty() { + None + } else { + Some(unvisited[agent.gen_range(unvisited.len())]) + } +} + +pub(crate) fn best_action_from_action_values( + action_values: impl Iterator, + num_actions: ActionAlphabet, + agent: &mut dyn AgentSimulator, +) -> Action { + let mut best_actions = Vec::new(); + let mut best_value = -f64::INFINITY; + + for (action_idx, value) in action_values { + match value.total_cmp(&best_value) { + std::cmp::Ordering::Greater => { + best_value = value; + best_actions.clear(); + best_actions.push(action_idx as Action); + } + std::cmp::Ordering::Equal => best_actions.push(action_idx as Action), + std::cmp::Ordering::Less => {} + } + } + + if best_actions.is_empty() { + return agent.gen_range(num_actions.get()) as Action; + } + + best_actions[agent.gen_range(best_actions.len())] +} + +pub(crate) type PerceptMap = HashMap; diff --git a/crates/infotheory/src/aixi/mcts/parallel_uct.rs b/crates/infotheory/src/aixi/mcts/parallel_uct.rs new file mode 100644 index 00000000..eab2cad2 --- /dev/null +++ b/crates/infotheory/src/aixi/mcts/parallel_uct.rs @@ -0,0 +1,2314 @@ +use super::{ + AgentSimulator, PerceptMap, PerceptOutcome, best_action_from_action_values, + choose_uniform_unvisited, ensure_action_slots, prune_key, random_rollout, +}; +#[cfg(test)] +use crate::aixi::common::ActionAlphabet; +use crate::aixi::common::{Action, PerceptVal, Reward}; +use rayon::prelude::*; +use std::collections::HashMap; +use std::fmt; +use std::num::NonZeroUsize; +#[cfg(test)] +use std::sync::{Arc, Mutex}; + +const PARALLEL_PLANNER_SEED_SALT: u64 = 0x9E37_79B9_7F4A_7C15; + +/// Construction-time errors for [`ParallelUctPlanner`]. +/// +/// `workers == 0` is type-prevented at the API boundary by +/// [`NonZeroUsize`], so the only remaining failure mode is an out-of-range +/// `bu_uct_m_max`. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ParallelUctPlannerInitError { + /// `bu_uct_m_max` was provided but is not strictly inside `(0, 1)`. + InvalidBuUctMMax, +} + +impl fmt::Display for ParallelUctPlannerInitError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::InvalidBuUctMMax => write!(f, "parallel_uct bu_uct_m_max must be in (0, 1)"), + } + } +} + +impl std::error::Error for ParallelUctPlannerInitError {} + +/// Search-time errors for [`ParallelUctPlanner::search`]. +/// +/// These are contract violations detected at the public boundary before any +/// rollout work is dispatched. +/// +/// Note that `samples == 0` is always accepted (even when `agent.horizon() == 0`): +/// the retained root is pruned to the supplied percept history and an action is +/// selected from whatever completed root statistics are already available. +/// If no completed root action values exist (e.g. on a fresh tree), the +/// returned action is chosen uniformly at random from the action alphabet. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ParallelUctSearchError { + /// `samples > 0` was requested but `agent.horizon() == 0`. + /// + /// A positive simulation budget over a zero-step horizon has no + /// well-defined rollout depth, so the planner refuses the call rather + /// than silently returning an arbitrary action. + PositiveSamplesRequirePositiveHorizon, +} + +impl fmt::Display for ParallelUctSearchError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::PositiveSamplesRequirePositiveHorizon => { + write!( + f, + "parallel_uct requires agent.horizon() >= 1 when samples > 0" + ) + } + } + } +} + +impl std::error::Error for ParallelUctSearchError {} + +/// Explicit parallel UCT planner state. +/// +/// This backend implements explicit WU-UCT accounting and BU-core thresholding +/// plus grouped backpropagation over deterministic completion epochs. It does +/// not claim the full supplementary BU-UCT expansion scheduler. +pub struct ParallelUctPlanner { + state: ParallelPlannerState, + workers: NonZeroUsize, + bu_uct_m_max: Option, +} + +impl ParallelUctPlanner { + /// Construct a parallel UCT planner. + /// + /// `workers` is type-enforced non-zero. `bu_uct_m_max == None` selects + /// WU-UCT; `Some(x)` with `x \in (0, 1)` selects BU-UCT thresholding. + pub fn new( + workers: NonZeroUsize, + bu_uct_m_max: Option, + ) -> Result { + if matches!(bu_uct_m_max, Some(m_max) if !(0.0 < m_max && m_max < 1.0)) { + return Err(ParallelUctPlannerInitError::InvalidBuUctMMax); + } + Ok(Self { + state: if bu_uct_m_max.is_some() { + ParallelPlannerState::Bu(ParallelRuntime::new()) + } else { + ParallelPlannerState::Wu(ParallelRuntime::new()) + }, + workers, + bu_uct_m_max, + }) + } + + /// Run a parallel UCT search and return the recommended next action. + /// + /// This is the validated public entry point. It enforces the planner's + /// search-time contract before delegating to the internal driver shared + /// with the in-crate, already-validated MC-AIXI call sites. + /// + /// # Parameters + /// + /// - `agent`: simulator providing the action alphabet, planning horizon, + /// reward bounds, and rollout state. Mutated through the simulator's + /// own contract during expansion and rollouts. + /// - `prev_obs_stream`: most recent observation, decomposed into its + /// per-bit `PerceptVal` symbols, used to root the search tree. + /// - `prev_rew`: reward delivered alongside `prev_obs_stream`. + /// - `prev_act`: action that produced `(prev_obs_stream, prev_rew)`. + /// - `samples`: simulation budget (number of rollouts). `0` is a valid + /// request: it prunes the retained root to the supplied percept history + /// and then selects an action from the currently retained completed root + /// statistics without launching any rollouts. + /// + /// If no completed root action values are available, the returned action + /// is chosen uniformly at random from the action alphabet. + /// + /// # Errors + /// + /// Returns [`ParallelUctSearchError::PositiveSamplesRequirePositiveHorizon`] + /// when `samples > 0` and `agent.horizon() == 0`. No worker tasks are + /// spawned and no planner search state is mutated when the contract fails. + /// + /// # Concurrency and cost + /// + /// Dispatch parallelism is capped by `workers` (fixed at construction; see + /// [`ParallelUctPlanner::new`]) and by the requested `samples` (small + /// budgets may use fewer worker tasks). Completed rollout batches are + /// evaluated through Rayon. A successful positive-budget call performs + /// exactly `samples` rollout dispatches, each bounded by `agent.horizon()`, + /// and applies their updates in deterministic completion-epoch order. + /// + /// # Randomness + /// + /// `search` consumes randomness from `agent` to seed per-task simulator + /// clones and to break ties between equal-valued actions (and, when no + /// completed root statistics exist, to fall back to a uniform random + /// action). As a result, even a `samples == 0` call may advance the + /// simulator RNG state. + pub fn search( + &mut self, + agent: &mut dyn AgentSimulator, + prev_obs_stream: &[PerceptVal], + prev_rew: Reward, + prev_act: Action, + samples: usize, + ) -> Result { + let horizon = agent.horizon(); + if samples > 0 && horizon == 0 { + return Err(ParallelUctSearchError::PositiveSamplesRequirePositiveHorizon); + } + Ok(self.search_validated_with_horizon( + agent, + prev_obs_stream, + prev_rew, + prev_act, + samples, + horizon, + )) + } + + pub(crate) fn search_validated( + &mut self, + agent: &mut dyn AgentSimulator, + prev_obs_stream: &[PerceptVal], + prev_rew: Reward, + prev_act: Action, + samples: usize, + ) -> Action { + let horizon = agent.horizon(); + debug_assert!( + samples == 0 || horizon > 0, + "parallel_uct validated search requires agent.horizon() >= 1 when samples > 0" + ); + self.search_validated_with_horizon( + agent, + prev_obs_stream, + prev_rew, + prev_act, + samples, + horizon, + ) + } + + fn search_validated_with_horizon( + &mut self, + agent: &mut dyn AgentSimulator, + prev_obs_stream: &[PerceptVal], + prev_rew: Reward, + prev_act: Action, + samples: usize, + horizon: usize, + ) -> Action { + let workers = self.workers.get(); + match &mut self.state { + ParallelPlannerState::Wu(runtime) => search_runtime::( + runtime, + agent, + SearchRuntimeParams { + prev_obs_stream, + prev_rew, + prev_act, + samples, + horizon, + workers, + bu_uct_m_max: None, + }, + ), + ParallelPlannerState::Bu(runtime) => search_runtime::( + runtime, + agent, + SearchRuntimeParams { + prev_obs_stream, + prev_rew, + prev_act, + samples, + horizon, + workers, + bu_uct_m_max: self.bu_uct_m_max, + }, + ), + } + } +} + +struct SearchRuntimeParams<'a> { + prev_obs_stream: &'a [PerceptVal], + prev_rew: Reward, + prev_act: Action, + samples: usize, + horizon: usize, + workers: usize, + bu_uct_m_max: Option, +} + +fn search_runtime( + runtime: &mut ParallelRuntime, + agent: &mut dyn AgentSimulator, + params: SearchRuntimeParams<'_>, +) -> Action { + let SearchRuntimeParams { + prev_obs_stream, + prev_rew, + prev_act, + samples, + horizon, + workers, + bu_uct_m_max, + } = params; + prune_tree(runtime, agent, prev_obs_stream, prev_rew, prev_act); + + debug_assert!(workers > 0); + let logical_workers = workers.min(samples.max(1)); + let planner_seed = agent.gen_f64().to_bits(); + let gamma = agent.discount_gamma().clamp(0.0, 1.0); + + let mut dispatched = 0usize; + if samples > 0 { + let root_is_fresh = runtime.root.as_ref().is_some_and(|root| root.visits == 0); + if root_is_fresh { + let task_index = 0usize; + let mut local_agent = + agent.boxed_clone_with_seed(planner_task_seed(planner_seed, task_index)); + local_agent.begin_discardable_simulation(); + let bootstrap = bootstrap_root(runtime, local_agent.as_mut(), horizon, task_index); + complete_update_batch(runtime, std::slice::from_ref(&bootstrap), gamma); + dispatched = 1; + } + } + + while dispatched < samples { + let batch_size = logical_workers.min(samples - dispatched); + let mut pending = Vec::with_capacity(batch_size); + + for batch_index in 0..batch_size { + let task_index = dispatched + batch_index; + let mut local_agent = + agent.boxed_clone_with_seed(planner_task_seed(planner_seed, task_index)); + local_agent.begin_discardable_simulation(); + + let dispatch = { + let root = runtime.root.as_mut().expect("parallel_uct root missing"); + dispatch_rollout::( + root, + &mut runtime.next_node_id, + local_agent.as_mut(), + horizon, + workers, + bu_uct_m_max, + ) + }; + pending.push(PendingRollout { + task_index, + agent: local_agent, + remaining_horizon: dispatch.remaining_horizon, + path: dispatch.path, + }); + } + + let completed = pending + .into_par_iter() + .map(|mut task| CompletedRollout { + task_index: task.task_index, + path: task.path, + tail_reward: random_rollout(task.agent.as_mut(), task.remaining_horizon), + }) + .collect::>(); + let mut completed = completed; + completed.sort_by_key(|task| task.task_index); + complete_update_batch(runtime, &completed, gamma); + + dispatched += batch_size; + } + + if samples > 0 { + let root = runtime.root.as_ref().expect("parallel_uct root missing"); + return best_action_after_positive_budget(root, agent); + } + + best_action(runtime.root.as_ref(), agent) +} + +fn planner_task_seed(planner_seed: u64, task_index: usize) -> u64 { + planner_seed ^ ((task_index as u64).wrapping_mul(PARALLEL_PLANNER_SEED_SALT)) +} + +fn best_action( + root: Option<&DecisionNode>, + agent: &mut dyn AgentSimulator, +) -> Action { + let Some(root) = root else { + return agent.gen_range(agent.get_num_actions().get()) as Action; + }; + best_action_from_action_values( + root.action_edges + .iter() + .enumerate() + .filter_map(|(action_idx, edge)| { + edge.as_ref() + .filter(|edge| edge.completed_n() > 0) + .map(|edge| (action_idx, edge.completed_q())) + }), + agent.get_num_actions(), + agent, + ) +} + +fn best_action_after_positive_budget( + root: &DecisionNode, + agent: &mut dyn AgentSimulator, +) -> Action { + debug_assert!( + root.action_edges + .iter() + .filter_map(Option::as_ref) + .any(|edge| edge.completed_n() > 0), + "positive-budget parallel_uct search must leave at least one completed root edge" + ); + + best_action(Some(root), agent) +} + +#[cfg(test)] +fn root_has_completed_edge(root: &DecisionNode) -> bool { + root.action_edges + .iter() + .filter_map(Option::as_ref) + .any(|edge| edge.completed_n() > 0) +} + +fn prune_tree( + runtime: &mut ParallelRuntime, + agent: &dyn AgentSimulator, + prev_obs_stream: &[PerceptVal], + prev_rew: Reward, + prev_act: Action, +) { + let Some(mut old_root) = runtime.root.take() else { + runtime.root = Some(runtime.fresh_decision_node()); + return; + }; + + let action_edge = old_root + .action_edges + .get_mut(prev_act as usize) + .and_then(Option::take); + let Some(mut action_edge) = action_edge else { + runtime.root = Some(runtime.fresh_decision_node()); + return; + }; + + let key = prune_key(agent, prev_obs_stream, prev_rew); + runtime.root = action_edge + .chance_mut() + .percept_children + .remove(&key) + .or_else(|| Some(runtime.fresh_decision_node())); +} + +fn bootstrap_root( + runtime: &mut ParallelRuntime, + agent: &mut dyn AgentSimulator, + remaining_horizon: usize, + task_index: usize, +) -> CompletedRollout { + debug_assert!(remaining_horizon > 0); + let root = runtime.root.as_mut().expect("parallel_uct root missing"); + let action_idx = choose_bootstrap_action(root, agent); + + agent.model_update_action(action_idx as Action); + let (observations, immediate_reward) = agent.gen_percepts_and_update(); + let outcome = PerceptOutcome::new(observations, immediate_reward); + + let parent_node_id = root.id; + let edge = root.action_edges[action_idx] + .as_mut() + .expect("parallel_uct bootstrap action edge missing"); + if !edge.chance().percept_children.contains_key(&outcome) { + let id = runtime.next_node_id; + runtime.next_node_id += 1; + edge.chance_mut() + .percept_children + .insert(outcome.clone(), DecisionNode::new(id)); + } + edge.on_incomplete_update(); + let child_node_id = edge + .chance() + .percept_children + .get(&outcome) + .expect("parallel_uct bootstrap percept child missing") + .id; + + let path = vec![ParallelPathStep { + parent_node_id, + action_idx, + child_node_id, + outcome, + }]; + let tail_reward = random_rollout(agent, remaining_horizon.saturating_sub(1)); + CompletedRollout { + task_index, + path, + tail_reward, + } +} + +fn choose_bootstrap_action( + root: &mut DecisionNode, + agent: &mut dyn AgentSimulator, +) -> usize { + let num_actions = agent.get_num_actions(); + ensure_action_slots(&mut root.action_edges, num_actions.get()); + + let mut unvisited = Vec::new(); + for action_idx in 0..num_actions.get() { + match root.action_edges.get(action_idx).and_then(Option::as_ref) { + None => unvisited.push(action_idx), + Some(edge) if edge.completed_n() == 0 && edge.effective_visits() == 0 => { + unvisited.push(action_idx); + } + Some(_) => {} + } + } + + let selected = unvisited[agent.gen_range(unvisited.len())]; + if root.action_edges[selected].is_none() { + root.action_edges[selected] = Some(M::Edge::new()); + } + selected +} + +fn dispatch_rollout( + node: &mut DecisionNode, + next_node_id: &mut u64, + agent: &mut dyn AgentSimulator, + remaining_horizon: usize, + workers: usize, + bu_uct_m_max: Option, +) -> DispatchRollout { + let mut path = Vec::new(); + let remaining_horizon = dispatch_rollout_into::( + node, + next_node_id, + agent, + remaining_horizon, + workers, + bu_uct_m_max, + &mut path, + ); + DispatchRollout { + remaining_horizon, + path, + } +} + +fn dispatch_rollout_into( + node: &mut DecisionNode, + next_node_id: &mut u64, + agent: &mut dyn AgentSimulator, + remaining_horizon: usize, + workers: usize, + bu_uct_m_max: Option, + path: &mut Vec, +) -> usize { + if remaining_horizon == 0 || node.visits == 0 { + return remaining_horizon; + } + + let num_actions = agent.get_num_actions(); + ensure_action_slots(&mut node.action_edges, num_actions.get()); + + let action_idx = if let Some(unvisited) = + choose_uniform_unvisited(agent, &node.action_edges, num_actions.get()) + { + node.action_edges[unvisited] = Some(M::Edge::new()); + unvisited + } else { + let Some(action_idx) = + select_existing_action(node, agent, remaining_horizon, workers, bu_uct_m_max) + else { + return remaining_horizon; + }; + action_idx + }; + + agent.model_update_action(action_idx as Action); + let (observations, immediate_reward) = agent.gen_percepts_and_update(); + let outcome = PerceptOutcome::new(observations, immediate_reward); + let parent_node_id = node.id; + + let edge = node.action_edges[action_idx] + .as_mut() + .expect("parallel_uct action edge missing"); + if !edge.chance().percept_children.contains_key(&outcome) { + let id = *next_node_id; + *next_node_id += 1; + edge.chance_mut() + .percept_children + .insert(outcome.clone(), DecisionNode::new(id)); + } + edge.on_incomplete_update(); + let child = edge + .chance_mut() + .percept_children + .get_mut(&outcome) + .expect("parallel_uct percept child missing after insertion"); + let child_node_id = child.id; + + path.push(ParallelPathStep { + parent_node_id, + action_idx, + child_node_id, + outcome, + }); + + dispatch_rollout_into::( + child, + next_node_id, + agent, + remaining_horizon - 1, + workers, + bu_uct_m_max, + path, + ) +} + +fn select_existing_action( + node: &DecisionNode, + agent: &mut dyn AgentSimulator, + remaining_horizon: usize, + workers: usize, + bu_uct_m_max: Option, +) -> Option { + let total_overline_n = node + .action_edges + .iter() + .filter_map(Option::as_ref) + .map(EdgeOps::effective_visits) + .sum::(); + let log_total = ((total_overline_n.max(1)) as f64).ln().max(0.0); + let c = agent.get_explore_exploit_ratio().max(0.0); + + let mut best_score = -f64::INFINITY; + let mut best_action = None; + let mut num_maximal_actions = 0usize; + + for (action_idx, edge) in node.action_edges.iter().enumerate() { + let Some(edge) = edge.as_ref() else { + continue; + }; + let overline_n = edge.effective_visits(); + if overline_n == 0 || !M::edge_is_selectable(edge, workers, bu_uct_m_max) { + continue; + } + + let normalized_value = agent.norm_reward_for_horizon(edge.completed_q(), remaining_horizon); + let exploration = c * ((2.0 * log_total) / (overline_n as f64)).sqrt(); + let score = normalized_value + exploration; + debug_assert!( + score.is_finite(), + "parallel_uct UCB score must be finite for visited action edges" + ); + + match score.total_cmp(&best_score) { + std::cmp::Ordering::Greater => { + best_score = score; + best_action = Some(action_idx); + num_maximal_actions = 1; + } + std::cmp::Ordering::Equal => { + num_maximal_actions += 1; + if agent.gen_range(num_maximal_actions) == 0 { + best_action = Some(action_idx); + } + } + std::cmp::Ordering::Less => {} + } + } + + best_action +} + +#[cfg(test)] +fn incomplete_update(runtime: &mut ParallelRuntime, path: &[ParallelPathStep]) { + if path.is_empty() { + return; + } + let mut current = runtime.root.as_mut().expect("parallel_uct root missing"); + for step in path { + let edge = current.action_edges[step.action_idx] + .as_mut() + .expect("parallel_uct action edge missing during incomplete_update"); + edge.on_incomplete_update(); + current = edge + .chance_mut() + .percept_children + .get_mut(&step.outcome) + .expect("parallel_uct percept child missing during incomplete_update"); + } +} + +fn complete_update_batch( + runtime: &mut ParallelRuntime, + completed: &[CompletedRollout], + gamma: f64, +) { + let mut epoch_state = M::EpochState::default(); + for task in completed { + let root = runtime.root.as_mut().expect("parallel_uct root missing"); + complete_update_node::( + root, + &task.path, + 0, + task.tail_reward, + gamma, + &mut epoch_state, + ); + } +} + +fn complete_update_node( + node: &mut DecisionNode, + path: &[ParallelPathStep], + depth: usize, + tail_reward: f64, + gamma: f64, + epoch_state: &mut M::EpochState, +) -> f64 { + node.visits += 1; + if depth == path.len() { + return tail_reward; + } + + let step = &path[depth]; + let edge = node.action_edges[step.action_idx] + .as_mut() + .expect("parallel_uct action edge missing during complete_update"); + let child = edge + .chance_mut() + .percept_children + .get_mut(&step.outcome) + .expect("parallel_uct percept child missing during complete_update"); + let downstream = + complete_update_node::(child, path, depth + 1, tail_reward, gamma, epoch_state); + + let reward = (step.outcome.reward() as f64) + gamma * downstream; + M::complete_edge( + edge, + BuEpochKey { + parent_node_id: step.parent_node_id, + action_idx: step.action_idx, + child_node_id: step.child_node_id, + }, + reward, + epoch_state, + ); + reward +} + +enum ParallelPlannerState { + Wu(ParallelRuntime), + Bu(ParallelRuntime), +} + +struct ParallelRuntime { + root: Option>, + next_node_id: u64, +} + +impl ParallelRuntime { + fn new() -> Self { + Self { + root: Some(DecisionNode::new(0)), + next_node_id: 1, + } + } + + fn fresh_decision_node(&mut self) -> DecisionNode { + let id = self.next_node_id; + self.next_node_id += 1; + DecisionNode::new(id) + } +} + +trait ModeState: Copy { + type Edge: EdgeOps; + type EpochState: Default; + + fn edge_is_selectable(edge: &Self::Edge, workers: usize, bu_uct_m_max: Option) -> bool; + + fn complete_edge( + edge: &mut Self::Edge, + key: BuEpochKey, + reward: f64, + epoch_state: &mut Self::EpochState, + ); +} + +trait EdgeOps: Clone { + fn new() -> Self; + fn chance(&self) -> &ChanceNode; + fn chance_mut(&mut self) -> &mut ChanceNode; + fn effective_visits(&self) -> u32; + fn completed_q(&self) -> f64; + fn completed_n(&self) -> u32; + fn on_incomplete_update(&mut self); +} + +#[derive(Clone, Copy)] +struct WuMode; + +#[derive(Clone, Copy)] +struct BuMode; + +#[derive(Clone)] +struct DecisionNode { + id: u64, + visits: u32, + action_edges: Vec>, +} + +impl DecisionNode { + fn new(id: u64) -> Self { + Self { + id, + visits: 0, + action_edges: Vec::new(), + } + } +} + +#[derive(Clone)] +struct ChanceNode { + percept_children: PerceptMap>, +} + +impl Default for ChanceNode { + fn default() -> Self { + Self { + percept_children: PerceptMap::default(), + } + } +} + +#[derive(Clone)] +struct WuActionEdge { + q: f64, + n: u32, + o: u32, + child: ChanceNode, +} + +impl EdgeOps for WuActionEdge { + fn new() -> Self { + Self { + q: 0.0, + n: 0, + o: 0, + child: ChanceNode::default(), + } + } + + fn chance(&self) -> &ChanceNode { + &self.child + } + + fn chance_mut(&mut self) -> &mut ChanceNode { + &mut self.child + } + + fn effective_visits(&self) -> u32 { + self.n + self.o + } + + fn completed_q(&self) -> f64 { + self.q + } + + fn completed_n(&self) -> u32 { + self.n + } + + fn on_incomplete_update(&mut self) { + self.o += 1; + } +} + +impl ModeState for WuMode { + type Edge = WuActionEdge; + type EpochState = (); + + fn edge_is_selectable(_edge: &Self::Edge, _workers: usize, _bu_uct_m_max: Option) -> bool { + true + } + + fn complete_edge( + edge: &mut Self::Edge, + _key: BuEpochKey, + reward: f64, + _epoch_state: &mut Self::EpochState, + ) { + edge.o = edge.o.saturating_sub(1); + edge.q = (reward + (edge.n as f64) * edge.q) / ((edge.n + 1) as f64); + edge.n += 1; + } +} + +#[derive(Clone)] +struct BuActionEdge { + q: f64, + n: u32, + o: u32, + // Paper-style BU incomplete-occupancy statistic updated only on + // `incomplete_update`; it is not a live mirror of the current `o` count. + o_bar: f64, + child: ChanceNode, +} + +impl EdgeOps for BuActionEdge { + fn new() -> Self { + Self { + q: 0.0, + n: 0, + o: 0, + o_bar: 0.0, + child: ChanceNode::default(), + } + } + + fn chance(&self) -> &ChanceNode { + &self.child + } + + fn chance_mut(&mut self) -> &mut ChanceNode { + &mut self.child + } + + fn effective_visits(&self) -> u32 { + self.n + self.o + } + + fn completed_q(&self) -> f64 { + self.q + } + + fn completed_n(&self) -> u32 { + self.n + } + + fn on_incomplete_update(&mut self) { + self.o += 1; + let overline_n = self.effective_visits(); + if overline_n > 0 { + self.o_bar = + (((overline_n - 1) as f64) * self.o_bar + (self.o as f64)) / (overline_n as f64); + } + } +} + +impl ModeState for BuMode { + type Edge = BuActionEdge; + type EpochState = BuEpochState; + + fn edge_is_selectable(edge: &Self::Edge, workers: usize, bu_uct_m_max: Option) -> bool { + let m_max = bu_uct_m_max.expect("BU mode requires bu_uct_m_max"); + edge.o_bar < m_max * (workers as f64) + } + + fn complete_edge( + edge: &mut Self::Edge, + key: BuEpochKey, + reward: f64, + epoch_state: &mut Self::EpochState, + ) { + edge.o = edge.o.saturating_sub(1); + // BU-core Part 1 intentionally keeps the paper-style one-sided `o_bar` + // lifecycle: completion decrements live `o` but does not recompute + // `o_bar`, so thresholding continues to use the accumulated incomplete + // occupancy statistic rather than current live occupancy. + edge.update_bu_epoch(key, reward, epoch_state); + } +} + +impl BuActionEdge { + fn update_bu_epoch(&mut self, key: BuEpochKey, reward: f64, epoch_state: &mut BuEpochState) { + use std::collections::hash_map::Entry; + + match epoch_state.groups.entry(key) { + Entry::Vacant(entry) => { + entry.insert(BuGroupStat { + mean: reward, + count: 1, + }); + self.q = if self.n == 0 { + reward + } else { + (((self.n as f64) * self.q) + reward) / ((self.n + 1) as f64) + }; + self.n += 1; + } + Entry::Occupied(mut entry) => { + let old_mean = entry.get().mean; + let stat = entry.get_mut(); + stat.count += 1; + stat.mean = old_mean + (reward - old_mean) / (stat.count as f64); + if self.n > 0 { + self.q += (stat.mean - old_mean) / (self.n as f64); + } + } + } + } +} + +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +struct BuEpochKey { + parent_node_id: u64, + action_idx: usize, + child_node_id: u64, +} + +#[derive(Clone, Copy)] +struct BuGroupStat { + mean: f64, + count: u32, +} + +#[derive(Default)] +struct BuEpochState { + groups: HashMap, +} + +struct DispatchRollout { + remaining_horizon: usize, + path: Vec, +} + +struct PendingRollout { + task_index: usize, + agent: Box, + remaining_horizon: usize, + path: Vec, +} + +struct CompletedRollout { + task_index: usize, + path: Vec, + tail_reward: f64, +} + +#[derive(Clone)] +struct ParallelPathStep { + parent_node_id: u64, + action_idx: usize, + child_node_id: u64, + outcome: PerceptOutcome, +} + +#[cfg(test)] +type WuDecisionNode = DecisionNode; +#[cfg(test)] +type BuDecisionNode = DecisionNode; +#[cfg(test)] +type BuChanceNode = ChanceNode; + +#[cfg(test)] +impl ParallelUctPlanner { + fn wu_root(&self) -> &WuDecisionNode { + match &self.state { + ParallelPlannerState::Wu(runtime) => runtime.root.as_ref().expect("WU root"), + ParallelPlannerState::Bu(_) => panic!("expected WU planner"), + } + } + + fn bu_root(&self) -> &BuDecisionNode { + match &self.state { + ParallelPlannerState::Bu(runtime) => runtime.root.as_ref().expect("BU root"), + ParallelPlannerState::Wu(_) => panic!("expected BU planner"), + } + } + + fn set_wu_root(&mut self, root: WuDecisionNode) { + match &mut self.state { + ParallelPlannerState::Wu(runtime) => runtime.root = Some(root), + ParallelPlannerState::Bu(_) => panic!("expected WU planner"), + } + } + + fn set_bu_root(&mut self, root: BuDecisionNode) { + match &mut self.state { + ParallelPlannerState::Bu(runtime) => runtime.root = Some(root), + ParallelPlannerState::Wu(_) => panic!("expected BU planner"), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::aixi::mcts::RhoUctPlanner; + use std::sync::atomic::{AtomicUsize, Ordering}; + + /// Test helper: build a `NonZeroUsize` worker count, panicking if zero. + /// + /// `NonZeroUsize` is the type-enforced API for `ParallelUctPlanner::new`, + /// so test fixtures opt into a tiny helper rather than repeating + /// `NonZeroUsize::new(N).expect(..)` at every call site. + fn workers(n: usize) -> NonZeroUsize { + NonZeroUsize::new(n).expect("test fixtures must use non-zero worker counts") + } + + #[derive(Clone)] + struct DeterministicRewardAgent { + last_action: Action, + emit_reward: bool, + } + + impl AgentSimulator for DeterministicRewardAgent { + fn get_num_actions(&self) -> ActionAlphabet { + ActionAlphabet::try_from_usize(2).expect("test fixture action alphabet must be valid") + } + + fn get_num_observation_bits(&self) -> usize { + 1 + } + + fn get_num_reward_bits(&self) -> usize { + 1 + } + + fn horizon(&self) -> usize { + 1 + } + + fn max_reward(&self) -> Reward { + 1 + } + + fn min_reward(&self) -> Reward { + 0 + } + + fn get_explore_exploit_ratio(&self) -> f64 { + 0.0 + } + + fn model_update_action(&mut self, action: Action) { + self.last_action = action; + self.emit_reward = false; + } + + fn gen_percept_and_update(&mut self, _bits: usize) -> u64 { + if self.emit_reward { + self.emit_reward = false; + self.last_action + } else { + self.emit_reward = true; + 0 + } + } + + fn model_revert(&mut self, _steps: usize) { + self.emit_reward = false; + } + + fn gen_range(&mut self, _end: usize) -> usize { + 0 + } + + fn gen_f64(&mut self) -> f64 { + 0.0 + } + + fn boxed_clone_with_seed(&self, seed: u64) -> Box { + let _ = seed; + Box::new(self.clone()) + } + } + + #[derive(Clone)] + struct ThresholdProbeAgent { + num_actions: usize, + model_updates: usize, + range_result: usize, + } + + impl AgentSimulator for ThresholdProbeAgent { + fn get_num_actions(&self) -> ActionAlphabet { + ActionAlphabet::try_from_usize(self.num_actions) + .expect("test fixture action alphabet must be valid") + } + + fn get_num_observation_bits(&self) -> usize { + 1 + } + + fn get_num_reward_bits(&self) -> usize { + 1 + } + + fn horizon(&self) -> usize { + 1 + } + + fn max_reward(&self) -> Reward { + 1 + } + + fn min_reward(&self) -> Reward { + 0 + } + + fn get_explore_exploit_ratio(&self) -> f64 { + 0.0 + } + + fn model_update_action(&mut self, _action: Action) { + self.model_updates += 1; + } + + fn gen_percept_and_update(&mut self, _bits: usize) -> u64 { + 0 + } + + fn model_revert(&mut self, _steps: usize) {} + + fn gen_range(&mut self, end: usize) -> usize { + self.range_result.min(end.saturating_sub(1)) + } + + fn gen_f64(&mut self) -> f64 { + 0.0 + } + + fn boxed_clone_with_seed(&self, _seed: u64) -> Box { + Box::new(self.clone()) + } + } + + #[derive(Clone)] + struct CounterAgent { + clone_count: Arc, + begin_count: Arc, + discardable_begin_count: Arc, + model_updates: Arc, + planning_horizon: usize, + last_action: Action, + emit_reward: bool, + } + + impl CounterAgent { + fn new_with_horizon(planning_horizon: usize) -> Self { + Self { + clone_count: Arc::new(AtomicUsize::new(0)), + begin_count: Arc::new(AtomicUsize::new(0)), + discardable_begin_count: Arc::new(AtomicUsize::new(0)), + model_updates: Arc::new(AtomicUsize::new(0)), + planning_horizon, + last_action: 0, + emit_reward: false, + } + } + } + + impl AgentSimulator for CounterAgent { + fn get_num_actions(&self) -> ActionAlphabet { + ActionAlphabet::try_from_usize(2).expect("test fixture action alphabet must be valid") + } + + fn get_num_observation_bits(&self) -> usize { + 1 + } + + fn get_num_reward_bits(&self) -> usize { + 1 + } + + fn horizon(&self) -> usize { + self.planning_horizon + } + + fn max_reward(&self) -> Reward { + 1 + } + + fn min_reward(&self) -> Reward { + 0 + } + + fn get_explore_exploit_ratio(&self) -> f64 { + 0.0 + } + + fn begin_simulation(&mut self) { + self.begin_count.fetch_add(1, Ordering::SeqCst); + } + + fn begin_discardable_simulation(&mut self) { + self.discardable_begin_count.fetch_add(1, Ordering::SeqCst); + } + + fn model_update_action(&mut self, action: Action) { + self.last_action = action; + self.emit_reward = false; + self.model_updates.fetch_add(1, Ordering::SeqCst); + } + + fn gen_percept_and_update(&mut self, _bits: usize) -> u64 { + if self.emit_reward { + self.emit_reward = false; + self.last_action + } else { + self.emit_reward = true; + 0 + } + } + + fn model_revert(&mut self, _steps: usize) { + self.emit_reward = false; + } + + fn gen_range(&mut self, _end: usize) -> usize { + 0 + } + + fn gen_f64(&mut self) -> f64 { + 0.0 + } + + fn boxed_clone_with_seed(&self, _seed: u64) -> Box { + self.clone_count.fetch_add(1, Ordering::SeqCst); + Box::new(self.clone()) + } + } + + #[derive(Clone)] + struct SeedRecordingAgent { + recorded_seeds: Arc>>, + last_action: Action, + emit_reward: bool, + clone_seed: u64, + } + + impl SeedRecordingAgent { + fn new() -> Self { + Self { + recorded_seeds: Arc::new(Mutex::new(Vec::new())), + last_action: 0, + emit_reward: false, + clone_seed: 0, + } + } + } + + impl AgentSimulator for SeedRecordingAgent { + fn get_num_actions(&self) -> ActionAlphabet { + ActionAlphabet::try_from_usize(2).expect("test fixture action alphabet must be valid") + } + + fn get_num_observation_bits(&self) -> usize { + 1 + } + + fn get_num_reward_bits(&self) -> usize { + 1 + } + + fn horizon(&self) -> usize { + 1 + } + + fn max_reward(&self) -> Reward { + 1 + } + + fn min_reward(&self) -> Reward { + 0 + } + + fn get_explore_exploit_ratio(&self) -> f64 { + 0.0 + } + + fn model_update_action(&mut self, action: Action) { + self.last_action = action; + self.emit_reward = false; + } + + fn gen_percept_and_update(&mut self, _bits: usize) -> u64 { + if self.emit_reward { + self.emit_reward = false; + (self.clone_seed ^ self.last_action) & 1 + } else { + self.emit_reward = true; + 0 + } + } + + fn model_revert(&mut self, _steps: usize) { + self.emit_reward = false; + } + + fn gen_range(&mut self, _end: usize) -> usize { + 0 + } + + fn gen_f64(&mut self) -> f64 { + 0.25 + } + + fn boxed_clone_with_seed(&self, seed: u64) -> Box { + self.recorded_seeds.lock().expect("seed list").push(seed); + Box::new(Self { + recorded_seeds: Arc::clone(&self.recorded_seeds), + last_action: 0, + emit_reward: false, + clone_seed: seed, + }) + } + } + + fn step( + parent_node_id: u64, + action_idx: usize, + child_node_id: u64, + observation: u64, + reward: Reward, + ) -> ParallelPathStep { + ParallelPathStep { + parent_node_id, + action_idx, + child_node_id, + outcome: PerceptOutcome::new(vec![observation], reward), + } + } + + fn two_step_wu_root() -> WuDecisionNode { + WuDecisionNode { + id: 10, + visits: 2, + action_edges: vec![ + Some(WuActionEdge { + q: 1.0, + n: 1, + o: 0, + child: ChanceNode { + percept_children: HashMap::from([( + PerceptOutcome::new(vec![0], 0), + WuDecisionNode { + id: 11, + visits: 2, + action_edges: vec![ + Some(WuActionEdge { + q: 1.0, + n: 1, + o: 0, + child: ChanceNode { + percept_children: HashMap::from([( + PerceptOutcome::new(vec![0], 0), + WuDecisionNode::new(12), + )]), + }, + }), + Some(WuActionEdge { + q: 0.0, + n: 1, + o: 0, + child: ChanceNode::default(), + }), + ], + }, + )]), + }, + }), + Some(WuActionEdge { + q: 0.0, + n: 1, + o: 0, + child: ChanceNode::default(), + }), + ], + } + } + + fn retained_parent_for(next_root: WuDecisionNode) -> WuDecisionNode { + WuDecisionNode { + id: 9, + visits: 1, + action_edges: vec![Some(WuActionEdge { + q: 0.0, + n: 1, + o: 0, + child: ChanceNode { + percept_children: HashMap::from([(PerceptOutcome::new(vec![0], 0), next_root)]), + }, + })], + } + } + + // NOTE: `workers == 0` is now type-enforced at the API boundary by + // `ParallelUctPlanner::new` accepting `NonZeroUsize`, so a runtime test + // analogous to the previous `planner_new_rejects_zero_workers_at_api_boundary` + // is structurally impossible here and would not even compile. + + #[test] + fn planner_new_rejects_invalid_bu_threshold_at_api_boundary() { + for invalid in [0.0, 1.0, -0.1, 1.1] { + let err = match ParallelUctPlanner::new(workers(2), Some(invalid)) { + Ok(_) => panic!("invalid BU threshold must be rejected"), + Err(err) => err, + }; + assert_eq!(err, ParallelUctPlannerInitError::InvalidBuUctMMax); + } + } + + #[test] + fn wu_workers_one_matches_rho_uct_on_deterministic_agent() { + let mut seq_agent = DeterministicRewardAgent { + last_action: 0, + emit_reward: false, + }; + let mut par_agent = seq_agent.clone(); + + let mut sequential = RhoUctPlanner::new(); + let mut parallel = + ParallelUctPlanner::new(workers(1), None).expect("valid parallel_uct planner"); + + let seq_action = sequential.search(&mut seq_agent, &[0], 0, 0, 16); + let par_action = parallel + .search(&mut par_agent, &[0], 0, 0, 16) + .expect("positive-horizon parallel_uct search"); + + assert_eq!(seq_action, par_action); + assert_eq!(parallel.wu_root().visits, 16); + } + + #[test] + fn fresh_root_bootstrap_produces_completed_root_edge_for_positive_budget() { + let mut agent = DeterministicRewardAgent { + last_action: 0, + emit_reward: false, + }; + let mut planner = + ParallelUctPlanner::new(workers(4), None).expect("valid parallel_uct planner"); + let action = planner + .search(&mut agent, &[0], 0, 0, 4) + .expect("positive-horizon parallel_uct search"); + + let root = planner.wu_root(); + assert!(action < 2); + assert_eq!(root.visits, 4); + assert!( + root_has_completed_edge(root), + "positive-budget search on a fresh retained root must complete at least one root edge" + ); + assert!( + root.action_edges + .iter() + .filter_map(Option::as_ref) + .all(|edge| edge.o == 0), + "all incomplete counts must be cleared after the search batch completes" + ); + } + + #[test] + fn zero_sample_budget_does_not_bootstrap_or_clone_even_at_zero_horizon() { + let mut agent = CounterAgent::new_with_horizon(0); + let mut planner = + ParallelUctPlanner::new(workers(4), None).expect("valid parallel_uct planner"); + + let action = planner + .search(&mut agent, &[0], 0, 0, 0) + .expect("zero-sample parallel_uct search should not require positive horizon"); + assert_eq!(action, 0); + assert_eq!(agent.clone_count.load(Ordering::SeqCst), 0); + assert_eq!(agent.begin_count.load(Ordering::SeqCst), 0); + assert_eq!(agent.discardable_begin_count.load(Ordering::SeqCst), 0); + assert_eq!(agent.model_updates.load(Ordering::SeqCst), 0); + let root = planner.wu_root(); + assert_eq!(root.visits, 0); + assert!( + !root_has_completed_edge(root), + "zero-budget search must not synthesize completed root edges" + ); + } + + #[test] + fn positive_budget_search_rejects_zero_horizon_at_api_boundary() { + let mut agent = CounterAgent::new_with_horizon(0); + let mut planner = + ParallelUctPlanner::new(workers(4), None).expect("valid parallel_uct planner"); + + let err = planner + .search(&mut agent, &[0], 0, 0, 1) + .expect_err("positive-budget zero-horizon search must be rejected"); + assert_eq!( + err, + ParallelUctSearchError::PositiveSamplesRequirePositiveHorizon + ); + assert_eq!(agent.clone_count.load(Ordering::SeqCst), 0); + assert_eq!(agent.begin_count.load(Ordering::SeqCst), 0); + assert_eq!(agent.discardable_begin_count.load(Ordering::SeqCst), 0); + assert_eq!(agent.model_updates.load(Ordering::SeqCst), 0); + let root = planner.wu_root(); + assert_eq!(root.visits, 0); + assert!( + !root_has_completed_edge(root), + "rejected zero-horizon search must leave the retained root untouched" + ); + } + + #[test] + fn single_sample_bootstrap_avoids_empty_root_fallback() { + let mut agent = DeterministicRewardAgent { + last_action: 0, + emit_reward: false, + }; + let mut planner = + ParallelUctPlanner::new(workers(4), None).expect("valid parallel_uct planner"); + + let action = planner + .search(&mut agent, &[0], 0, 0, 1) + .expect("positive-horizon parallel_uct search"); + assert_eq!(action, 0); + let root = planner.wu_root(); + assert_eq!(root.visits, 1); + assert!( + root_has_completed_edge(root), + "single-sample retained-root bootstrap must leave one completed root edge" + ); + } + + #[test] + fn positive_budget_uses_discardable_simulation_hook_for_clones() { + let mut agent = CounterAgent::new_with_horizon(2); + let mut planner = + ParallelUctPlanner::new(workers(2), None).expect("valid parallel_uct planner"); + + let _action = planner + .search(&mut agent, &[0], 0, 0, 5) + .expect("positive-horizon parallel_uct search"); + + assert_eq!(agent.clone_count.load(Ordering::SeqCst), 5); + assert_eq!(agent.discardable_begin_count.load(Ordering::SeqCst), 5); + assert_eq!( + agent.begin_count.load(Ordering::SeqCst), + 0, + "parallel_uct cloned rollouts must not open reversible simulation scopes", + ); + } + + #[test] + fn retained_root_same_percept_branch_reuses_bootstrapped_subtree_skeleton() { + let mut agent = DeterministicRewardAgent { + last_action: 0, + emit_reward: false, + }; + let mut planner = + ParallelUctPlanner::new(workers(2), None).expect("valid parallel_uct planner"); + let action = planner + .search(&mut agent, &[0], 0, 0, 1) + .expect("positive-horizon parallel_uct search"); + let root = planner.wu_root(); + let edge = root.action_edges[action as usize] + .as_ref() + .expect("root edge"); + let retained = edge + .chance() + .percept_children + .get(&PerceptOutcome::new(vec![0], 0)) + .expect("retained subtree"); + let retained_id = retained.id; + + let _follow_up = planner + .search(&mut agent, &[0], 0, action, 0) + .expect("zero-sample parallel_uct search should not require positive horizon"); + let next_root = planner.wu_root(); + assert_eq!(next_root.id, retained_id); + assert_eq!(next_root.visits, 1); + } + + #[test] + fn prune_to_fresh_root_bootstrap_preserves_positive_budget_invariant() { + let mut agent = DeterministicRewardAgent { + last_action: 0, + emit_reward: false, + }; + let mut planner = + ParallelUctPlanner::new(workers(2), None).expect("valid parallel_uct planner"); + planner.set_wu_root(WuDecisionNode { + id: 0, + visits: 8, + action_edges: vec![Some(WuActionEdge { + q: 0.0, + n: 1, + o: 0, + child: ChanceNode { + percept_children: HashMap::from([( + PerceptOutcome::new(vec![0], 0), + WuDecisionNode::new(1), + )]), + }, + })], + }); + + let second_action = planner + .search(&mut agent, &[0], 0, 0, 1) + .expect("positive-horizon parallel_uct search"); + assert_eq!(second_action, 0); + let root = planner.wu_root(); + assert_eq!(root.visits, 1); + assert!( + root_has_completed_edge(root), + "prune-to-fresh-root with positive budget must still complete a root edge" + ); + } + + #[test] + fn bootstrap_and_batched_rollouts_use_absolute_task_indices_for_seeding() { + let mut agent = SeedRecordingAgent::new(); + let mut planner = + ParallelUctPlanner::new(workers(4), None).expect("valid parallel_uct planner"); + + let _action = planner + .search(&mut agent, &[0], 0, 0, 5) + .expect("positive-horizon parallel_uct search"); + + let planner_seed = 0.25f64.to_bits(); + let expected = (0..5) + .map(|task_index| planner_task_seed(planner_seed, task_index)) + .collect::>(); + let seen = agent.recorded_seeds.lock().expect("seed list").clone(); + assert_eq!(seen, expected); + } + + #[test] + fn dispatch_rollout_records_root_to_leaf_path_and_updates_each_edge_once() { + let mut node = two_step_wu_root(); + let mut next_node_id = 13u64; + let mut agent = DeterministicRewardAgent { + last_action: 0, + emit_reward: false, + }; + + let dispatch = + dispatch_rollout::(&mut node, &mut next_node_id, &mut agent, 3, 1, None); + assert_eq!(dispatch.remaining_horizon, 1); + assert_eq!(dispatch.path.len(), 2); + assert_eq!(dispatch.path[0].parent_node_id, 10); + assert_eq!(dispatch.path[0].action_idx, 0); + assert_eq!(dispatch.path[0].child_node_id, 11); + assert_eq!(dispatch.path[0].outcome, PerceptOutcome::new(vec![0], 0)); + assert_eq!(dispatch.path[1].parent_node_id, 11); + assert_eq!(dispatch.path[1].action_idx, 0); + assert_eq!(dispatch.path[1].child_node_id, 12); + assert_eq!(dispatch.path[1].outcome, PerceptOutcome::new(vec![0], 0)); + + let root_edge = node.action_edges[0].as_ref().expect("root edge"); + let child = root_edge + .chance() + .percept_children + .get(&PerceptOutcome::new(vec![0], 0)) + .expect("child node"); + let child_edge = child.action_edges[0].as_ref().expect("child edge"); + assert_eq!(root_edge.o, 1); + assert_eq!(child_edge.o, 1); + } + + #[test] + fn ordinary_dispatch_path_completes_single_rollout_without_residual_incomplete_counts() { + let mut agent = CounterAgent::new_with_horizon(3); + let mut planner = + ParallelUctPlanner::new(workers(1), None).expect("valid parallel_uct planner"); + planner.set_wu_root(retained_parent_for(two_step_wu_root())); + + let action = planner + .search(&mut agent, &[0], 0, 0, 1) + .expect("positive-horizon parallel_uct search"); + assert_eq!(action, 0); + + let root = planner.wu_root(); + assert_eq!(root.id, 10); + let root_edge = root.action_edges[0].as_ref().expect("root edge"); + let child = root_edge + .chance() + .percept_children + .get(&PerceptOutcome::new(vec![0], 0)) + .expect("child node"); + let child_edge = child.action_edges[0].as_ref().expect("child edge"); + assert_eq!(root_edge.o, 0); + assert_eq!(child_edge.o, 0); + assert_eq!(root_edge.n, 2); + assert_eq!(child_edge.n, 2); + } + + #[test] + fn bu_thresholding_skips_oversubscribed_edges() { + let mut agent = DeterministicRewardAgent { + last_action: 0, + emit_reward: false, + }; + let mut node = BuDecisionNode::new(0); + node.visits = 8; + node.action_edges = vec![ + Some(BuActionEdge { + q: 0.9, + n: 4, + o: 0, + o_bar: 2.0, + child: BuChanceNode::default(), + }), + Some(BuActionEdge { + q: 0.1, + n: 4, + o: 0, + o_bar: 0.0, + child: BuChanceNode::default(), + }), + ]; + + let selected = select_existing_action::(&node, &mut agent, 1, 2, Some(0.5)); + assert_eq!( + selected, + Some(1), + "BU-UCT should skip edges whose average incomplete count exceeds the threshold" + ); + } + + #[test] + fn bu_thresholding_returns_none_when_all_edges_are_oversubscribed() { + let mut agent = DeterministicRewardAgent { + last_action: 0, + emit_reward: false, + }; + let mut node = BuDecisionNode::new(0); + node.visits = 8; + node.action_edges = vec![ + Some(BuActionEdge { + q: 0.9, + n: 4, + o: 0, + o_bar: 2.0, + child: BuChanceNode::default(), + }), + Some(BuActionEdge { + q: 0.1, + n: 4, + o: 0, + o_bar: 2.0, + child: BuChanceNode::default(), + }), + ]; + + let selected = select_existing_action::(&node, &mut agent, 1, 2, Some(0.5)); + assert_eq!(selected, None); + } + + #[test] + fn bu_thresholding_stops_at_current_node_when_all_expanded_children_forbidden() { + let mut node = BuDecisionNode::new(0); + node.visits = 8; + node.action_edges = vec![ + Some(BuActionEdge { + q: 0.9, + n: 4, + o: 0, + o_bar: 2.0, + child: BuChanceNode::default(), + }), + Some(BuActionEdge { + q: 0.1, + n: 4, + o: 0, + o_bar: 2.0, + child: BuChanceNode::default(), + }), + ]; + let mut next_node_id = 1u64; + let mut agent = ThresholdProbeAgent { + num_actions: 2, + model_updates: 0, + range_result: 0, + }; + + let dispatch = + dispatch_rollout::(&mut node, &mut next_node_id, &mut agent, 3, 2, Some(0.5)); + assert!( + dispatch.path.is_empty(), + "when every expanded child is threshold-forbidden, BU-core must stop at the current node" + ); + assert_eq!(dispatch.remaining_horizon, 3); + assert_eq!( + agent.model_updates, 0, + "threshold stop must not traverse or expand a forbidden edge" + ); + } + + #[test] + fn bu_threshold_stop_expands_unexpanded_legal_action_synchronously() { + let mut node = BuDecisionNode::new(0); + node.visits = 8; + node.action_edges = vec![ + Some(BuActionEdge { + q: 0.9, + n: 4, + o: 0, + o_bar: 2.0, + child: BuChanceNode::default(), + }), + Some(BuActionEdge { + q: 0.1, + n: 4, + o: 0, + o_bar: 2.0, + child: BuChanceNode::default(), + }), + None, + ]; + let mut next_node_id = 7u64; + let mut agent = ThresholdProbeAgent { + num_actions: 3, + model_updates: 0, + range_result: 0, + }; + + let dispatch = + dispatch_rollout::(&mut node, &mut next_node_id, &mut agent, 3, 2, Some(0.5)); + assert_eq!(dispatch.path.len(), 1); + assert_eq!(dispatch.path[0].action_idx, 2); + assert_eq!(dispatch.path[0].parent_node_id, 0); + assert_eq!(dispatch.path[0].child_node_id, 7); + assert_eq!(dispatch.remaining_horizon, 2); + assert_eq!(agent.model_updates, 1); + assert!( + node.action_edges[2].is_some(), + "BU-core must synchronously expand an unexpanded legal action after threshold stop" + ); + let edge = node.action_edges[2].as_ref().expect("expanded edge"); + let child = edge + .chance() + .percept_children + .get(&PerceptOutcome::new(vec![0], 0)) + .expect("expanded child"); + assert_eq!(edge.o, 1); + assert_eq!(child.id, dispatch.path[0].child_node_id); + } + + #[test] + fn bu_thresholding_uses_configured_worker_budget_not_batch_size() { + let mut planner = + ParallelUctPlanner::new(workers(4), Some(0.8)).expect("valid parallel_uct planner"); + let retained_root = BuDecisionNode { + id: 1, + visits: 8, + action_edges: vec![Some(BuActionEdge { + q: 0.2, + n: 1, + o: 0, + o_bar: 1.0, + child: BuChanceNode::default(), + })], + }; + planner.set_bu_root(BuDecisionNode { + id: 0, + visits: 8, + action_edges: vec![Some(BuActionEdge { + n: 1, + q: 0.0, + o: 0, + o_bar: 0.0, + child: BuChanceNode { + percept_children: HashMap::from([( + PerceptOutcome::new(vec![0], 0), + retained_root, + )]), + }, + })], + }); + let mut agent = ThresholdProbeAgent { + num_actions: 1, + model_updates: 0, + range_result: 0, + }; + + let action = planner + .search(&mut agent, &[0], 0, 0, 1) + .expect("positive-horizon parallel_uct search"); + assert_eq!(action, 0); + let root = planner.bu_root(); + let edge = root.action_edges[0].as_ref().expect("root edge"); + assert_eq!( + edge.n, 2, + "with configured workers=4 and m_max=0.8, O_bar=1.0 must remain admissible even for samples=1" + ); + } + + #[test] + fn bu_grouped_backpropagation_keeps_group_count_constant_for_same_child_origin() { + let mut edge = BuActionEdge::new(); + let mut epoch_state = BuEpochState::default(); + let key = BuEpochKey { + parent_node_id: 0, + action_idx: 0, + child_node_id: 7, + }; + + edge.update_bu_epoch(key, 1.0, &mut epoch_state); + assert_eq!(edge.n, 1); + assert_eq!(edge.q, 1.0); + + edge.update_bu_epoch(key, 0.0, &mut epoch_state); + assert_eq!(edge.n, 1); + assert!((edge.q - 0.5).abs() < 1e-12); + } + + #[test] + fn bu_batch_updates_q_once_from_same_origin_group_mean() { + let mut planner = + ParallelUctPlanner::new(workers(4), Some(0.8)).expect("valid parallel_uct planner"); + planner.set_bu_root(BuDecisionNode { + id: 0, + visits: 0, + action_edges: vec![Some(BuActionEdge { + q: 0.0, + n: 0, + o: 2, + o_bar: 0.0, + child: BuChanceNode { + percept_children: HashMap::from([( + PerceptOutcome::new(vec![0], 0), + BuDecisionNode::new(1), + )]), + }, + })], + }); + + let completed = vec![ + CompletedRollout { + task_index: 1, + path: vec![step(0, 0, 1, 0, 0)], + tail_reward: 1.0, + }, + CompletedRollout { + task_index: 0, + path: vec![step(0, 0, 1, 0, 0)], + tail_reward: 0.0, + }, + ]; + match &mut planner.state { + ParallelPlannerState::Bu(runtime) => complete_update_batch(runtime, &completed, 1.0), + ParallelPlannerState::Wu(_) => panic!("expected BU planner"), + } + + let root = planner.bu_root(); + let edge = root.action_edges[0].as_ref().expect("root edge"); + assert_eq!(root.visits, 2); + assert_eq!(edge.n, 1); + assert_eq!(edge.o, 0); + assert!((edge.q - 0.5).abs() < 1e-12); + } + + #[test] + fn bu_batch_distinguishes_different_child_origins_at_same_ancestor_edge() { + let mut planner = + ParallelUctPlanner::new(workers(4), Some(0.8)).expect("valid parallel_uct planner"); + planner.set_bu_root(BuDecisionNode { + id: 0, + visits: 0, + action_edges: vec![Some(BuActionEdge { + q: 0.0, + n: 0, + o: 2, + o_bar: 0.0, + child: BuChanceNode { + percept_children: HashMap::from([ + (PerceptOutcome::new(vec![0], 0), BuDecisionNode::new(1)), + (PerceptOutcome::new(vec![1], 0), BuDecisionNode::new(2)), + ]), + }, + })], + }); + + let completed = vec![ + CompletedRollout { + task_index: 0, + path: vec![step(0, 0, 1, 0, 0)], + tail_reward: 1.0, + }, + CompletedRollout { + task_index: 1, + path: vec![step(0, 0, 2, 1, 0)], + tail_reward: 0.0, + }, + ]; + match &mut planner.state { + ParallelPlannerState::Bu(runtime) => complete_update_batch(runtime, &completed, 1.0), + ParallelPlannerState::Wu(_) => panic!("expected BU planner"), + } + + let root = planner.bu_root(); + let edge = root.action_edges[0].as_ref().expect("root edge"); + assert_eq!(edge.n, 2); + assert!((edge.q - 0.5).abs() < 1e-12); + } + + #[test] + fn bu_grouping_is_local_to_each_ancestor_edge() { + let mut planner = + ParallelUctPlanner::new(workers(4), Some(0.8)).expect("valid parallel_uct planner"); + planner.set_bu_root(BuDecisionNode { + id: 0, + visits: 0, + action_edges: vec![Some(BuActionEdge { + q: 0.0, + n: 0, + o: 2, + o_bar: 0.0, + child: BuChanceNode { + percept_children: HashMap::from([( + PerceptOutcome::new(vec![0], 0), + BuDecisionNode { + id: 1, + visits: 0, + action_edges: vec![Some(BuActionEdge { + q: 0.0, + n: 0, + o: 2, + o_bar: 0.0, + child: BuChanceNode { + percept_children: HashMap::from([ + (PerceptOutcome::new(vec![10], 0), BuDecisionNode::new(2)), + (PerceptOutcome::new(vec![11], 0), BuDecisionNode::new(3)), + ]), + }, + })], + }, + )]), + }, + })], + }); + + let completed = vec![ + CompletedRollout { + task_index: 0, + path: vec![step(0, 0, 1, 0, 0), step(1, 0, 2, 10, 0)], + tail_reward: 1.0, + }, + CompletedRollout { + task_index: 1, + path: vec![step(0, 0, 1, 0, 0), step(1, 0, 3, 11, 0)], + tail_reward: 0.0, + }, + ]; + match &mut planner.state { + ParallelPlannerState::Bu(runtime) => complete_update_batch(runtime, &completed, 1.0), + ParallelPlannerState::Wu(_) => panic!("expected BU planner"), + } + + let root = planner.bu_root(); + let root_edge = root.action_edges[0].as_ref().expect("root edge"); + let child = root_edge + .chance() + .percept_children + .get(&PerceptOutcome::new(vec![0], 0)) + .expect("child node"); + let child_edge = child.action_edges[0].as_ref().expect("child edge"); + + assert_eq!(root_edge.n, 1); + assert_eq!(child_edge.n, 2); + } + + #[test] + fn incomplete_update_tracks_o_bar_recurrence() { + let mut planner = + ParallelUctPlanner::new(workers(4), Some(0.8)).expect("valid parallel_uct planner"); + planner.set_bu_root(BuDecisionNode { + id: 0, + visits: 0, + action_edges: vec![Some(BuActionEdge { + q: 0.0, + n: 0, + o: 0, + o_bar: 0.0, + child: BuChanceNode { + percept_children: HashMap::from([( + PerceptOutcome::new(vec![0], 0), + BuDecisionNode::new(1), + )]), + }, + })], + }); + + let path = vec![step(0, 0, 1, 0, 0)]; + match &mut planner.state { + ParallelPlannerState::Bu(runtime) => incomplete_update(runtime, &path), + ParallelPlannerState::Wu(_) => panic!("expected BU planner"), + } + let root = planner.bu_root(); + let edge = root.action_edges[0].as_ref().expect("edge"); + assert_eq!(edge.o, 1); + assert!((edge.o_bar - 1.0).abs() < 1e-12); + + match &mut planner.state { + ParallelPlannerState::Bu(runtime) => incomplete_update(runtime, &path), + ParallelPlannerState::Wu(_) => panic!("expected BU planner"), + } + let root = planner.bu_root(); + let edge = root.action_edges[0].as_ref().expect("edge"); + assert_eq!(edge.o, 2); + assert!((edge.o_bar - 1.5).abs() < 1e-12); + } + + #[test] + fn bu_thresholding_uses_o_bar_not_live_o_after_completion() { + let mut planner = + ParallelUctPlanner::new(workers(4), Some(0.5)).expect("valid parallel_uct planner"); + planner.set_bu_root(BuDecisionNode { + id: 0, + visits: 8, + action_edges: vec![ + Some(BuActionEdge { + q: 0.9, + n: 4, + o: 1, + o_bar: 2.0, + child: BuChanceNode { + percept_children: HashMap::from([( + PerceptOutcome::new(vec![0], 0), + BuDecisionNode::new(1), + )]), + }, + }), + Some(BuActionEdge { + q: 0.1, + n: 4, + o: 0, + o_bar: 0.0, + child: BuChanceNode::default(), + }), + ], + }); + + let completed = vec![CompletedRollout { + task_index: 0, + path: vec![step(0, 0, 1, 0, 0)], + tail_reward: 0.0, + }]; + match &mut planner.state { + ParallelPlannerState::Bu(runtime) => complete_update_batch(runtime, &completed, 1.0), + ParallelPlannerState::Wu(_) => panic!("expected BU planner"), + } + + let root = planner.bu_root(); + let forbidden = root.action_edges[0].as_ref().expect("forbidden edge"); + assert_eq!( + forbidden.o, 0, + "completion must clear the live incomplete count" + ); + assert!( + (forbidden.o_bar - 2.0).abs() < 1e-12, + "BU-core Part 1 keeps the paper-style one-sided o_bar statistic after completion" + ); + + let mut agent = DeterministicRewardAgent { + last_action: 0, + emit_reward: false, + }; + let selected = select_existing_action::(root, &mut agent, 1, 4, Some(0.5)); + assert_eq!( + selected, + Some(1), + "post-completion BU thresholding must continue to follow o_bar rather than current live o" + ); + } + + #[test] + fn bu_bootstrap_uses_singleton_epoch_completion() { + let mut agent = DeterministicRewardAgent { + last_action: 0, + emit_reward: false, + }; + let mut planner = + ParallelUctPlanner::new(workers(4), Some(0.5)).expect("valid parallel_uct planner"); + + let action = planner + .search(&mut agent, &[0], 0, 0, 1) + .expect("positive-horizon parallel_uct search"); + assert_eq!(action, 0); + let root = planner.bu_root(); + assert_eq!(root.visits, 1); + let edge = root.action_edges[0].as_ref().expect("bootstrap edge"); + assert_eq!(edge.n, 1); + assert_eq!(edge.o, 0); + } + + #[test] + fn final_root_choice_uses_completed_q_not_bu_thresholding() { + let mut planner = + ParallelUctPlanner::new(workers(2), Some(0.5)).expect("valid parallel_uct planner"); + planner.set_bu_root(BuDecisionNode { + id: 0, + visits: 8, + action_edges: vec![ + Some(BuActionEdge { + q: 0.9, + n: 4, + o: 0, + o_bar: 2.0, + child: BuChanceNode::default(), + }), + Some(BuActionEdge { + q: 0.1, + n: 4, + o: 0, + o_bar: 0.0, + child: BuChanceNode::default(), + }), + ], + }); + + let mut agent = DeterministicRewardAgent { + last_action: 0, + emit_reward: false, + }; + let action = best_action::(Some(planner.bu_root()), &mut agent); + assert_eq!(action, 0); + } + + #[test] + fn parallel_search_is_thread_count_independent() { + fn run_with_threads(threads: usize) -> (Action, u32, Vec<(u32, u32)>) { + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(threads) + .build() + .expect("thread pool"); + pool.install(|| { + let mut agent = DeterministicRewardAgent { + last_action: 0, + emit_reward: false, + }; + let mut planner = ParallelUctPlanner::new(workers(4), Some(0.8)) + .expect("valid parallel_uct planner"); + let action = planner + .search(&mut agent, &[0], 0, 0, 32) + .expect("positive-horizon parallel_uct search"); + let root = planner.bu_root(); + let child_stats = root + .action_edges + .iter() + .map(|edge| edge.as_ref().map_or((0, 0), |edge| (edge.n, edge.o))) + .collect::>(); + (action, root.visits, child_stats) + }) + } + + let one = run_with_threads(1); + let two = run_with_threads(2); + let four = run_with_threads(4); + + assert_eq!(one, two); + assert_eq!(two, four); + } +} diff --git a/crates/infotheory/src/aixi/mcts/rho_uct.rs b/crates/infotheory/src/aixi/mcts/rho_uct.rs new file mode 100644 index 00000000..6af7e056 --- /dev/null +++ b/crates/infotheory/src/aixi/mcts/rho_uct.rs @@ -0,0 +1,814 @@ +use super::{ + AgentSimulator, PerceptMap, PerceptOutcome, best_action_from_action_values, + choose_uniform_unvisited, ensure_action_slots, prune_key, random_rollout, +}; +#[cfg(test)] +use crate::aixi::common::ActionAlphabet; +use crate::aixi::common::{Action, PerceptVal, Reward}; + +/// Sequential `rho_uct` planner state. +/// +/// The first-visit decision-node shortcut intentionally follows +/// `aixictwx.tex` Algorithm 2: when `T(h) = 0`, the planner performs a +/// rollout from `h` rather than forcing an action/chance expansion first. +pub struct RhoUctPlanner { + root: Option, +} + +impl RhoUctPlanner { + pub fn new() -> Self { + Self { + root: Some(RhoUctNode::new(false)), + } + } + + pub fn search( + &mut self, + agent: &mut dyn AgentSimulator, + prev_obs_stream: &[PerceptVal], + prev_rew: Reward, + prev_act: Action, + samples: usize, + ) -> Action { + self.prune_tree(agent, prev_obs_stream, prev_rew, prev_act); + + let horizon = agent.horizon(); + let root = self.root.as_mut().expect("rho_uct root missing"); + for _ in 0..samples { + agent.begin_simulation(); + root.sample(agent, horizon, horizon); + } + root.best_action(agent) + } + + fn prune_tree( + &mut self, + agent: &dyn AgentSimulator, + prev_obs_stream: &[PerceptVal], + prev_rew: Reward, + prev_act: Action, + ) { + let Some(mut old_root) = self.root.take() else { + self.root = Some(RhoUctNode::new(false)); + return; + }; + + let action_child = old_root + .action_children + .get_mut(prev_act as usize) + .and_then(Option::take); + + let Some(mut chance_child) = action_child else { + self.root = Some(RhoUctNode::new(false)); + return; + }; + + let key = prune_key(agent, prev_obs_stream, prev_rew); + self.root = chance_child + .percept_children + .remove(&key) + .or_else(|| Some(RhoUctNode::new(false))); + } +} + +impl Default for RhoUctPlanner { + fn default() -> Self { + Self::new() + } +} + +#[derive(Clone)] +struct RhoUctNode { + visits: u32, + mean: f64, + is_chance_node: bool, + action_children: Vec>, + percept_children: PerceptMap, +} + +impl RhoUctNode { + fn new(is_chance_node: bool) -> Self { + Self { + visits: 0, + mean: 0.0, + is_chance_node, + action_children: Vec::new(), + percept_children: PerceptMap::default(), + } + } + + fn best_action(&self, agent: &mut dyn AgentSimulator) -> Action { + best_action_from_action_values( + self.action_children + .iter() + .enumerate() + .filter_map(|(action_idx, child)| { + child.as_ref().map(|node| (action_idx, node.mean)) + }), + agent.get_num_actions(), + agent, + ) + } + + fn sample( + &mut self, + agent: &mut dyn AgentSimulator, + remaining_horizon: usize, + total_horizon: usize, + ) -> f64 { + if remaining_horizon == 0 { + agent.model_revert(total_horizon); + return 0.0; + } + + let reward = if self.is_chance_node { + let (observations, immediate_reward) = agent.gen_percepts_and_update(); + let key = PerceptOutcome::new(observations, immediate_reward); + let child = self + .percept_children + .entry(key) + .or_insert_with(|| RhoUctNode::new(false)); + (immediate_reward as f64) + + agent.discount_gamma() * child.sample(agent, remaining_horizon - 1, total_horizon) + } else if self.visits == 0 { + let reward = random_rollout(agent, remaining_horizon); + agent.model_revert(total_horizon); + reward + } else { + let (child, _action) = self.select_action(agent, remaining_horizon); + child.sample(agent, remaining_horizon, total_horizon) + }; + + self.mean = (reward + (self.visits as f64) * self.mean) / ((self.visits + 1) as f64); + self.visits += 1; + reward + } + + fn select_action( + &mut self, + agent: &mut dyn AgentSimulator, + remaining_horizon: usize, + ) -> (&mut RhoUctNode, Action) { + let num_actions = agent.get_num_actions(); + ensure_action_slots(&mut self.action_children, num_actions.get()); + + let action_idx = if let Some(unvisited) = + choose_uniform_unvisited(agent, &self.action_children, num_actions.get()) + { + self.action_children[unvisited] = Some(RhoUctNode::new(true)); + unvisited + } else { + let log_visits = (self.visits as f64).ln().max(0.0); + let c = agent.get_explore_exploit_ratio().max(0.0); + let mut best_score = -f64::INFINITY; + let mut best_action = None; + let mut num_maximal_actions = 0usize; + + for (action_idx, child) in self.action_children.iter().enumerate() { + let Some(child) = child.as_ref() else { + continue; + }; + let normalized_value = agent.norm_reward_for_horizon(child.mean, remaining_horizon); + let exploration = if child.visits == 0 { + f64::INFINITY + } else { + c * (log_visits / (child.visits as f64)).sqrt() + }; + let score = normalized_value + exploration; + debug_assert!( + score.is_finite(), + "rho_uct UCB score must be finite for visited action children" + ); + match score.total_cmp(&best_score) { + std::cmp::Ordering::Greater => { + best_score = score; + best_action = Some(action_idx); + num_maximal_actions = 1; + } + std::cmp::Ordering::Equal => { + num_maximal_actions += 1; + if agent.gen_range(num_maximal_actions) == 0 { + best_action = Some(action_idx); + } + } + std::cmp::Ordering::Less => {} + } + } + + best_action.expect("rho_uct decision node must have a maximal action") + }; + + let action = action_idx as Action; + agent.model_update_action(action); + ( + self.action_children[action_idx] + .as_mut() + .expect("rho_uct action child missing"), + action, + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::aixi::common::ObservationKeyMode; + + #[derive(Clone)] + struct DummyAgent { + num_actions: usize, + obs_bits: usize, + rew_bits: usize, + horizon: usize, + min_reward: Reward, + max_reward: Reward, + discount_gamma: f64, + explore_exploit_ratio: f64, + key_mode: ObservationKeyMode, + last_range_end: usize, + range_result: usize, + } + + impl DummyAgent { + fn new(obs_bits: usize, key_mode: ObservationKeyMode) -> Self { + Self { + num_actions: 4, + obs_bits, + rew_bits: 8, + horizon: 5, + min_reward: -1, + max_reward: 1, + discount_gamma: 1.0, + explore_exploit_ratio: 1.0, + key_mode, + last_range_end: 0, + range_result: 0, + } + } + } + + impl AgentSimulator for DummyAgent { + fn get_num_actions(&self) -> ActionAlphabet { + ActionAlphabet::try_from_usize(self.num_actions) + .expect("test fixture action alphabet must be valid") + } + + fn get_num_observation_bits(&self) -> usize { + self.obs_bits + } + + fn observation_key_mode(&self) -> ObservationKeyMode { + self.key_mode + } + + fn get_num_reward_bits(&self) -> usize { + self.rew_bits + } + + fn horizon(&self) -> usize { + self.horizon + } + + fn max_reward(&self) -> Reward { + self.max_reward + } + + fn min_reward(&self) -> Reward { + self.min_reward + } + + fn discount_gamma(&self) -> f64 { + self.discount_gamma + } + + fn get_explore_exploit_ratio(&self) -> f64 { + self.explore_exploit_ratio + } + + fn model_update_action(&mut self, _action: Action) {} + + fn gen_percept_and_update(&mut self, _bits: usize) -> u64 { + 0 + } + + fn model_revert(&mut self, _steps: usize) {} + + fn gen_range(&mut self, end: usize) -> usize { + self.last_range_end = end; + self.range_result.min(end.saturating_sub(1)) + } + + fn gen_f64(&mut self) -> f64 { + 0.0 + } + + fn boxed_clone_with_seed(&self, _seed: u64) -> Box { + Box::new(self.clone()) + } + } + + fn build_planner_with_key( + agent: &DummyAgent, + prev_act: Action, + prev_obs_stream: &[PerceptVal], + prev_rew: Reward, + kept_mean: f64, + kept_visits: u32, + ) -> RhoUctPlanner { + let mut old_root = RhoUctNode::new(false); + old_root.action_children.resize(prev_act as usize + 1, None); + + let mut chance_child = RhoUctNode::new(true); + let mut kept = RhoUctNode::new(false); + kept.mean = kept_mean; + kept.visits = kept_visits; + + let key = prune_key(agent, prev_obs_stream, prev_rew); + chance_child.percept_children.insert(key, kept); + old_root.action_children[prev_act as usize] = Some(chance_child); + + RhoUctPlanner { + root: Some(old_root), + } + } + + #[test] + fn prune_tree_keeps_matching_subtree() { + let prev_act = 2u64; + let prev_obs_stream = vec![9u64, 2u64, 7u64]; + let prev_rew: Reward = 3; + + let agent = DummyAgent::new(3, ObservationKeyMode::FullStream); + let mut planner = + build_planner_with_key(&agent, prev_act, &prev_obs_stream, prev_rew, 123.0, 7); + + planner.prune_tree(&agent, &prev_obs_stream, prev_rew, prev_act); + + let root = planner.root.as_ref().expect("rho_uct root should exist"); + assert!(!root.is_chance_node); + assert_eq!(root.mean, 123.0); + assert_eq!(root.visits, 7); + } + + #[test] + fn prune_tree_resets_when_action_missing() { + let prev_act = 10u64; + let prev_obs_stream = vec![1u64]; + let prev_rew: Reward = 0; + + let agent = DummyAgent::new(1, ObservationKeyMode::FullStream); + let mut planner = RhoUctPlanner::new(); + + planner.prune_tree(&agent, &prev_obs_stream, prev_rew, prev_act); + + let root = planner.root.as_ref().expect("rho_uct root"); + assert!(!root.is_chance_node); + assert_eq!(root.visits, 0); + assert_eq!(root.mean, 0.0); + } + + #[test] + fn prune_tree_resets_when_reward_mismatch_shares_observation_key() { + let prev_act = 1u64; + let prev_obs_stream = vec![4u64, 5u64]; + let kept_rew: Reward = -2; + let requested_rew: Reward = 2; + + let agent = DummyAgent::new(6, ObservationKeyMode::FullStream); + let mut planner = + build_planner_with_key(&agent, prev_act, &prev_obs_stream, kept_rew, 77.0, 11); + + planner.prune_tree(&agent, &prev_obs_stream, requested_rew, prev_act); + + let root = planner.root.as_ref().expect("rho_uct root"); + assert_eq!(root.visits, 0); + assert_eq!(root.mean, 0.0); + } + + #[test] + fn deeper_remaining_horizon_changes_ucb_normalization() { + let mut root = RhoUctNode::new(false); + root.visits = 32; + root.action_children = vec![ + Some(RhoUctNode { + visits: 8, + mean: 2.0, + is_chance_node: true, + action_children: Vec::new(), + percept_children: PerceptMap::default(), + }), + Some(RhoUctNode { + visits: 8, + mean: 2.0, + is_chance_node: true, + action_children: Vec::new(), + percept_children: PerceptMap::default(), + }), + ]; + + let mut shallow = DummyAgent::new(1, ObservationKeyMode::FullStream); + shallow.horizon = 2; + shallow.min_reward = -2; + shallow.max_reward = 3; + + let mut deep = shallow.clone(); + deep.horizon = 8; + + let normalized_shallow = + shallow.norm_reward_for_horizon(root.action_children[0].as_ref().unwrap().mean, 2); + let normalized_deep = + deep.norm_reward_for_horizon(root.action_children[0].as_ref().unwrap().mean, 8); + assert!( + normalized_shallow > normalized_deep, + "same mean return should normalize differently when the remaining horizon changes" + ); + } + + #[test] + fn discounted_normalization_uses_remaining_horizon_bounds() { + let mut agent = DummyAgent::new(1, ObservationKeyMode::FullStream); + agent.min_reward = -1; + agent.max_reward = 3; + agent.discount_gamma = 0.5; + + let two_step = agent.norm_reward_for_horizon(1.0, 2); + let four_step = agent.norm_reward_for_horizon(1.0, 4); + assert_ne!(two_step, four_step); + } + + #[test] + fn undiscounted_normalization_is_action_equivalent_to_aixictwx_scaling() { + let mut agent = DummyAgent::new(1, ObservationKeyMode::FullStream); + agent.min_reward = -2; + agent.max_reward = 3; + agent.discount_gamma = 1.0; + + let remaining_horizon = 4usize; + let lower_value = -1.5; + let higher_value = 0.25; + let normalized_order = agent + .norm_reward_for_horizon(lower_value, remaining_horizon) + .total_cmp(&agent.norm_reward_for_horizon(higher_value, remaining_horizon)); + let paper_range = + (remaining_horizon as f64) * ((agent.max_reward - agent.min_reward) as f64); + let paper_order = (lower_value / paper_range).total_cmp(&(higher_value / paper_range)); + + assert_eq!(normalized_order, paper_order); + } + + #[test] + fn discounted_normalization_matches_discounted_finite_horizon_formula() { + let mut agent = DummyAgent::new(1, ObservationKeyMode::FullStream); + agent.min_reward = -1; + agent.max_reward = 3; + agent.discount_gamma = 0.5; + + let horizon = 4usize; + let reward = 1.0; + let discounted_sum = 1.0 + 0.5 + 0.25 + 0.125; + let min_cumulative = -discounted_sum; + let max_cumulative = 3.0 * discounted_sum; + let expected = (reward - min_cumulative) / (max_cumulative - min_cumulative); + + assert!((agent.norm_reward_for_horizon(reward, horizon) - expected).abs() < 1e-12); + } + + #[test] + fn rho_uct_select_action_uses_remaining_horizon_ucb_scaling() { + let mut root = RhoUctNode::new(false); + root.visits = 16; + root.action_children = vec![ + Some(RhoUctNode { + visits: 4, + mean: -1.5, + is_chance_node: true, + action_children: Vec::new(), + percept_children: PerceptMap::default(), + }), + Some(RhoUctNode { + visits: 8, + mean: -0.5, + is_chance_node: true, + action_children: Vec::new(), + percept_children: PerceptMap::default(), + }), + ]; + + let mut agent = DummyAgent::new(1, ObservationKeyMode::FullStream); + agent.num_actions = 2; + agent.horizon = 5; + agent.min_reward = -2; + agent.max_reward = 3; + agent.discount_gamma = 0.5; + agent.explore_exploit_ratio = 0.25; + agent.range_result = 0; + + let remaining_horizon = 3usize; + let log_visits = (root.visits as f64).ln(); + let c = agent.get_explore_exploit_ratio(); + let left_score = + agent.norm_reward_for_horizon(-1.5, remaining_horizon) + c * (log_visits / 4.0).sqrt(); + let right_score = + agent.norm_reward_for_horizon(-0.5, remaining_horizon) + c * (log_visits / 8.0).sqrt(); + assert!( + right_score > left_score, + "expected action 1 to win under the rho_uct paper formula" + ); + + let (_child, action) = root.select_action(&mut agent, remaining_horizon); + assert_eq!(action, 1); + } + + #[test] + fn rho_uct_select_action_breaks_exact_ties_uniformly_via_agent_rng() { + let mut root = RhoUctNode::new(false); + root.visits = 16; + root.action_children = vec![ + Some(RhoUctNode { + visits: 4, + mean: 1.0, + is_chance_node: true, + action_children: Vec::new(), + percept_children: PerceptMap::default(), + }), + Some(RhoUctNode { + visits: 4, + mean: 1.0, + is_chance_node: true, + action_children: Vec::new(), + percept_children: PerceptMap::default(), + }), + ]; + + let mut agent = DummyAgent::new(1, ObservationKeyMode::FullStream); + agent.num_actions = 2; + agent.min_reward = 0; + agent.max_reward = 2; + agent.range_result = 0; + + let (_child, action) = root.select_action(&mut agent, 2); + assert_eq!(agent.last_range_end, 2); + assert_eq!(action, 1); + } + + #[derive(Clone)] + struct DeterministicRewardAgent { + last_action: Action, + emit_reward: bool, + } + + impl AgentSimulator for DeterministicRewardAgent { + fn get_num_actions(&self) -> ActionAlphabet { + ActionAlphabet::try_from_usize(2).expect("test fixture action alphabet must be valid") + } + + fn get_num_observation_bits(&self) -> usize { + 1 + } + + fn get_num_reward_bits(&self) -> usize { + 1 + } + + fn horizon(&self) -> usize { + 1 + } + + fn max_reward(&self) -> Reward { + 1 + } + + fn min_reward(&self) -> Reward { + 0 + } + + fn get_explore_exploit_ratio(&self) -> f64 { + 0.0 + } + + fn model_update_action(&mut self, action: Action) { + self.last_action = action; + self.emit_reward = false; + } + + fn gen_percept_and_update(&mut self, _bits: usize) -> u64 { + if self.emit_reward { + self.emit_reward = false; + self.last_action + } else { + self.emit_reward = true; + 0 + } + } + + fn model_revert(&mut self, _steps: usize) { + self.emit_reward = false; + } + + fn gen_range(&mut self, _end: usize) -> usize { + 0 + } + + fn gen_f64(&mut self) -> f64 { + 0.0 + } + + fn boxed_clone_with_seed(&self, _seed: u64) -> Box { + Box::new(self.clone()) + } + } + + #[test] + fn rho_uct_is_not_promoted_to_parallel_for_multiple_samples() { + let mut agent = DeterministicRewardAgent { + last_action: 0, + emit_reward: false, + }; + let mut planner = RhoUctPlanner::new(); + let action = planner.search(&mut agent, &[0], 0, 0, 8); + let root = planner.root.as_ref().expect("rho_uct root"); + assert!(action < 2); + assert_eq!(root.visits, 8); + assert!( + root.action_children + .iter() + .filter_map(Option::as_ref) + .any(|child| child.visits > 0), + "sequential rho_uct should accumulate completed action visits directly in the shared tree" + ); + } + + #[derive(Clone)] + struct RolloutProbeAgent { + num_actions: usize, + revert_calls: usize, + last_revert_steps: usize, + action_updates: usize, + } + + impl AgentSimulator for RolloutProbeAgent { + fn get_num_actions(&self) -> ActionAlphabet { + ActionAlphabet::try_from_usize(self.num_actions) + .expect("test fixture action alphabet must be valid") + } + + fn get_num_observation_bits(&self) -> usize { + 1 + } + + fn get_num_reward_bits(&self) -> usize { + 1 + } + + fn horizon(&self) -> usize { + 3 + } + + fn max_reward(&self) -> Reward { + 1 + } + + fn min_reward(&self) -> Reward { + 0 + } + + fn get_explore_exploit_ratio(&self) -> f64 { + 0.0 + } + + fn model_update_action(&mut self, _action: Action) { + self.action_updates += 1; + } + + fn gen_percept_and_update(&mut self, _bits: usize) -> u64 { + 0 + } + + fn model_revert(&mut self, steps: usize) { + self.revert_calls += 1; + self.last_revert_steps = steps; + } + + fn gen_range(&mut self, _end: usize) -> usize { + 0 + } + + fn gen_f64(&mut self) -> f64 { + 0.0 + } + + fn boxed_clone_with_seed(&self, _seed: u64) -> Box { + Box::new(self.clone()) + } + } + + #[test] + fn rho_uct_first_visit_decision_node_rolls_out_before_tree_expansion() { + let mut root = RhoUctNode::new(false); + let mut agent = RolloutProbeAgent { + num_actions: 2, + revert_calls: 0, + last_revert_steps: 0, + action_updates: 0, + }; + + let reward = root.sample(&mut agent, 3, 3); + assert_eq!(reward, 0.0); + assert_eq!(root.visits, 1); + assert!( + root.action_children.is_empty(), + "first-visit decision-node handling should perform a rollout without eagerly expanding action edges" + ); + assert!( + agent.action_updates > 0, + "rollout should still simulate actions before backup" + ); + assert_eq!(agent.revert_calls, 1); + assert_eq!(agent.last_revert_steps, 3); + } + + #[derive(Clone)] + struct ChanceNodeProbeAgent { + sequence: [u64; 2], + cursor: usize, + revert_calls: usize, + } + + impl AgentSimulator for ChanceNodeProbeAgent { + fn get_num_actions(&self) -> ActionAlphabet { + ActionAlphabet::try_from_usize(2).expect("test fixture action alphabet must be valid") + } + + fn get_num_observation_bits(&self) -> usize { + 1 + } + + fn get_num_reward_bits(&self) -> usize { + 1 + } + + fn horizon(&self) -> usize { + 1 + } + + fn max_reward(&self) -> Reward { + 1 + } + + fn min_reward(&self) -> Reward { + 0 + } + + fn discount_gamma(&self) -> f64 { + 0.5 + } + + fn get_explore_exploit_ratio(&self) -> f64 { + 0.0 + } + + fn model_update_action(&mut self, _action: Action) {} + + fn gen_percept_and_update(&mut self, _bits: usize) -> u64 { + let idx = self.cursor.min(self.sequence.len().saturating_sub(1)); + self.cursor = self.cursor.saturating_add(1); + self.sequence[idx] + } + + fn model_revert(&mut self, _steps: usize) { + self.revert_calls += 1; + } + + fn gen_range(&mut self, _end: usize) -> usize { + 0 + } + + fn gen_f64(&mut self) -> f64 { + 0.0 + } + + fn boxed_clone_with_seed(&self, _seed: u64) -> Box { + Box::new(self.clone()) + } + } + + #[test] + fn rho_uct_chance_node_sampling_backs_up_immediate_reward() { + let mut node = RhoUctNode::new(true); + let mut agent = ChanceNodeProbeAgent { + sequence: [0, 1], + cursor: 0, + revert_calls: 0, + }; + + let reward = node.sample(&mut agent, 1, 1); + assert_eq!(reward, 1.0); + assert_eq!(node.visits, 1); + assert!((node.mean - 1.0).abs() < 1e-12); + assert_eq!(node.percept_children.len(), 1); + assert_eq!( + agent.revert_calls, 1, + "chance-node child sample should terminate at horizon and revert once" + ); + } +} diff --git a/src/aixi/mod.rs b/crates/infotheory/src/aixi/mod.rs similarity index 54% rename from src/aixi/mod.rs rename to crates/infotheory/src/aixi/mod.rs index 7a378421..fb5a1595 100644 --- a/src/aixi/mod.rs +++ b/crates/infotheory/src/aixi/mod.rs @@ -12,16 +12,38 @@ //! //! ## VM Backend //! -//! A high-performance Firecracker-based VM environment is available via the `vm` feature: +//! A high-performance Firecracker-based VM environment is available via the `aixi-vm` +//! (or legacy alias `vm`) feature: //! //! - **NyxVmEnvironment**: Uses nyx-lite for 10,000+ resets/second (requires KVM). +#[cfg(feature = "aixi")] pub mod agent; +#[cfg(feature = "aixi")] pub mod aiqi; pub mod common; +#[cfg(feature = "aixi")] pub mod environment; +#[cfg(all(feature = "aixi", feature = "aixi-gameengine"))] +pub mod gameengine; +#[cfg(feature = "aixi")] pub mod mcts; +#[cfg(feature = "aixi")] pub mod model; -pub(crate) mod rate_backend; -#[cfg(feature = "vm")] +#[cfg(feature = "aixi")] +pub mod planner_agent; +#[cfg(feature = "aixi")] +pub(crate) mod planner_runtime; +#[cfg(feature = "aixi")] +pub(crate) mod planner_spec; +#[cfg(feature = "aixi")] +pub(crate) mod return_law; +#[cfg(all(test, feature = "aixi"))] +pub(crate) mod test_envs; +#[cfg(all(feature = "aixi", feature = "vm"))] pub mod vm_nyx; +#[cfg(feature = "aixi")] +pub mod warmstart; +#[cfg(feature = "aixi")] +#[doc(hidden)] +pub mod warmstart_contract; diff --git a/src/aixi/model.rs b/crates/infotheory/src/aixi/model.rs similarity index 51% rename from src/aixi/model.rs rename to crates/infotheory/src/aixi/model.rs index 44bf1c26..97e8836f 100644 --- a/src/aixi/model.rs +++ b/crates/infotheory/src/aixi/model.rs @@ -4,18 +4,35 @@ //! for learning from history and predicting future symbols. Different implementations //! provide different complexity vs performance trade-offs. -use crate::RateBackend; -use crate::aixi::rate_backend::rate_backend_contains_zpaq; -use crate::ctw::{ContextTree, FacContextTree}; +use crate::api::{ + BitStreamSemantics, CompiledRateBackend, RateBackend, RateBackendBitSession, + RateBackendBitSessionCheckpoint, +}; +#[cfg(feature = "backend-ctw")] +use crate::backends::ctw::{ContextTree, FacContextTree}; +#[cfg(feature = "backend-rosa")] +use crate::backends::rosaplus::{RosaPlus, RosaTx}; +#[cfg(feature = "backend-zpaq")] +use crate::backends::zpaq_rate::ZpaqRateModel; +#[cfg(any(feature = "backend-mamba", feature = "backend-rwkv"))] +use crate::error::{InfotheoryError, InfotheoryResult}; #[cfg(feature = "backend-mamba")] use crate::mambazip::{Compressor as MambaCompressor, Model as MambaModel, State as MambaState}; use crate::mixture::{ DEFAULT_MIN_PROB, OnlineBytePredictor, RateBackendPredictor, RateBackendPredictorCheckpoint, }; -use crate::rosaplus::{RosaPlus, RosaTx}; +use crate::prediction::binary_prediction_from_log_probs; +#[cfg(any( + feature = "backend-rosa", + feature = "backend-mamba", + feature = "backend-rwkv" +))] +use crate::prediction::binary_prediction_from_probs; #[cfg(feature = "backend-rwkv")] use crate::rwkvzip::{Compressor as RwkvCompressor, Model as RwkvModel, State as RwkvState}; -use crate::zpaq_rate::ZpaqRateModel; +use crate::spec::SpecError; +use std::error::Error; +use std::fmt; #[cfg(any(feature = "backend-mamba", feature = "backend-rwkv"))] use std::sync::Arc; @@ -60,6 +77,29 @@ pub trait Predictor: Send { /// until the matching `rollback_scope` call. fn begin_rollback_scope(&mut self) {} + /// Whether [`Self::begin_rollback_scope`] and [`Self::rollback_scope`] are supported. + /// + /// This is a capability advertisement for external or future planner + /// strategies that need to choose between scoped and per-symbol rollback + /// before mutating the predictor. The built-in planners do not currently + /// need the probe: MCTS attempts the scope and falls back based on + /// [`Self::rollback_scope`]'s return value, while return-law evaluation + /// deliberately uses balanced per-symbol rollback. + fn supports_rollback_scope(&self) -> bool { + false + } + + /// Begins a simulation scope whose mutated predictor state will be discarded. + /// + /// This is useful for cloned MCTS workers: they need speculative updates to + /// avoid per-symbol rollback journals, but they do not need a checkpoint that + /// can restore the clone because the clone is dropped after the rollout. + /// Implementations that do not provide a specialized discardable mode fall + /// back to a normal reversible rollback scope. + fn begin_discardable_scope(&mut self) { + self.begin_rollback_scope(); + } + /// Rolls back to the last scope opened with `begin_rollback_scope`. /// /// Returns `true` when a scope rollback was performed, allowing callers to skip @@ -81,57 +121,42 @@ pub trait Predictor: Send { /// Creates a boxed clone of this predictor. fn boxed_clone(&self) -> Box; -} - -#[inline] -fn binary_prob_floor(min_prob: f64) -> f64 { - if min_prob.is_finite() { - min_prob.clamp(1e-12, 0.499_999_999_999) - } else { - 1e-12 - } -} - -#[inline] -fn normalized_binary_prob_pair_from_probs(p0: f64, p1: f64, min_prob: f64) -> (f64, f64) { - let p0 = if p0.is_finite() && p0 > 0.0 { p0 } else { 0.0 }; - let p1 = if p1.is_finite() && p1 > 0.0 { p1 } else { 0.0 }; - let sum = p0 + p1; - if !sum.is_finite() || sum <= 0.0 { - return (0.5, 0.5); - } - let floor = binary_prob_floor(min_prob); - let q1 = (p1 / sum).clamp(floor, 1.0 - floor); - (1.0 - q1, q1) -} -#[inline] -fn normalized_binary_prob_pair_from_log_probs(logp0: f64, logp1: f64, min_prob: f64) -> (f64, f64) { - let max_log = logp0.max(logp1); - if !max_log.is_finite() { - return (0.5, 0.5); + /// Clear transient conditioning history while preserving all committed learning state. + /// + /// Called between independent teacher traces during warm-start to prevent the terminal + /// conditioning context of one trace from influencing predictions at the start of the next. + /// + /// # Contract + /// + /// After a successful call, the predictor must behave as if it were freshly initialized + /// with respect to context-dependent predictions (e.g. the sliding-window context suffix + /// used to navigate a CTW tree is reset to empty). It must retain **all** committed + /// learned model state induced by prior updates, while clearing only transient + /// conditioning context. Implementations that clear learned counts (e.g. by calling + /// a full `clear()`) violate this contract. + /// + /// # Errors + /// + /// Return `Err` only when the backend has no meaningful way to isolate conditioning + /// state from learned parameters (e.g. a fully stateful streaming model where the + /// two are inseparable). The default no-op is appropriate for backends whose context + /// is already isolated or resets naturally. + fn reset_conditioning_history(&mut self) -> Result<(), String> { + Ok(()) } - let p0 = if logp0.is_finite() { - (logp0 - max_log).exp() - } else { - 0.0 - }; - let p1 = if logp1.is_finite() { - (logp1 - max_log).exp() - } else { - 0.0 - }; - normalized_binary_prob_pair_from_probs(p0, p1, min_prob) } /// A predictor using the Action-Conditional CTW algorithm. /// /// AC-CTW uses a single context tree for all bits in sequence. /// For better type information exploitation, use `FacCtwPredictor`. +#[cfg(feature = "backend-ctw")] pub struct CtwPredictor { tree: ContextTree, } +#[cfg(feature = "backend-ctw")] impl CtwPredictor { /// Creates a new `CtwPredictor` with the specified context depth. pub fn new(depth: usize) -> Self { @@ -141,6 +166,7 @@ impl CtwPredictor { } } +#[cfg(feature = "backend-ctw")] impl Predictor for CtwPredictor { fn update(&mut self, sym: bool) { self.tree.update(sym); @@ -160,6 +186,10 @@ impl Predictor for CtwPredictor { self.tree.predict(sym) } + fn predict_one(&mut self) -> f64 { + self.tree.predict_one() + } + fn model_name(&self) -> String { format!("AC-CTW(d={})", self.tree.depth()) } @@ -169,6 +199,11 @@ impl Predictor for CtwPredictor { tree: self.tree.clone(), }) } + + fn reset_conditioning_history(&mut self) -> Result<(), String> { + self.tree.truncate_history(0); + Ok(()) + } } /// A predictor using the Factorized Action-Conditional CTW (FAC-CTW) algorithm. @@ -178,6 +213,7 @@ impl Predictor for CtwPredictor { /// information within percepts, as described in Veness et al. (2011) Section 5. /// /// This is the recommended CTW variant for MC-AIXI agents. +#[cfg(feature = "backend-ctw")] pub struct FacCtwPredictor { tree: FacContextTree, /// Current bit index within a percept (cycles 0..num_bits). @@ -186,6 +222,7 @@ pub struct FacCtwPredictor { num_bits: usize, } +#[cfg(feature = "backend-ctw")] impl FacCtwPredictor { /// Creates a new `FacCtwPredictor`. /// @@ -200,6 +237,7 @@ impl FacCtwPredictor { } } +#[cfg(feature = "backend-ctw")] impl Predictor for FacCtwPredictor { fn update(&mut self, sym: bool) { self.tree.update(sym, self.current_bit); @@ -228,6 +266,10 @@ impl Predictor for FacCtwPredictor { self.tree.predict(sym, self.current_bit) } + fn predict_one(&mut self) -> f64 { + self.tree.predict_one(self.current_bit) + } + fn model_name(&self) -> String { format!("FAC-CTW(D={}, k={})", self.tree.base_depth(), self.num_bits) } @@ -239,17 +281,25 @@ impl Predictor for FacCtwPredictor { num_bits: self.num_bits, }) } + + fn reset_conditioning_history(&mut self) -> Result<(), String> { + self.tree.reset_history_only(); + self.current_bit = 0; + Ok(()) + } } /// A predictor using the ROSA-Plus (Rapid Online Suffix Automaton + Witten-Bell Smoother) algorithm. /// /// ROSA is a (practically) sub-quadratic suffix automaton based language model that /// can handle very long contexts efficiently. +#[cfg(feature = "backend-rosa")] pub struct RosaPredictor { model: RosaPlus, history: Vec, } +#[cfg(feature = "backend-rosa")] impl RosaPredictor { /// Creates a new `RosaPredictor` with a maximum context length for the fallback LM. /// Note: deterministic ROSA uses the full SAM and is not capped by `max_order`. @@ -265,6 +315,7 @@ impl RosaPredictor { } } +#[cfg(feature = "backend-rosa")] impl Predictor for RosaPredictor { fn update(&mut self, sym: bool) { let mut tx = self.model.begin_tx(); @@ -283,12 +334,12 @@ impl Predictor for RosaPredictor { } fn predict_prob(&mut self, sym: bool) -> f64 { - let (p0, p1) = normalized_binary_prob_pair_from_probs( + binary_prediction_from_probs( self.model.prob_for_last(0), self.model.prob_for_last(1), DEFAULT_MIN_PROB, - ); - if sym { p1 } else { p0 } + ) + .prob(sym) } fn model_name(&self) -> String { @@ -301,12 +352,21 @@ impl Predictor for RosaPredictor { history: self.history.clone(), }) } + + fn reset_conditioning_history(&mut self) -> Result<(), String> { + // Preserve trained SAM/LM parameters while dropping transient cursor and + // rollback journal state so independent traces start from empty context. + self.model.reset_conditioning_cursor(); + self.history.clear(); + Ok(()) + } } /// A predictor using ZPAQ as a streaming rate model. /// /// This maintains a full history so it can rebuild state on revert and handle /// any misuse where `predict_prob` is called without a matching `update`. +#[cfg(feature = "backend-zpaq")] pub struct ZpaqPredictor { method: String, min_prob: f64, @@ -315,6 +375,7 @@ pub struct ZpaqPredictor { pending: Option<(u8, f64)>, } +#[cfg(feature = "backend-zpaq")] impl ZpaqPredictor { /// Create a ZPAQ-backed predictor from a `method` and probability floor. pub fn new(method: String, min_prob: f64) -> Self { @@ -366,6 +427,7 @@ impl ZpaqPredictor { } } +#[cfg(feature = "backend-zpaq")] impl Predictor for ZpaqPredictor { fn update(&mut self, sym: bool) { let byte = if sym { 1u8 } else { 0u8 }; @@ -393,8 +455,7 @@ impl Predictor for ZpaqPredictor { fn predict_prob(&mut self, sym: bool) -> f64 { let preferred_symbol = if sym { 1u8 } else { 0u8 }; let (logp0, logp1) = self.binary_log_prob_pair(preferred_symbol); - let (p0, p1) = normalized_binary_prob_pair_from_log_probs(logp0, logp1, self.min_prob); - if sym { p1 } else { p0 } + binary_prediction_from_log_probs(logp0, logp1, self.min_prob).prob(sym) } fn model_name(&self) -> String { @@ -412,18 +473,156 @@ impl Predictor for ZpaqPredictor { } } -/// A generic bit-level predictor backed by any [`RateBackend`]. +/// Default AIXI bit-stream interpretation for generic rate backends. +/// +/// Planner interfaces usually expose one-bit actions, observations, rewards, +/// and labels. Binary-token semantics is therefore the planner-safe default; +/// byte-packed semantics remains available when every interface segment is +/// explicitly byte-aligned. +pub fn default_aixi_bit_stream_semantics() -> BitStreamSemantics { + BitStreamSemantics::BinaryTokens +} + +/// A generic bit-level predictor backed by the shared [`RateBackendBitSession`]. /// -/// This adapter maps boolean symbols to bytes `{0,1}` and forwards them to the -/// workspace-wide rate backend abstraction. It prioritizes correctness and -/// backend coverage over rollback efficiency. +/// This bridge owns only AIXI rollback bookkeeping. Bit prediction semantics, +/// byte-prefix factorization, and backend-specific stream behavior stay in the +/// library-wide session API. pub struct RateBackendBitPredictor { - backend: RateBackend, - max_order: i64, - min_prob: f64, - predictor: RateBackendPredictor, + backend: CompiledRateBackend, + semantics: BitStreamSemantics, + state: RateBackendBitPredictorState, journal: Vec, rollback_scopes: Vec, + discardable_scopes: usize, +} + +#[derive(Clone)] +enum RateBackendBitPredictorState { + BinaryTokens { + predictor: Box, + min_prob: f64, + }, + Session(Box), +} + +/// Error returned while constructing or initializing a rate-backend bit predictor. +#[derive(Debug)] +#[non_exhaustive] +pub enum RateBackendBitPredictorError { + /// Backend compilation failed. + Compile(SpecError), + /// ZPAQ-backed predictors cannot satisfy reversible bit-predictor semantics. + UnsupportedZpaq, + /// Runtime predictor construction failed. + Runtime(String), + /// Predictor stream initialization failed. + StreamStart(String), +} + +/// Error returned while constructing a predictor from a compiled rate backend. +#[derive(Debug)] +#[non_exhaustive] +pub enum PredictorBuildError { + /// Generic bit-level predictor construction failed. + BitPredictor(RateBackendBitPredictorError), +} + +impl fmt::Display for PredictorBuildError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::BitPredictor(err) => write!(f, "{err}"), + } + } +} + +impl Error for PredictorBuildError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::BitPredictor(err) => Some(err), + } + } +} + +impl From for PredictorBuildError { + fn from(value: RateBackendBitPredictorError) -> Self { + Self::BitPredictor(value) + } +} + +impl fmt::Display for RateBackendBitPredictorError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Compile(err) => write!(f, "{err}"), + Self::UnsupportedZpaq => { + f.write_str("RateBackendBitPredictor does not support zpaq backends; use a non-zpaq rate_backend") + } + Self::Runtime(err) => write!(f, "{err}"), + Self::StreamStart(err) => { + write!(f, "failed to start RateBackend predictor stream: {err}") + } + } + } +} + +impl Error for RateBackendBitPredictorError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Compile(err) => Some(err), + _ => None, + } + } +} + +impl From for RateBackendBitPredictorError { + fn from(value: SpecError) -> Self { + Self::Compile(value) + } +} + +/// Configuration for constructing a [`RateBackendBitPredictor`]. +#[derive(Clone)] +#[non_exhaustive] +pub struct RateBackendBitPredictorConfig { + /// Compiled backend used by the bit-level adapter. + pub backend: CompiledRateBackend, + /// Probability floor used when normalizing binary probabilities. + pub min_prob: f64, + /// Bit-stream semantics used by the shared bit session. + pub semantics: BitStreamSemantics, +} + +impl RateBackendBitPredictorConfig { + /// Compile a rate backend into a bit-predictor configuration. + pub fn compile( + backend: RateBackend, + min_prob: f64, + ) -> Result { + let compiled = backend + .compile() + .map_err(RateBackendBitPredictorError::from)?; + Ok(Self { + backend: compiled, + min_prob, + semantics: default_aixi_bit_stream_semantics(), + }) + } + + /// Compile a rate backend into a bit-predictor configuration with explicit semantics. + pub fn compile_with_semantics( + backend: RateBackend, + min_prob: f64, + semantics: BitStreamSemantics, + ) -> Result { + let compiled = backend + .compile() + .map_err(RateBackendBitPredictorError::from)?; + Ok(Self { + backend: compiled, + min_prob, + semantics, + }) + } } #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -435,72 +634,193 @@ enum RateBackendJournalKind { #[derive(Clone)] struct RateBackendJournalEntry { kind: RateBackendJournalKind, - checkpoint: RateBackendPredictorCheckpoint, + checkpoint: RateBackendBitPredictorCheckpoint, } #[derive(Clone)] struct RateBackendRollbackScope { - checkpoint: RateBackendPredictorCheckpoint, + checkpoint: RateBackendBitPredictorCheckpoint, journal_len: usize, } -impl RateBackendBitPredictor { - /// Create a new bit-level adapter from a rate backend. - pub fn new(backend: RateBackend, max_order: i64) -> Result { - Self::new_with_min_prob(backend, max_order, DEFAULT_MIN_PROB) - } +#[derive(Clone)] +enum RateBackendBitPredictorCheckpoint { + BinaryTokens(RateBackendPredictorCheckpoint), + Session(Box), +} - /// Create a new bit-level adapter with an explicit probability floor. - pub fn new_with_min_prob( - backend: RateBackend, - max_order: i64, +impl RateBackendBitPredictorState { + fn binary_tokens( + backend: &CompiledRateBackend, min_prob: f64, - ) -> Result { - if rate_backend_contains_zpaq(&backend) { - return Err( - "RateBackendBitPredictor does not support zpaq backends; use a non-zpaq rate_backend" - .to_string(), - ); - } + ) -> Result { let mut predictor = - RateBackendPredictor::from_backend(backend.clone(), max_order, min_prob); + crate::runtime::build_rate_backend_binary_token_predictor(backend, min_prob) + .map_err(RateBackendBitPredictorError::Runtime)?; predictor .begin_stream(None) - .map_err(|err| format!("failed to start RateBackend predictor stream: {err}"))?; - Ok(Self { + .map_err(RateBackendBitPredictorError::StreamStart)?; + Ok(Self::BinaryTokens { + predictor: Box::new(predictor), + min_prob, + }) + } + + fn session( + backend: CompiledRateBackend, + semantics: BitStreamSemantics, + min_prob: f64, + ) -> Result { + RateBackendBitSession::from_backend_with_min_prob(backend, None, semantics, min_prob) + .map(Box::new) + .map(Self::Session) + .map_err(|err| RateBackendBitPredictorError::Runtime(err.to_string())) + } + + fn checkpoint(&mut self) -> RateBackendBitPredictorCheckpoint { + match self { + Self::BinaryTokens { predictor, .. } => { + RateBackendBitPredictorCheckpoint::BinaryTokens(predictor.checkpoint()) + } + Self::Session(session) => { + RateBackendBitPredictorCheckpoint::Session(Box::new(session.checkpoint())) + } + } + } + + fn restore_checkpoint(&mut self, checkpoint: &RateBackendBitPredictorCheckpoint) { + match (self, checkpoint) { + ( + Self::BinaryTokens { predictor, .. }, + RateBackendBitPredictorCheckpoint::BinaryTokens(checkpoint), + ) => predictor.restore_checkpoint(checkpoint), + (Self::Session(session), RateBackendBitPredictorCheckpoint::Session(checkpoint)) => { + session + .restore_checkpoint(checkpoint) + .expect("RateBackendBitPredictor session checkpoint must match its session"); + } + _ => panic!("RateBackendBitPredictor checkpoint kind mismatch"), + } + } + + fn clear_checkpoints_if_supported(&mut self) { + match self { + Self::BinaryTokens { predictor, .. } => predictor.clear_checkpoints_if_supported(), + Self::Session(session) => session.clear_checkpoints_if_supported(), + } + } + + fn update(&mut self, sym: bool) { + match self { + Self::BinaryTokens { predictor, .. } => predictor.update(u8::from(sym)), + Self::Session(session) => session + .try_observe_bit(sym) + .expect("RateBackendBitPredictor update must satisfy configured bit semantics"), + } + } + + fn update_frozen(&mut self, sym: bool) { + match self { + Self::BinaryTokens { predictor, .. } => predictor.update_frozen(u8::from(sym)), + Self::Session(session) => session.try_condition_bit(sym).expect( + "RateBackendBitPredictor conditioning update must satisfy configured bit semantics", + ), + } + } + + fn predict_prob(&mut self, sym: bool) -> f64 { + match self { + Self::BinaryTokens { + predictor, + min_prob, + } => binary_prediction_from_log_probs( + predictor.log_prob(0), + predictor.log_prob(1), + *min_prob, + ) + .prob(sym), + Self::Session(session) => session.predict_bit().prob(sym), + } + } + + fn begin_discardable_scope(&mut self) { + if let Self::Session(session) = self { + session.begin_discardable_scope(); + } + } + + fn clear_discardable_scopes(&mut self) { + if let Self::Session(session) = self { + session.clear_discardable_scopes(); + } + } + + fn reset_frozen(&mut self) -> Result<(), String> { + match self { + Self::BinaryTokens { predictor, .. } => predictor.reset_frozen(None), + Self::Session(session) => session.reset_frozen(None).map_err(|err| err.to_string()), + } + } +} + +impl RateBackendBitPredictor { + /// Create a new bit-level adapter. + pub fn new( + config: RateBackendBitPredictorConfig, + ) -> Result { + let RateBackendBitPredictorConfig { backend, - max_order, min_prob, - predictor, + semantics, + } = config; + if backend.contains_zpaq() { + return Err(RateBackendBitPredictorError::UnsupportedZpaq); + } + let state = match semantics { + BitStreamSemantics::BinaryTokens => { + RateBackendBitPredictorState::binary_tokens(&backend, min_prob)? + } + BitStreamSemantics::BytePacked { .. } => { + RateBackendBitPredictorState::session(backend.clone(), semantics, min_prob)? + } + }; + Ok(Self { + backend, + semantics, + state, journal: Vec::new(), rollback_scopes: Vec::new(), + discardable_scopes: 0, }) } - #[inline(always)] - fn bit_to_byte(sym: bool) -> u8 { - if sym { 1u8 } else { 0u8 } - } - fn clone_state(&self) -> Self { Self { backend: self.backend.clone(), - max_order: self.max_order, - min_prob: self.min_prob, - predictor: self.predictor.clone(), + semantics: self.semantics, + state: self.state.clone(), journal: self.journal.clone(), rollback_scopes: self.rollback_scopes.clone(), + discardable_scopes: self.discardable_scopes, } } + fn should_capture_symbol_checkpoint(&self) -> bool { + self.rollback_scopes.is_empty() && self.discardable_scopes == 0 + } + fn checkpoint(&mut self, kind: RateBackendJournalKind) -> RateBackendJournalEntry { RateBackendJournalEntry { kind, - checkpoint: self.predictor.checkpoint(), + checkpoint: self.state.checkpoint(), } } fn restore_last(&mut self, expected_kind: RateBackendJournalKind) { + assert_eq!( + self.discardable_scopes, 0, + "RateBackendBitPredictor per-symbol rollback after a discardable simulation scope is unsupported" + ); assert!( self.rollback_scopes.is_empty(), "RateBackendBitPredictor per-symbol rollback inside active scope is unsupported" @@ -514,36 +834,36 @@ impl RateBackendBitPredictor { "RateBackendBitPredictor rollback kind mismatch: expected {expected_kind:?}, got {:?}", entry.kind ); - self.predictor.restore_checkpoint(&entry.checkpoint); + self.state.restore_checkpoint(&entry.checkpoint); if self.rollback_scopes.is_empty() && self.journal.is_empty() { - self.predictor.clear_checkpoints_if_supported(); + self.state.clear_checkpoints_if_supported(); } } } impl Predictor for RateBackendBitPredictor { fn update(&mut self, sym: bool) { - if self.rollback_scopes.is_empty() { + if self.should_capture_symbol_checkpoint() { let checkpoint = self.checkpoint(RateBackendJournalKind::Update); self.journal.push(checkpoint); } - self.predictor.update(Self::bit_to_byte(sym)); + self.state.update(sym); } fn commit_update(&mut self, sym: bool) { - self.predictor.update(Self::bit_to_byte(sym)); + self.state.update(sym); } fn update_history(&mut self, sym: bool) { - if self.rollback_scopes.is_empty() { + if self.should_capture_symbol_checkpoint() { let checkpoint = self.checkpoint(RateBackendJournalKind::FrozenUpdate); self.journal.push(checkpoint); } - self.predictor.update_frozen(Self::bit_to_byte(sym)); + self.state.update_frozen(sym); } fn commit_update_history(&mut self, sym: bool) { - self.predictor.update_frozen(Self::bit_to_byte(sym)); + self.state.update_frozen(sym); } fn revert(&mut self) { @@ -555,44 +875,129 @@ impl Predictor for RateBackendBitPredictor { } fn begin_rollback_scope(&mut self) { - let checkpoint = self.predictor.checkpoint(); + assert_eq!( + self.discardable_scopes, 0, + "RateBackendBitPredictor cannot open a reversible rollback scope inside a discardable simulation scope" + ); + let checkpoint = self.state.checkpoint(); self.rollback_scopes.push(RateBackendRollbackScope { checkpoint, journal_len: self.journal.len(), }); } + fn supports_rollback_scope(&self) -> bool { + true + } + + fn begin_discardable_scope(&mut self) { + self.discardable_scopes = self.discardable_scopes.saturating_add(1); + self.state.begin_discardable_scope(); + } + fn rollback_scope(&mut self) -> bool { + assert_eq!( + self.discardable_scopes, 0, + "RateBackendBitPredictor rollback after a discardable simulation scope is unsupported" + ); let Some(scope) = self.rollback_scopes.pop() else { return false; }; - self.predictor.restore_checkpoint(&scope.checkpoint); + self.state.restore_checkpoint(&scope.checkpoint); self.journal.truncate(scope.journal_len); if self.rollback_scopes.is_empty() && self.journal.is_empty() { - self.predictor.clear_checkpoints_if_supported(); + self.state.clear_checkpoints_if_supported(); } true } fn predict_prob(&mut self, sym: bool) -> f64 { - let (p0, p1) = normalized_binary_prob_pair_from_log_probs( - self.predictor.log_prob(0), - self.predictor.log_prob(1), - self.min_prob, - ); - if sym { p1 } else { p0 } + self.state.predict_prob(sym) } fn model_name(&self) -> String { format!( - "RateBackendBits({})", - RateBackendPredictor::default_name(&self.backend, self.max_order) + "RateBackendBits({}, {:?})", + self.backend.default_name(), + self.semantics ) } fn boxed_clone(&self) -> Box { Box::new(self.clone_state()) } + + fn reset_conditioning_history(&mut self) -> Result<(), String> { + self.journal.clear(); + self.rollback_scopes.clear(); + self.discardable_scopes = 0; + self.state.clear_discardable_scopes(); + self.state.reset_frozen() + } +} + +/// Build the predictor used by the MC-AIXI runtime from a compiled backend. +pub(crate) fn build_mc_aixi_predictor( + backend: &CompiledRateBackend, + #[allow(unused_variables)] percept_bits: usize, + semantics: BitStreamSemantics, +) -> Result, PredictorBuildError> { + if semantics == BitStreamSemantics::BinaryTokens + && backend.supports_native_bit_prediction() + && backend.supports_reversible_bit_updates() + { + match backend.canonical_spec() { + #[cfg(feature = "backend-ctw")] + RateBackend::FacCtw { base_depth, .. } => { + return Ok(Box::new(FacCtwPredictor::new(*base_depth, percept_bits))); + } + #[cfg(feature = "backend-ctw")] + RateBackend::Ctw { depth } => { + return Ok(Box::new(CtwPredictor::new(*depth))); + } + #[cfg(feature = "backend-rosa")] + RateBackend::RosaPlus { max_order } => { + return Ok(Box::new(RosaPredictor::new(*max_order))); + } + _ => {} + } + } + Ok(Box::new(build_compiled_bit_predictor(backend, semantics)?)) +} + +/// Build the predictor used by the AIQI runtime from a compiled backend. +pub(crate) fn build_aiqi_predictor( + backend: &CompiledRateBackend, + #[allow(unused_variables)] return_bits: usize, + semantics: BitStreamSemantics, +) -> Result, PredictorBuildError> { + if semantics == BitStreamSemantics::BinaryTokens + && backend.supports_native_bit_prediction() + && backend.supports_reversible_bit_updates() + { + match backend.canonical_spec() { + #[cfg(feature = "backend-ctw")] + RateBackend::Ctw { depth } => return Ok(Box::new(CtwPredictor::new(*depth))), + #[cfg(feature = "backend-ctw")] + RateBackend::FacCtw { base_depth, .. } => { + return Ok(Box::new(FacCtwPredictor::new(*base_depth, return_bits))); + } + _ => {} + } + } + Ok(Box::new(build_compiled_bit_predictor(backend, semantics)?)) +} + +fn build_compiled_bit_predictor( + backend: &CompiledRateBackend, + semantics: BitStreamSemantics, +) -> Result { + RateBackendBitPredictor::new(RateBackendBitPredictorConfig { + backend: backend.clone(), + min_prob: DEFAULT_MIN_PROB, + semantics, + }) + .map_err(PredictorBuildError::BitPredictor) } #[cfg(feature = "backend-rwkv")] @@ -626,9 +1031,16 @@ impl RwkvPredictor { } /// Creates a new `RwkvPredictor` from a method string. - pub fn from_method(method: &str) -> Result { - let mut compressor = - RwkvCompressor::new_from_method(method).map_err(|err| err.to_string())?; + pub fn from_method(method: &str) -> InfotheoryResult { + let spec = crate::rwkvzip::parse_method_spec(method) + .map_err(|err| InfotheoryError::invalid_backend_config(err.to_string()))?; + Self::from_method_spec(&spec) + } + + /// Creates a new `RwkvPredictor` from a parsed method spec. + pub fn from_method_spec(method: &crate::rwkvzip::MethodSpec) -> InfotheoryResult { + let mut compressor = RwkvCompressor::new_from_method_spec(method) + .map_err(|err| InfotheoryError::invalid_backend_config(err.to_string()))?; compressor.forward_to_internal_pdf(0); Ok(Self { compressor, @@ -665,12 +1077,12 @@ impl Predictor for RwkvPredictor { } fn predict_prob(&mut self, sym: bool) -> f64 { - let (p0, p1) = normalized_binary_prob_pair_from_probs( + binary_prediction_from_probs( self.compressor.pdf_buffer[0], self.compressor.pdf_buffer[1], DEFAULT_MIN_PROB, - ); - if sym { p1 } else { p0 } + ) + .prob(sym) } fn model_name(&self) -> String { @@ -711,9 +1123,16 @@ impl MambaPredictor { } /// Creates a new `MambaPredictor` from a method string. - pub fn from_method(method: &str) -> Result { - let mut compressor = - MambaCompressor::new_from_method(method).map_err(|err| err.to_string())?; + pub fn from_method(method: &str) -> InfotheoryResult { + let spec = crate::mambazip::parse_method_spec(method) + .map_err(|err| InfotheoryError::invalid_backend_config(err.to_string()))?; + Self::from_method_spec(&spec) + } + + /// Creates a new `MambaPredictor` from a parsed method spec. + pub fn from_method_spec(method: &crate::mambazip::MethodSpec) -> InfotheoryResult { + let mut compressor = MambaCompressor::new_from_method_spec(method) + .map_err(|err| InfotheoryError::invalid_backend_config(err.to_string()))?; let mut pdf = vec![0.0f64; compressor.vocab_size()]; compressor.forward_to_pdf(0, &mut pdf); compressor.pdf_buffer.clone_from(&pdf); @@ -754,12 +1173,12 @@ impl Predictor for MambaPredictor { } fn predict_prob(&mut self, sym: bool) -> f64 { - let (p0, p1) = normalized_binary_prob_pair_from_probs( + binary_prediction_from_probs( self.compressor.pdf_buffer[0], self.compressor.pdf_buffer[1], DEFAULT_MIN_PROB, - ); - if sym { p1 } else { p0 } + ) + .prob(sym) } fn model_name(&self) -> String { @@ -774,7 +1193,7 @@ impl Predictor for MambaPredictor { } } -#[cfg(test)] +#[cfg(all(test, feature = "all-backends"))] mod tests { use super::*; @@ -815,10 +1234,19 @@ mod tests { signature } + fn bit_predictor(backend: RateBackend) -> RateBackendBitPredictor { + let config = RateBackendBitPredictorConfig::compile_with_semantics( + backend, + DEFAULT_MIN_PROB, + BitStreamSemantics::BinaryTokens, + ) + .expect("rate backend bit predictor config should compile"); + RateBackendBitPredictor::new(config).expect("rate backend predictor should initialize") + } + #[test] fn committed_rate_backend_updates_do_not_grow_journal() { - let mut predictor = RateBackendBitPredictor::new(RateBackend::RosaPlus, 8) - .expect("rate backend predictor should initialize"); + let mut predictor = bit_predictor(RateBackend::RosaPlus { max_order: 8 }); for idx in 0..512usize { predictor.commit_update((idx & 1) == 0); @@ -831,10 +1259,37 @@ mod tests { ); } + #[cfg(feature = "backend-rosa")] + #[test] + fn rosa_predictor_conditioning_reset_clears_cursor_and_rollback_history() { + let mut predictor = RosaPredictor::new(8); + for &bit in &[true, false, true, true, false] { + predictor.commit_update(bit); + } + assert!( + !predictor.history.is_empty(), + "precondition: rollback journal should be populated after committed updates" + ); + predictor.model.advance_conditioning_byte(1); + predictor.model.advance_conditioning_byte(0); + + predictor + .reset_conditioning_history() + .expect("rosa conditioning reset should succeed"); + assert!( + predictor.history.is_empty(), + "conditioning reset must clear rollback journal state" + ); + assert_eq!( + predictor.model.conditioning_cursor(), + 0, + "conditioning reset must return predictive cursor to root state" + ); + } + #[test] fn reversible_rate_backend_update_paths_round_trip_exactly() { - let mut predictor = RateBackendBitPredictor::new(RateBackend::RosaPlus, 8) - .expect("rate backend predictor should initialize"); + let mut predictor = bit_predictor(RateBackend::RosaPlus { max_order: 8 }); for &bit in &[true, false, true, true, false, false, true] { predictor.commit_update(bit); } @@ -875,8 +1330,7 @@ mod tests { #[test] fn long_committed_history_does_not_contaminate_clone_rollback_state() { - let mut predictor = RateBackendBitPredictor::new(RateBackend::RosaPlus, 8) - .expect("rate backend predictor should initialize"); + let mut predictor = bit_predictor(RateBackend::RosaPlus { max_order: 8 }); for idx in 0..2048usize { predictor.commit_update((idx & 7) < 3); @@ -907,8 +1361,7 @@ mod tests { #[test] fn rollback_scope_restores_simulation_state_without_growing_journal() { - let mut predictor = RateBackendBitPredictor::new(RateBackend::RosaPlus, 8) - .expect("rate backend predictor should initialize"); + let mut predictor = bit_predictor(RateBackend::RosaPlus { max_order: 8 }); for &bit in &[true, false, true, false, true] { predictor.commit_update(bit); } @@ -933,10 +1386,30 @@ mod tests { } } + #[test] + fn discardable_scope_suppresses_rate_backend_rollback_bookkeeping() { + let mut predictor = bit_predictor(RateBackend::RosaPlus { max_order: 8 }); + predictor.begin_discardable_scope(); + + for idx in 0..512usize { + predictor.update((idx & 1) == 0); + predictor.update_history((idx % 3) == 0); + } + + assert!( + predictor.journal.is_empty(), + "discardable cloned rollouts must not retain per-symbol checkpoints" + ); + assert!( + predictor.rollback_scopes.is_empty(), + "discardable cloned rollouts must not retain reversible scope checkpoints" + ); + assert_eq!(predictor.discardable_scopes, 1); + } + #[test] fn cloned_predictor_carries_only_active_scope_snapshots() { - let mut predictor = RateBackendBitPredictor::new(RateBackend::RosaPlus, 8) - .expect("rate backend predictor should initialize"); + let mut predictor = bit_predictor(RateBackend::RosaPlus { max_order: 8 }); for idx in 0..1024usize { predictor.commit_update((idx & 3) == 0); } @@ -956,39 +1429,24 @@ mod tests { #[test] fn generic_rate_backend_bit_predictors_normalize_binary_mass() { assert_binary_predictor_normalizes( - Box::new( - RateBackendBitPredictor::new(RateBackend::RosaPlus, 8) - .expect("generic rosa predictor"), - ), + Box::new(bit_predictor(RateBackend::RosaPlus { max_order: 8 })), "generic-rosa", ); assert_binary_predictor_normalizes( - Box::new( - RateBackendBitPredictor::new( - RateBackend::Ppmd { - order: 4, - memory_mb: 8, - }, - 8, - ) - .expect("generic ppmd predictor"), - ), + Box::new(bit_predictor(RateBackend::Ppmd { + order: 4, + memory_mb: 8, + })), "generic-ppmd", ); assert_binary_predictor_normalizes( - Box::new( - RateBackendBitPredictor::new( - RateBackend::Match { - hash_bits: 16, - min_len: 2, - max_len: 32, - base_mix: 0.05, - confidence_scale: 1.0, - }, - 8, - ) - .expect("generic match predictor"), - ), + Box::new(bit_predictor(RateBackend::Match { + hash_bits: 16, + min_len: 2, + max_len: 32, + base_mix: 0.05, + confidence_scale: 1.0, + })), "generic-match", ); } @@ -1018,3 +1476,76 @@ mod tests { assert_binary_predictor_normalizes(Box::new(predictor), "mamba"); } } + +#[cfg(all(test, feature = "aixi", feature = "backend-ctw"))] +mod build_mc_aixi_predictor_tests { + use super::{CtwPredictor, FacCtwPredictor, Predictor, build_mc_aixi_predictor}; + use crate::api::{BitStreamSemantics, RateBackend}; + + const EPSILON: f64 = 1.0e-12; + + #[test] + fn ctw_predictor_predict_one_matches_true_probability() { + let mut fast = CtwPredictor::new(6); + let mut general = CtwPredictor::new(6); + + for bit in [true, false, true, true, false, false, true, false] { + fast.update(bit); + general.update(bit); + } + + let delta = (fast.predict_one() - general.predict_prob(true)).abs(); + assert!( + delta <= EPSILON, + "ctw predict_one must match predict_prob(true), delta={delta}" + ); + } + + #[test] + fn fac_ctw_predictor_predict_one_matches_current_lane_true_probability() { + let mut fast = FacCtwPredictor::new(4, 3); + let mut general = FacCtwPredictor::new(4, 3); + + for bit in [ + true, false, true, false, false, true, true, true, false, true, + ] { + fast.update(bit); + general.update(bit); + } + + let delta = (fast.predict_one() - general.predict_prob(true)).abs(); + assert!( + delta <= EPSILON, + "fac-ctw predict_one must match predict_prob(true), delta={delta}" + ); + } + + /// `encoding_bits` / `msb_first` on the rate spec describe byte-level behavior; + /// BinaryTokens planner native path uses [`FacCtwPredictor`] + `percept_bits` lanes. + #[test] + fn build_mc_aixi_predictor_selects_fac_ctw_native_predictor() { + let backend = RateBackend::FacCtw { + base_depth: 8, + num_percept_bits: 8, + encoding_bits: 8, + msb_first: Some(true), + } + .compile() + .expect("compile fac-ctw backend"); + + let caps = backend.capabilities(); + assert!(caps.supports_native_bit_prediction); + assert!(caps.supports_reversible_bit_updates); + + let percept_bits: usize = 8; + let predictor = + build_mc_aixi_predictor(&backend, percept_bits, BitStreamSemantics::BinaryTokens) + .expect("build mc-aixi predictor"); + + let name = Predictor::model_name(predictor.as_ref()); + assert!( + name.starts_with("FAC-CTW"), + "BinaryTokens + native FacCtw must use FacCtwPredictor fast path, got {name}" + ); + } +} diff --git a/crates/infotheory/src/aixi/planner_agent.rs b/crates/infotheory/src/aixi/planner_agent.rs new file mode 100644 index 00000000..15d3cfdb --- /dev/null +++ b/crates/infotheory/src/aixi/planner_agent.rs @@ -0,0 +1,1164 @@ +//! Public planner-agent substrate for AIXI-family controllers. +//! +//! The runtime surface is intentionally limited to executable base controllers: +//! MC-AIXI, discounted AIQI, and exact-\(J_H\) warm-start AIQI. Failed +//! meta-controller experiments are not part of this module. + +use crate::aixi::agent::Agent; +use crate::aixi::aiqi::AiqiAgent; +use crate::aixi::common::{ + Action, EXPLORE_RANDOM_SALT, RandomGenerator, Reward, resolve_random_seed, +}; +use crate::aixi::environment::Environment; +use crate::aixi::planner_runtime::validate_environment_interface; +use crate::aixi::warmstart::{ + WarmStartExactJhAgent, WarmStartExactJhError, WarmStartExactJhTeacherDataset, +}; +use crate::spec::{CompiledPlannerController, CompiledPlannerRunSpec, PlannerRuntimeSpec}; +use std::error::Error; +use std::fmt; + +pub use crate::aixi::planner_runtime::{ + build_planner_environment, compile_planner_run_document, + load_warmstart_exact_jh_teacher_dataset, validate_action_alphabet, + validate_warmstart_exact_jh_teacher_contract, +}; + +/// Planner cycle phase. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub enum PlannerPhase { + /// Learning/exploration phase. + Learn, + /// Evaluation/greedy phase. + Eval, +} + +/// Planner learn/eval schedule derived from a runtime specification. +#[derive(Clone, Copy, Debug, PartialEq)] +#[non_exhaustive] +pub struct PlannerSchedule { + /// Number of learning cycles. + pub learn_cycles: usize, + /// Number of evaluation cycles. + pub eval_cycles: usize, + /// Extra exploration probability at learning step zero. + pub explore_epsilon: f64, + /// Per-step exploration decay factor. + pub explore_gamma: f64, +} + +impl PlannerSchedule { + /// Construct a schedule with zero extra exploration. + pub fn new(learn_cycles: usize, eval_cycles: usize) -> Self { + Self { + learn_cycles, + eval_cycles, + explore_epsilon: 0.0, + explore_gamma: 1.0, + } + } + + /// Derive the semantic execution schedule from a compiled runtime spec. + pub fn from_runtime(runtime: &PlannerRuntimeSpec) -> Self { + let terminate_lifetime: usize = runtime.terminate_lifetime; + let (learn_cycles, eval_cycles) = match (runtime.learn_cycles, runtime.eval_cycles) { + (Some(learn), Some(eval)) => (learn, eval), + (Some(learn), None) => (learn, 0usize), + (None, Some(eval)) => (terminate_lifetime, eval), + (None, None) => (terminate_lifetime, 0usize), + }; + Self { + learn_cycles, + eval_cycles, + explore_epsilon: runtime.explore_epsilon, + explore_gamma: runtime.explore_gamma, + } + } + + /// Extra exploration probability at the given global planner step. + pub fn extra_exploration(&self, step: usize) -> f64 { + if self.explore_epsilon > 0.0 { + let exponent = i32::try_from(step).unwrap_or(i32::MAX); + (self.explore_epsilon * self.explore_gamma.powi(exponent)).min(1.0) + } else { + 0.0 + } + } +} + +/// Stable action-provenance labels used by planner JSONL telemetry. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub enum PlannerActionProvenance { + /// Greedy controller action. + Greedy, + /// Exploratory controller action. + Exploratory, +} + +impl PlannerActionProvenance { + /// Stable JSONL string representation. + pub fn as_str(self) -> &'static str { + match self { + Self::Greedy => "greedy", + Self::Exploratory => "exploratory", + } + } + + /// Parse a normative JSONL provenance string. + pub fn from_jsonl_str(value: &str) -> Result { + match value { + "greedy" => Ok(Self::Greedy), + "exploratory" => Ok(Self::Exploratory), + other => Err(WarmStartExactJhError::InvalidTelemetry { + reason: format!("unknown action provenance '{other}'"), + }), + } + } +} + +/// Outcome of one planner-environment cycle. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct PlannerCycleOutcome { + /// Observation stream visible at the decision point. + pub pre_observations: Vec, + /// Reward visible at the decision point. + pub pre_reward: Reward, + /// Selected action. + pub action: Action, + /// Post-action observation stream. + pub observations: Vec, + /// Post-action reward. + pub reward: Reward, + /// Provenance for the selected action. + pub provenance: PlannerActionProvenance, +} + +/// Observer strategy for ordered planner-cycle telemetry. +/// +/// Implementations receive a controller's native event ordering, which is one of: +/// - *decision-percept*: the percept for step `t`, then action `t`; after the +/// final scheduled action, one terminal successor percept at step +/// `N = learn_cycles + eval_cycles`. +/// - *action-then-post-percept*: action `t`, then the reached percept, both at +/// step `t`, with no separate terminal percept. +/// +/// Observers must tolerate either ordering and a percept reported at `step == N`. +/// Each concrete controller documents which ordering it produces. +pub trait PlannerCycleObserver { + /// Observe a percept event for `step`. + /// + /// Under the decision-percept ordering, `step` may equal the total number of + /// scheduled cycles `N` for the terminal successor percept; observers must + /// accept this one-past-the-last-cycle index. + fn observe_percept( + &mut self, + step: usize, + observations: &[u64], + reward: Reward, + ) -> Result<(), PlannerAgentError>; + + /// Observe an action event for `step`. + fn observe_action( + &mut self, + step: usize, + action: Action, + provenance: PlannerActionProvenance, + ) -> Result<(), PlannerAgentError>; + + /// Mark the end of one complete cycle. + fn end_cycle(&mut self, step: usize) -> Result<(), PlannerAgentError>; +} + +/// No-op observer for executions that do not need telemetry. +#[derive(Clone, Copy, Debug, Default)] +pub struct NullPlannerObserver; + +impl PlannerCycleObserver for NullPlannerObserver { + fn observe_percept( + &mut self, + _step: usize, + _observations: &[u64], + _reward: Reward, + ) -> Result<(), PlannerAgentError> { + Ok(()) + } + + fn observe_action( + &mut self, + _step: usize, + _action: Action, + _provenance: PlannerActionProvenance, + ) -> Result<(), PlannerAgentError> { + Ok(()) + } + + fn end_cycle(&mut self, _step: usize) -> Result<(), PlannerAgentError> { + Ok(()) + } +} + +impl PlannerCycleObserver for () { + fn observe_percept( + &mut self, + _step: usize, + _observations: &[u64], + _reward: Reward, + ) -> Result<(), PlannerAgentError> { + Ok(()) + } + + fn observe_action( + &mut self, + _step: usize, + _action: Action, + _provenance: PlannerActionProvenance, + ) -> Result<(), PlannerAgentError> { + Ok(()) + } + + fn end_cycle(&mut self, _step: usize) -> Result<(), PlannerAgentError> { + Ok(()) + } +} + +/// Validate observation stream length. +pub fn validate_obs_stream_len(expected: usize, actual: usize) -> Result<(), PlannerAgentError> { + if actual != expected { + return Err(PlannerAgentError::EnvironmentInterface { + reason: format!( + "observation stream length mismatch: expected {expected}, got {actual}" + ), + }); + } + Ok(()) +} + +/// Runtime environment state for planner episodes. +pub struct PlannerEnvironment { + env: Box, + observation_stream_len: usize, + observations: Vec, + reward: Reward, +} + +impl PlannerEnvironment { + /// Construct a validated planner environment state. + pub fn new( + compiled: &CompiledPlannerRunSpec, + env: Box, + ) -> Result { + Self::new_with_seed( + compiled, + env, + resolve_random_seed(compiled.runtime().random_seed), + ) + } + + /// Construct a validated planner environment state with an explicit seed. + pub fn new_with_seed( + compiled: &CompiledPlannerRunSpec, + mut env: Box, + random_seed: u64, + ) -> Result { + validate_environment_interface(compiled, env.as_ref()).map_err(|err| { + PlannerAgentError::EnvironmentInterface { + reason: err.to_string(), + } + })?; + env.set_random_seed(random_seed); + let observation_stream_len: usize = compiled.interface().observation_stream_len; + let observations = env.drain_observations(); + validate_obs_stream_len(observation_stream_len, observations.len())?; + let reward: Reward = env.get_reward(); + Ok(Self { + env, + observation_stream_len, + observations, + reward, + }) + } + + /// Current observation stream. + pub fn observations(&self) -> &[u64] { + &self.observations + } + + /// Current reward. + pub fn reward(&self) -> Reward { + self.reward + } + + /// Perform one action and update the current percept state. + pub fn perform_action(&mut self, action: Action) -> Result { + self.env.perform_action(action); + self.observations = self.env.drain_observations(); + validate_obs_stream_len(self.observation_stream_len, self.observations.len())?; + self.reward = self.env.get_reward(); + Ok(self.reward) + } +} + +/// Factory for constructing fresh environments for repeated episodes. +pub trait EnvironmentFactory { + /// Build a fresh environment. + fn build(&self) -> Result, PlannerAgentError>; +} + +/// Executable planner-agent abstraction. +pub trait PlannerAgent { + /// Reseed controller-side stochastic state. + /// + /// This does not clear learned model state or retained history. + fn reseed_for_episode(&mut self, _random_seed: u64) {} + + /// Start a fresh environment episode while preserving learned model state. + /// + /// Implementations should reset episode-local transient state such as a + /// previous-action pointer or retained search tree. They should not discard + /// learned predictor state unless the concrete controller documents that + /// policy separately. + fn reset_for_episode(&mut self, random_seed: u64) { + self.reseed_for_episode(random_seed); + } + + /// Execute one planner-environment cycle. + fn run_cycle( + &mut self, + phase: PlannerPhase, + step: usize, + schedule: &PlannerSchedule, + env: &mut PlannerEnvironment, + observer: &mut dyn PlannerCycleObserver, + ) -> Result; +} + +/// MC-AIXI planner adapter. +/// +/// Produces the decision-percept ordering (see [`PlannerCycleObserver`]). +pub struct McAixiPlannerAgent { + agent: Agent, + prev_action: Action, + explore_rng: RandomGenerator, +} + +impl McAixiPlannerAgent { + /// Construct an MC-AIXI planner from a compiled planner-run spec. + pub fn from_compiled(compiled: &CompiledPlannerRunSpec) -> Result { + let agent = Agent::from_compiled_planner_run(compiled) + .map_err(|err| PlannerAgentError::McAixi(err.to_string()))?; + let explore_rng: RandomGenerator = + RandomGenerator::from_seed(resolve_random_seed(compiled.runtime().random_seed)) + .fork_with(EXPLORE_RANDOM_SALT); + Ok(Self { + agent, + prev_action: 0, + explore_rng, + }) + } +} + +impl PlannerAgent for McAixiPlannerAgent { + fn reseed_for_episode(&mut self, random_seed: u64) { + self.agent.reseed_random(random_seed); + self.explore_rng = RandomGenerator::from_seed(random_seed).fork_with(EXPLORE_RANDOM_SALT); + } + + fn reset_for_episode(&mut self, random_seed: u64) { + self.reseed_for_episode(random_seed); + self.prev_action = 0; + self.agent.reset_planner_state(); + } + + fn run_cycle( + &mut self, + phase: PlannerPhase, + step: usize, + schedule: &PlannerSchedule, + env: &mut PlannerEnvironment, + observer: &mut dyn PlannerCycleObserver, + ) -> Result { + let pre_observations: Vec = env.observations().to_vec(); + let pre_reward: Reward = env.reward(); + observer.observe_percept(step, &pre_observations, pre_reward)?; + self.agent + .model_update_percept_stream(&pre_observations, pre_reward); + let mut provenance = PlannerActionProvenance::Greedy; + let action: Action = match phase { + PlannerPhase::Learn => { + let explore_p: f64 = schedule.extra_exploration(step); + if explore_p > 0.0 && self.explore_rng.gen_bool(explore_p) { + provenance = PlannerActionProvenance::Exploratory; + self.explore_rng.gen_range(env.env.get_num_actions().get()) as u64 + } else { + self.agent + .get_planned_action(&pre_observations, pre_reward, self.prev_action) + } + } + PlannerPhase::Eval => { + self.agent + .get_planned_action(&pre_observations, pre_reward, self.prev_action) + } + }; + observer.observe_action(step, action, provenance)?; + self.agent.model_update_action_external(action); + let reward: Reward = env.perform_action(action)?; + self.prev_action = action; + observer.end_cycle(step)?; + // The last post-action percept has no following decision cycle to emit it. + let total_cycles: usize = schedule.learn_cycles.saturating_add(schedule.eval_cycles); + if step.checked_add(1) == Some(total_cycles) { + observer.observe_percept(total_cycles, env.observations(), reward)?; + } + Ok(PlannerCycleOutcome { + pre_observations, + pre_reward, + action, + observations: env.observations().to_vec(), + reward, + provenance, + }) + } +} + +/// Discounted AIQI planner adapter. +/// +/// Produces the action-then-post-percept ordering (see [`PlannerCycleObserver`]). +pub struct AiqiDiscountedPlannerAgent { + agent: AiqiAgent, +} + +impl AiqiDiscountedPlannerAgent { + /// Construct a discounted-AIQI planner from a compiled planner-run spec. + pub fn from_compiled(compiled: &CompiledPlannerRunSpec) -> Result { + Ok(Self { + agent: AiqiAgent::from_compiled_planner_run(compiled) + .map_err(PlannerAgentError::Aiqi)?, + }) + } +} + +impl PlannerAgent for AiqiDiscountedPlannerAgent { + fn reseed_for_episode(&mut self, random_seed: u64) { + self.agent.reseed_random(random_seed); + } + + fn run_cycle( + &mut self, + phase: PlannerPhase, + step: usize, + schedule: &PlannerSchedule, + env: &mut PlannerEnvironment, + observer: &mut dyn PlannerCycleObserver, + ) -> Result { + let pre_observations: Vec = env.observations().to_vec(); + let pre_reward: Reward = env.reward(); + let (action, explored) = match phase { + PlannerPhase::Learn => self + .agent + .get_planned_action_with_extra_exploration_flag(schedule.extra_exploration(step)), + PlannerPhase::Eval => (self.agent.get_planned_action(), false), + }; + let provenance = if explored { + PlannerActionProvenance::Exploratory + } else { + PlannerActionProvenance::Greedy + }; + observer.observe_action(step, action, provenance)?; + let reward: Reward = env.perform_action(action)?; + observer.observe_percept(step, env.observations(), reward)?; + self.agent + .observe_transition(action, env.observations(), reward) + .map_err(PlannerAgentError::Aiqi)?; + observer.end_cycle(step)?; + Ok(PlannerCycleOutcome { + pre_observations, + pre_reward, + action, + observations: env.observations().to_vec(), + reward, + provenance, + }) + } +} + +/// Exact-\(J_H\) warm-start planner adapter. +/// +/// Produces the action-then-post-percept ordering (see [`PlannerCycleObserver`]). +pub struct WarmStartExactJhPlannerAgent { + agent: WarmStartExactJhAgent, +} + +impl WarmStartExactJhPlannerAgent { + /// Construct a warm-start planner from a compiled planner-run spec and teacher data. + pub fn from_compiled( + compiled: &CompiledPlannerRunSpec, + teacher: WarmStartExactJhTeacherDataset, + ) -> Result { + Ok(Self { + agent: WarmStartExactJhAgent::from_compiled_planner_run(compiled, teacher) + .map_err(PlannerAgentError::WarmStart)?, + }) + } +} + +impl PlannerAgent for WarmStartExactJhPlannerAgent { + fn reseed_for_episode(&mut self, random_seed: u64) { + self.agent.reseed_random(random_seed); + } + + fn run_cycle( + &mut self, + phase: PlannerPhase, + step: usize, + schedule: &PlannerSchedule, + env: &mut PlannerEnvironment, + observer: &mut dyn PlannerCycleObserver, + ) -> Result { + let pre_observations: Vec = env.observations().to_vec(); + let pre_reward: Reward = env.reward(); + let (action, explored) = match phase { + PlannerPhase::Learn => self + .agent + .try_get_planned_action_with_extra_exploration_flag( + schedule.extra_exploration(step), + ) + .map_err(PlannerAgentError::WarmStart)?, + PlannerPhase::Eval => ( + self.agent + .try_get_planned_action() + .map_err(PlannerAgentError::WarmStart)?, + false, + ), + }; + let provenance = if explored { + PlannerActionProvenance::Exploratory + } else { + PlannerActionProvenance::Greedy + }; + observer.observe_action(step, action, provenance)?; + let reward: Reward = env.perform_action(action)?; + observer.observe_percept(step, env.observations(), reward)?; + self.agent + .observe_transition(action, env.observations(), reward) + .map_err(PlannerAgentError::WarmStart)?; + observer.end_cycle(step)?; + Ok(PlannerCycleOutcome { + pre_observations, + pre_reward, + action, + observations: env.observations().to_vec(), + reward, + provenance, + }) + } +} + +enum PlannerControllerAgentKind { + McAixi(McAixiPlannerAgent), + AiqiDiscounted(AiqiDiscountedPlannerAgent), + WarmStartExactJh(WarmStartExactJhPlannerAgent), +} + +/// Runtime controller selected from a compiled `planner_run`. +pub struct PlannerControllerAgent { + inner: PlannerControllerAgentKind, +} + +impl PlannerControllerAgent { + /// Construct the executable base controller declared by `compiled`. + pub fn from_compiled(compiled: &CompiledPlannerRunSpec) -> Result { + let inner = match compiled.controller() { + CompiledPlannerController::McAixi { .. } => { + PlannerControllerAgentKind::McAixi(McAixiPlannerAgent::from_compiled(compiled)?) + } + CompiledPlannerController::AiqiDiscounted { .. } => { + PlannerControllerAgentKind::AiqiDiscounted( + AiqiDiscountedPlannerAgent::from_compiled(compiled)?, + ) + } + CompiledPlannerController::AiqiWarmstartExactJh { + teacher_dataset_asset, + .. + } => { + let teacher = + load_warmstart_exact_jh_teacher_dataset(compiled, teacher_dataset_asset)?; + PlannerControllerAgentKind::WarmStartExactJh( + WarmStartExactJhPlannerAgent::from_compiled(compiled, teacher)?, + ) + } + }; + Ok(Self { inner }) + } + + /// Canonical controller kind label. + pub fn controller_kind(&self) -> &'static str { + match &self.inner { + PlannerControllerAgentKind::McAixi(_) => "mc_aixi", + PlannerControllerAgentKind::AiqiDiscounted(_) => "aiqi_discounted", + PlannerControllerAgentKind::WarmStartExactJh(_) => "aiqi_warmstart_exact_jh", + } + } +} + +impl PlannerAgent for PlannerControllerAgent { + fn reseed_for_episode(&mut self, random_seed: u64) { + match &mut self.inner { + PlannerControllerAgentKind::McAixi(agent) => agent.reseed_for_episode(random_seed), + PlannerControllerAgentKind::AiqiDiscounted(agent) => { + agent.reseed_for_episode(random_seed); + } + PlannerControllerAgentKind::WarmStartExactJh(agent) => { + agent.reseed_for_episode(random_seed); + } + } + } + + fn reset_for_episode(&mut self, random_seed: u64) { + match &mut self.inner { + PlannerControllerAgentKind::McAixi(agent) => agent.reset_for_episode(random_seed), + PlannerControllerAgentKind::AiqiDiscounted(agent) => { + agent.reset_for_episode(random_seed); + } + PlannerControllerAgentKind::WarmStartExactJh(agent) => { + agent.reset_for_episode(random_seed); + } + } + } + + fn run_cycle( + &mut self, + phase: PlannerPhase, + step: usize, + schedule: &PlannerSchedule, + env: &mut PlannerEnvironment, + observer: &mut dyn PlannerCycleObserver, + ) -> Result { + match &mut self.inner { + PlannerControllerAgentKind::McAixi(agent) => { + agent.run_cycle(phase, step, schedule, env, observer) + } + PlannerControllerAgentKind::AiqiDiscounted(agent) => { + agent.run_cycle(phase, step, schedule, env, observer) + } + PlannerControllerAgentKind::WarmStartExactJh(agent) => { + agent.run_cycle(phase, step, schedule, env, observer) + } + } + } +} + +/// Executable planner-run session. +pub struct PlannerRunSession { + agent: PlannerControllerAgent, + environment: PlannerEnvironment, + schedule: PlannerSchedule, + next_step: usize, +} + +impl PlannerRunSession { + /// Construct a planner-run session from executable components. + pub fn new( + compiled: &CompiledPlannerRunSpec, + mut agent: PlannerControllerAgent, + env: Box, + ) -> Result { + let environment = PlannerEnvironment::new(compiled, env)?; + let schedule = PlannerSchedule::from_runtime(compiled.runtime()); + agent.reset_for_episode(resolve_random_seed(compiled.runtime().random_seed)); + Ok(Self { + agent, + environment, + schedule, + next_step: 0, + }) + } + + /// Execution schedule for this session. + pub fn schedule(&self) -> &PlannerSchedule { + &self.schedule + } + + /// Number of cycles already executed. + pub fn next_step(&self) -> usize { + self.next_step + } + + /// Phase of the next scheduled cycle, or `None` when the session is complete. + pub fn next_phase(&self) -> Option { + let total_cycles: usize = self + .schedule + .learn_cycles + .saturating_add(self.schedule.eval_cycles); + if self.next_step >= total_cycles { + return None; + } + Some(if self.next_step < self.schedule.learn_cycles { + PlannerPhase::Learn + } else { + PlannerPhase::Eval + }) + } + + /// Whether all scheduled cycles have been executed. + pub fn is_finished(&self) -> bool { + self.next_phase().is_none() + } + + /// Run the next scheduled cycle. + pub fn run_next_cycle( + &mut self, + observer: &mut dyn PlannerCycleObserver, + ) -> Result, PlannerAgentError> { + let Some(phase) = self.next_phase() else { + return Ok(None); + }; + let outcome = self.agent.run_cycle( + phase, + self.next_step, + &self.schedule, + &mut self.environment, + observer, + )?; + self.next_step = self.next_step.saturating_add(1); + Ok(Some(outcome)) + } +} + +/// Summary returned by [`run_episode`]. +#[derive(Clone, Debug, PartialEq)] +pub struct PlannerRunReport { + /// Sum of rewards over learning cycles. + pub learn_total_reward: Reward, + /// Sum of rewards over evaluation cycles. + pub eval_total_reward: Reward, + /// Number of learning cycles executed. + pub learn_cycles: usize, + /// Number of evaluation cycles executed. + pub eval_cycles: usize, +} + +/// Run one complete environment episode with a supplied agent and environment factory. +/// +/// The agent's learned model state is preserved across calls. Before the fresh +/// environment is used, the agent receives an episode-boundary reset so +/// controller-side randomness and episode-local transient state are aligned with +/// `random_seed`. +pub fn run_episode( + agent: &mut dyn PlannerAgent, + schedule: &PlannerSchedule, + env_factory: &dyn EnvironmentFactory, + random_seed: u64, + compiled: &CompiledPlannerRunSpec, +) -> Result { + let env = env_factory.build()?; + let mut environment = PlannerEnvironment::new_with_seed(compiled, env, random_seed)?; + agent.reset_for_episode(random_seed); + let mut observer = NullPlannerObserver; + let mut learn_total_reward: Reward = 0; + let mut eval_total_reward: Reward = 0; + for step in 0..schedule.learn_cycles { + let outcome = agent.run_cycle( + PlannerPhase::Learn, + step, + schedule, + &mut environment, + &mut observer, + )?; + learn_total_reward = learn_total_reward.saturating_add(outcome.reward); + } + for offset in 0..schedule.eval_cycles { + let step: usize = schedule.learn_cycles + offset; + let outcome = agent.run_cycle( + PlannerPhase::Eval, + step, + schedule, + &mut environment, + &mut observer, + )?; + eval_total_reward = eval_total_reward.saturating_add(outcome.reward); + } + Ok(PlannerRunReport { + learn_total_reward, + eval_total_reward, + learn_cycles: schedule.learn_cycles, + eval_cycles: schedule.eval_cycles, + }) +} + +/// Planner-agent runtime error. +#[derive(Debug)] +#[non_exhaustive] +pub enum PlannerAgentError { + /// MC-AIXI construction or execution error. + McAixi(String), + /// AIQI construction or execution error. + Aiqi(crate::aixi::aiqi::AiqiError), + /// Warm-start construction or execution error. + WarmStart(WarmStartExactJhError), + /// Environment interface mismatch. + EnvironmentInterface { + /// Human-readable reason. + reason: String, + }, + /// Environment construction failed. + Environment { + /// Human-readable reason. + reason: String, + }, + /// Observer or telemetry sink failed during cycle execution. + Observer { + /// Human-readable reason. + reason: String, + }, +} + +impl fmt::Display for PlannerAgentError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::McAixi(err) => write!(f, "{err}"), + Self::Aiqi(err) => write!(f, "{err}"), + Self::WarmStart(err) => write!(f, "{err}"), + Self::EnvironmentInterface { reason } + | Self::Environment { reason } + | Self::Observer { reason } => f.write_str(reason), + } + } +} + +impl Error for PlannerAgentError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Aiqi(err) => Some(err), + Self::WarmStart(err) => Some(err), + Self::McAixi(_) + | Self::EnvironmentInterface { .. } + | Self::Environment { .. } + | Self::Observer { .. } => None, + } + } +} + +#[cfg(all(test, feature = "backend-ctw"))] +mod tests { + use super::*; + use crate::aixi::common::{ActionAlphabet, PerceptVal}; + use crate::aixi::warmstart::{ + WarmStartExactJhTeacherDataset, WarmStartExactJhTeacherTrace, WarmStartExactJhTransition, + standalone_warmstart_teacher_contract_for_compiled_planner_run, + }; + use crate::spec::{SpecDocument, SpecEnvironment}; + use serde_json::json; + use std::path::{Path, PathBuf}; + use std::sync::Arc; + use std::sync::atomic::{AtomicU64, Ordering}; + use std::time::{SystemTime, UNIX_EPOCH}; + + static TEMP_TEST_PATH_COUNTER: AtomicU64 = AtomicU64::new(0); + + fn unique_temp_path(prefix: &str, suffix: &str) -> PathBuf { + let counter = TEMP_TEST_PATH_COUNTER.fetch_add(1, Ordering::Relaxed); + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_nanos()) + .unwrap_or(0); + std::env::temp_dir().join(format!( + "{prefix}-{}-{nanos}-{counter}{suffix}", + std::process::id() + )) + } + + fn action_alphabet(n: usize) -> ActionAlphabet { + ActionAlphabet::try_from_usize(n).expect("test action alphabet must be non-zero") + } + + #[test] + fn extra_exploration_decay_does_not_wrap_after_i32_limit() { + let schedule = PlannerSchedule { + learn_cycles: 0, + eval_cycles: 0, + explore_epsilon: 0.5, + explore_gamma: 0.5, + }; + + assert_eq!(schedule.extra_exploration(i32::MAX as usize + 1), 0.0); + } + + fn sample_warmstart_compiled_planner_run(teacher_path: &Path) -> CompiledPlannerRunSpec { + let document = SpecDocument::parse_json_value( + &json!({ + "schema_version": 1, + "kind": "planner_run", + "assets": [{ + "id": "teacher", + "path": teacher_path.to_string_lossy() + }], + "environment": { + "kind": "builtin", + "name": "coin_flip" + }, + "interface": { + "observation_bits": 2, + "observation_stream_len": 1, + "observation_key_mode": "full_stream", + "reward_bits": 2, + "agent_actions": action_alphabet(2).get() + }, + "controller": { + "kind": "aiqi_warmstart_exact_jh", + "predictor": { + "kind": "ctw", + "depth": 4 + }, + "return_horizon": 1, + "return_bins": 4, + "label_phase_period": 1, + "teacher_dataset_asset": "teacher", + "planner_simulations_per_step": 1 + }, + "runtime": { + "random_seed": 7, + "learn_cycles": 1, + "eval_cycles": 1, + "terminate_lifetime": 2, + "log_every": 1, + "perf": false, + "vm_perf_only": false, + "explore_epsilon": 0.0, + "explore_gamma": 1.0 + } + }), + Path::new("."), + ) + .expect("sample warmstart planner document"); + let SpecDocument::PlannerRun(spec) = document else { + panic!("expected planner_run document"); + }; + spec.compile_in(&SpecEnvironment::new(Path::new("."))) + .expect("sample warmstart planner run should compile") + } + + fn write_matching_warmstart_teacher(path: &Path, compiled: &CompiledPlannerRunSpec) { + let contract = standalone_warmstart_teacher_contract_for_compiled_planner_run(compiled) + .expect("standalone warmstart teacher contract"); + let dataset = WarmStartExactJhTeacherDataset::new( + contract, + vec![WarmStartExactJhTeacherTrace::new(vec![ + WarmStartExactJhTransition::new(0, vec![1], 1), + ])], + ); + std::fs::write( + path, + serde_json::to_vec(&dataset.to_json_value()).expect("teacher JSON"), + ) + .expect("write teacher dataset"); + } + + #[derive(Clone, Copy)] + struct CountingEnv { + observation: PerceptVal, + reward: Reward, + } + + impl Environment for CountingEnv { + fn perform_action(&mut self, action: Action) { + self.observation = (self.observation + action + 1) & 0b11; + self.reward = (self.reward + 1).min(1); + } + + fn get_observation(&self) -> PerceptVal { + self.observation + } + + fn get_reward(&self) -> Reward { + self.reward + } + + fn is_finished(&self) -> bool { + false + } + + fn get_observation_bits(&self) -> usize { + 2 + } + + fn get_reward_bits(&self) -> usize { + 2 + } + + fn get_action_bits(&self) -> usize { + 1 + } + } + + struct SeedRecordingEnv { + recorded_seed: Arc, + } + + impl Environment for SeedRecordingEnv { + fn perform_action(&mut self, _action: Action) {} + + fn get_observation(&self) -> PerceptVal { + 0 + } + + fn get_reward(&self) -> Reward { + 0 + } + + fn is_finished(&self) -> bool { + false + } + + fn get_observation_bits(&self) -> usize { + 2 + } + + fn get_reward_bits(&self) -> usize { + 2 + } + + fn get_action_bits(&self) -> usize { + 1 + } + + fn set_random_seed(&mut self, seed: u64) { + self.recorded_seed.store(seed, Ordering::SeqCst); + } + } + + struct SeedRecordingFactory { + recorded_seed: Arc, + } + + impl EnvironmentFactory for SeedRecordingFactory { + fn build(&self) -> Result, PlannerAgentError> { + Ok(Box::new(SeedRecordingEnv { + recorded_seed: Arc::clone(&self.recorded_seed), + })) + } + } + + struct SeedRecordingAgent { + recorded_seed: Arc, + reset_calls: Arc, + } + + impl PlannerAgent for SeedRecordingAgent { + fn reseed_for_episode(&mut self, random_seed: u64) { + self.recorded_seed.store(random_seed, Ordering::SeqCst); + } + + fn reset_for_episode(&mut self, random_seed: u64) { + self.recorded_seed.store(random_seed, Ordering::SeqCst); + self.reset_calls.fetch_add(1, Ordering::SeqCst); + } + + fn run_cycle( + &mut self, + phase: PlannerPhase, + step: usize, + _schedule: &PlannerSchedule, + env: &mut PlannerEnvironment, + observer: &mut dyn PlannerCycleObserver, + ) -> Result { + let pre_observations = env.observations().to_vec(); + let pre_reward = env.reward(); + observer.observe_action(step, 0, PlannerActionProvenance::Greedy)?; + let reward = env.perform_action(0)?; + observer.observe_percept(step, env.observations(), reward)?; + observer.end_cycle(step)?; + assert_eq!(phase, PlannerPhase::Learn); + Ok(PlannerCycleOutcome { + pre_observations, + pre_reward, + action: 0, + observations: env.observations().to_vec(), + reward, + provenance: PlannerActionProvenance::Greedy, + }) + } + } + + #[test] + fn run_episode_uses_explicit_seed_for_environment_and_agent() { + let compiled = sample_warmstart_compiled_planner_run(Path::new("teacher.json")); + let env_seed = Arc::new(AtomicU64::new(u64::MAX)); + let agent_seed = Arc::new(AtomicU64::new(u64::MAX)); + let reset_calls = Arc::new(AtomicU64::new(0)); + let factory = SeedRecordingFactory { + recorded_seed: Arc::clone(&env_seed), + }; + let mut agent = SeedRecordingAgent { + recorded_seed: Arc::clone(&agent_seed), + reset_calls: Arc::clone(&reset_calls), + }; + + let report = run_episode( + &mut agent, + &PlannerSchedule::new(1, 0), + &factory, + 99, + &compiled, + ) + .expect("seeded episode should run"); + + assert_eq!(report.learn_cycles, 1); + assert_eq!(env_seed.load(Ordering::SeqCst), 99); + assert_eq!(agent_seed.load(Ordering::SeqCst), 99); + assert_eq!(reset_calls.load(Ordering::SeqCst), 1); + } + + #[test] + fn planner_run_session_executes_warmstart_controller_from_compiled_spec() { + let teacher_path = unique_temp_path("planner-agent-warmstart-teacher", ".json"); + let compiled = sample_warmstart_compiled_planner_run(&teacher_path); + write_matching_warmstart_teacher(&teacher_path, &compiled); + + let controller = + PlannerControllerAgent::from_compiled(&compiled).expect("warmstart controller"); + assert_eq!(controller.controller_kind(), "aiqi_warmstart_exact_jh"); + let env = Box::new(CountingEnv { + observation: 1, + reward: 0, + }); + let mut session = + PlannerRunSession::new(&compiled, controller, env).expect("planner session"); + assert_eq!(session.schedule().learn_cycles, 1); + assert_eq!(session.schedule().eval_cycles, 1); + assert_eq!(session.next_phase(), Some(PlannerPhase::Learn)); + + let mut observer = NullPlannerObserver; + let learn = session + .run_next_cycle(&mut observer) + .expect("learn cycle") + .expect("learn outcome"); + assert_eq!(learn.pre_observations, vec![1]); + assert!(learn.action < 2); + assert_eq!(session.next_phase(), Some(PlannerPhase::Eval)); + + let eval = session + .run_next_cycle(&mut observer) + .expect("eval cycle") + .expect("eval outcome"); + assert!(eval.action < 2); + assert!(session.is_finished()); + assert!( + session + .run_next_cycle(&mut observer) + .expect("complete session") + .is_none() + ); + + let _ = std::fs::remove_file(teacher_path); + } +} + +impl From for PlannerAgentError { + fn from(value: anyhow::Error) -> Self { + Self::Environment { + reason: value.to_string(), + } + } +} diff --git a/crates/infotheory/src/aixi/planner_runtime.rs b/crates/infotheory/src/aixi/planner_runtime.rs new file mode 100644 index 00000000..7201b69e --- /dev/null +++ b/crates/infotheory/src/aixi/planner_runtime.rs @@ -0,0 +1,162 @@ +//! Compiled-spec bindings for planner-run documents. +//! +//! This module keeps filesystem asset loading and environment construction at +//! the edge of the planner runtime. Controller execution lives in +//! [`crate::aixi::planner_agent`]. + +use crate::aixi::common::ActionAlphabet; +use crate::aixi::environment::Environment; +#[cfg(feature = "aixi-gameengine")] +use crate::aixi::gameengine::build_builtin_environment as build_gameengine_builtin_environment; +#[cfg(feature = "vm")] +use crate::aixi::vm_nyx::{NyxVmConfig, NyxVmEnvironment}; +use crate::aixi::warmstart::{ + WarmStartExactJhTeacherDataset, validate_warmstart_teacher_dataset_for_compiled_planner_run, +}; +use crate::spec::{self, AssetRef, BuiltinEnvironmentSpec, CompiledPlannerRunSpec, SpecDocument}; +use std::path::Path; + +/// Load and validate a warm-start teacher dataset asset referenced by a compiled planner run. +pub fn load_warmstart_exact_jh_teacher_dataset( + compiled: &CompiledPlannerRunSpec, + asset_id: &str, +) -> anyhow::Result { + let binding = compiled + .resolved_assets() + .iter() + .find(|entry| entry.id == asset_id) + .ok_or_else(|| anyhow::anyhow!("unknown warm-start teacher_dataset_asset '{asset_id}'"))?; + let AssetRef::Filesystem(path) = &binding.asset; + let bytes = std::fs::read(path).map_err(|err| { + anyhow::anyhow!( + "failed to read warm-start teacher_dataset_asset '{}': {err}", + path.display() + ) + })?; + let teacher = + WarmStartExactJhTeacherDataset::from_json_slice(&bytes).map_err(anyhow::Error::msg)?; + validate_warmstart_teacher_dataset_for_compiled_planner_run(compiled, &teacher) + .map_err(anyhow::Error::msg)?; + Ok(teacher) +} + +/// Validate a teacher dataset contract against a compiled planner run. +pub fn validate_warmstart_exact_jh_teacher_contract( + compiled: &CompiledPlannerRunSpec, + teacher: &WarmStartExactJhTeacherDataset, +) -> anyhow::Result<()> { + crate::aixi::warmstart::validate_warmstart_teacher_against_compiled_planner_run( + compiled, + &teacher.contract, + ) + .map_err(|err| anyhow::anyhow!("{err}")) +} + +/// Compile a planner_run document from a filesystem path. +pub fn compile_planner_run_document( + path: &str, + caller: &str, +) -> anyhow::Result { + let config_dir = Path::new(path).parent().unwrap_or(Path::new(".")); + let document = spec::load_spec_document(path).map_err(anyhow::Error::msg)?; + match document { + SpecDocument::PlannerRun(spec) => spec + .compile_in(&spec::SpecEnvironment::new(config_dir)) + .map_err(anyhow::Error::msg), + other => Err(anyhow::anyhow!( + "{caller} expects a planner_run document, found kind '{}'", + other.kind_str() + )), + } +} + +fn build_builtin_environment(spec: BuiltinEnvironmentSpec) -> anyhow::Result> { + #[cfg(feature = "aixi-gameengine")] + { + build_gameengine_builtin_environment(spec).map_err(anyhow::Error::new) + } + #[cfg(not(feature = "aixi-gameengine"))] + { + Err(anyhow::anyhow!( + "builtin environment '{}' requires feature 'aixi-gameengine'", + spec.canonical_name() + )) + } +} + +/// Build the environment declared by a compiled planner run. +pub fn build_planner_environment( + compiled: &CompiledPlannerRunSpec, +) -> anyhow::Result<(Box, &'static str)> { + #[allow(unreachable_patterns)] + match &compiled.canonical_spec().environment { + spec::EnvironmentSpec::Builtin { builtin } => Ok(( + build_builtin_environment(*builtin)?, + builtin.canonical_name(), + )), + #[cfg(feature = "vm")] + spec::EnvironmentSpec::NyxVm(vm) => { + let config = NyxVmConfig::from_environment_spec(vm, compiled.resolved_assets()) + .map_err(anyhow::Error::msg)?; + Ok((Box::new(NyxVmEnvironment::new(config)?), "vm")) + } + #[cfg(not(feature = "vm"))] + other => Err(anyhow::anyhow!( + "unsupported environment variant '{}' in this build", + other.kind_str() + )), + } +} + +/// Validate that the environment action alphabet matches the planner interface. +pub fn validate_action_alphabet( + compiled: &CompiledPlannerRunSpec, + env: &dyn Environment, +) -> anyhow::Result<()> { + let actual: ActionAlphabet = env.get_num_actions(); + let expected: ActionAlphabet = compiled.interface().agent_actions; + if actual != expected { + return Err(anyhow::anyhow!( + "action_alphabet_mismatch: planner interface declares {} actions but environment exposes {}", + expected, + actual + )); + } + Ok(()) +} + +/// Validate the full environment interface against the compiled planner contract. +pub(crate) fn validate_environment_interface( + compiled: &CompiledPlannerRunSpec, + env: &dyn Environment, +) -> anyhow::Result<()> { + validate_action_alphabet(compiled, env)?; + let interface = compiled.interface(); + let expected_action_bits: usize = interface.agent_actions.action_bits(); + let actual_action_bits: usize = env.get_action_bits(); + if actual_action_bits != expected_action_bits { + return Err(anyhow::anyhow!( + "action_bits_mismatch: planner interface declares {} action bits for {} actions but environment exposes {}", + expected_action_bits, + interface.agent_actions, + actual_action_bits + )); + } + let actual_observation_bits: usize = env.get_observation_bits(); + if actual_observation_bits != interface.observation_bits { + return Err(anyhow::anyhow!( + "observation_bits_mismatch: planner interface declares {} observation bits but environment exposes {}", + interface.observation_bits, + actual_observation_bits + )); + } + let actual_reward_bits: usize = env.get_reward_bits(); + if actual_reward_bits != interface.reward_bits { + return Err(anyhow::anyhow!( + "reward_bits_mismatch: planner interface declares {} reward bits but environment exposes {}", + interface.reward_bits, + actual_reward_bits + )); + } + Ok(()) +} diff --git a/crates/infotheory/src/aixi/planner_spec.rs b/crates/infotheory/src/aixi/planner_spec.rs new file mode 100644 index 00000000..7bcdecf7 --- /dev/null +++ b/crates/infotheory/src/aixi/planner_spec.rs @@ -0,0 +1,61 @@ +//! Shared planner-run spec builder utilities for AIXI/AIQI controllers. + +use crate::aixi::common::{ActionAlphabet, ObservationKeyMode, resolve_random_seed}; +use crate::spec::{ + BuiltinEnvironmentSpec, ControllerSpec, EnvironmentSpec, PlannerInterfaceSpec, PlannerRunSpec, + PlannerRuntimeSpec, +}; + +/// Canonical planner/environment interface settings shared by AIXI-family builders. +#[derive(Clone, Copy, Debug)] +pub(crate) struct PlannerInterfaceConfig { + pub observation_bits: usize, + pub observation_stream_len: usize, + pub observation_key_mode: ObservationKeyMode, + pub reward_bits: usize, + pub agent_actions: ActionAlphabet, +} + +impl PlannerInterfaceConfig { + fn into_spec(self) -> PlannerInterfaceSpec { + PlannerInterfaceSpec { + observation_bits: self.observation_bits, + observation_stream_len: self.observation_stream_len.max(1), + observation_key_mode: self.observation_key_mode, + reward_bits: self.reward_bits, + agent_actions: self.agent_actions, + } + } +} + +/// Build the canonical planner-run spec used by AIXI-family agents. +/// +/// The builtin environment is a minimal placeholder for programmatic +/// `AgentConfig`/`AiqiConfig` construction: these callers provide the actual +/// environment object at execution time, while the interface section below is +/// the authoritative contract the agent uses. +pub(crate) fn build_default_planner_run_spec( + interface: PlannerInterfaceConfig, + controller: ControllerSpec, + random_seed: Option, +) -> PlannerRunSpec { + PlannerRunSpec { + assets: Vec::new(), + environment: EnvironmentSpec::Builtin { + builtin: BuiltinEnvironmentSpec::CoinFlip, + }, + interface: interface.into_spec(), + controller, + runtime: PlannerRuntimeSpec { + random_seed: Some(resolve_random_seed(random_seed)), + learn_cycles: None, + eval_cycles: None, + terminate_lifetime: 1, + log_every: 1, + perf: false, + vm_perf_only: false, + explore_epsilon: 0.0, + explore_gamma: 1.0, + }, + } +} diff --git a/crates/infotheory/src/aixi/return_law.rs b/crates/infotheory/src/aixi/return_law.rs new file mode 100644 index 00000000..b5c51ee3 --- /dev/null +++ b/crates/infotheory/src/aixi/return_law.rs @@ -0,0 +1,1035 @@ +//! Shared return-label law evaluation for AIQI-style controllers. +//! +//! AIQI and warm-start controllers differ in how labels are produced and decoded, +//! but both perform the same action-selection subproblem: condition a binary +//! predictor on an action, evaluate an autoregressive distribution over +//! finite return labels, normalize valid label mass, and take an expectation +//! under a controller-specific decoder. + +use crate::aixi::common::bits_for_cardinality; +use crate::aixi::model::Predictor; + +const PROBABILITY_FLOOR: f64 = 1e-12; + +/// Bit order used only for return-label codewords. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum ReturnLabelBitOrder { + /// Most-significant bit first, so shallow trie prefixes select contiguous + /// ordered value intervals. + MsbFirst, + /// Least-significant bit first. Kept for tests and comparisons against the + /// former implementation; actions, observations, and rewards still use + /// their existing LSB-first field encoders outside this module. + #[cfg(test)] + LsbFirst, +} + +/// Fixed-width binary code for finite return labels. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct ReturnLabelCodec { + bins: usize, + bits: usize, + order: ReturnLabelBitOrder, +} + +impl ReturnLabelCodec { + /// Build the value-monotone canonical return-label encoding. + /// + /// Labels are ordered by decoded value by the caller's semantic contract. + /// With MSB-first natural binary codewords, every prefix denotes a + /// contiguous interval of label indices, modulo the final invalid tail for + /// non-power-of-two alphabets. + pub(crate) fn value_monotone(bins: usize) -> Self { + assert!(bins > 0, "return-label alphabet must be non-empty"); + Self { + bins, + bits: bits_for_cardinality(bins), + order: ReturnLabelBitOrder::MsbFirst, + } + } + + /// Build the old LSB-first return-label encoding for regression checks. + #[cfg(test)] + pub(crate) fn lsb_first_for_test(bins: usize) -> Self { + assert!(bins > 0, "return-label alphabet must be non-empty"); + Self { + bins, + bits: bits_for_cardinality(bins), + order: ReturnLabelBitOrder::LsbFirst, + } + } + + /// Number of semantic return labels. + pub(crate) fn bins(self) -> usize { + self.bins + } + + /// Fixed codeword width. + pub(crate) fn bits(self) -> usize { + self.bits + } + + /// Encoding order. + #[cfg(test)] + pub(crate) fn order(self) -> ReturnLabelBitOrder { + self.order + } + + /// Return the encoded bit at stream depth `depth`. + pub(crate) fn bit_at(self, label: u64, depth: usize) -> bool { + debug_assert!(depth < self.bits); + let shift = match self.order { + ReturnLabelBitOrder::MsbFirst => self.bits - depth - 1, + #[cfg(test)] + ReturnLabelBitOrder::LsbFirst => depth, + }; + ((label >> shift) & 1) == 1 + } + + /// Update a partially decoded label value with the next stream bit. + fn append_to_partial_value(self, partial: u64, depth: usize, bit: bool) -> u64 { + let shift = match self.order { + ReturnLabelBitOrder::MsbFirst => self.bits - depth - 1, + #[cfg(test)] + ReturnLabelBitOrder::LsbFirst => depth, + }; + if bit { + partial | (1u64 << shift) + } else { + partial + } + } + + /// Return the valid label range under a trie prefix. + /// + /// The range is computed from the actual encoding relation, so it is valid + /// for both MSB-first and LSB-first comparison experiments. + #[cfg(test)] + pub(crate) fn label_range_for_prefix( + self, + prefix_value: u64, + depth: usize, + ) -> Option<(u64, u64)> { + debug_assert!(depth <= self.bits); + if self.order == ReturnLabelBitOrder::MsbFirst { + return self.msb_first_label_range_for_prefix(prefix_value, depth); + } + let mut min_label: Option = None; + let mut max_label: Option = None; + for label in 0..self.bins as u64 { + if self.label_matches_prefix(label, prefix_value, depth) { + min_label = Some(min_label.map_or(label, |current| current.min(label))); + max_label = Some(max_label.map_or(label, |current| current.max(label))); + } + } + min_label.zip(max_label) + } + + #[cfg(test)] + fn msb_first_label_range_for_prefix( + self, + prefix_value: u64, + depth: usize, + ) -> Option<(u64, u64)> { + if depth > self.bits || self.bins == 0 { + return None; + } + // `prefix_value` is indexed by trie depth; translate it into the + // corresponding high-bit numeric interval for the MSB-first code. + let mut base = 0u64; + for idx in 0..depth { + if ((prefix_value >> idx) & 1) == 1 { + base |= 1u64 << (self.bits - idx - 1); + } + } + let remaining_bits = self.bits.saturating_sub(depth); + let suffix_mask = low_bits_mask(remaining_bits); + let max_codeword = base | suffix_mask; + let last_valid = (self.bins as u64).saturating_sub(1); + if base > last_valid { + None + } else { + Some((base, max_codeword.min(last_valid))) + } + } + + #[cfg(test)] + fn label_matches_prefix(self, label: u64, prefix_value: u64, depth: usize) -> bool { + for idx in 0..depth { + if self.bit_at(label, idx) != (((prefix_value >> idx) & 1) == 1) { + return false; + } + } + true + } + + /// Commit a label to the predictor's learned stream. + pub(crate) fn push_label_commit(self, predictor: &mut dyn Predictor, label: u64) -> usize { + for depth in 0..self.bits { + predictor.commit_update(self.bit_at(label, depth)); + } + self.bits + } + + /// Add a label to transient conditioning history without committing it. + pub(crate) fn push_label_history(self, predictor: &mut dyn Predictor, label: u64) -> usize { + for depth in 0..self.bits { + predictor.update_history(self.bit_at(label, depth)); + } + self.bits + } +} + +/// How hypothetical return-prefix bits are applied while evaluating a label law. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum ReturnPrefixUpdate { + /// Update the predictor exactly as if the hypothetical return bits were + /// observed in the model stream, then roll the state back. + Training, + /// Update only transient conditioning history, then pop that history. + #[cfg(test)] + #[allow(dead_code)] + FrozenHistory, +} + +/// Complete exact evaluator variant. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum ReturnLawEvaluator { + /// Former exhaustive codeword loop; retained as a test/reference comparator. + #[cfg(test)] + LeafByLeaf, + /// Memoized shared-prefix trie evaluator, querying each reached internal + /// prefix once. + SharedPrefix, +} + +/// Operational counters for return-label law evaluation. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub(crate) struct ReturnLawEvalStats { + /// Number of logical next-bit probability queries. + pub logical_queries: usize, + /// Number of hypothetical label-prefix bit advances. + pub hypothetical_advances: usize, + /// Number of per-symbol rollbacks/pops. + pub rollbacks: usize, + /// Number of valid label leaves whose mass was accumulated. + pub valid_leaves: usize, + /// Number of invalid code leaves skipped before normalization. + pub invalid_leaves: usize, +} + +/// Normalized return-label law plus evaluation counters. +#[cfg(test)] +#[derive(Clone, Debug, PartialEq)] +pub(crate) struct ReturnLawDistribution { + /// Probabilities in semantic label-index order. + pub probabilities: Vec, + /// Whether evaluation used the uniform-label fallback. + pub used_uniform_fallback: bool, + /// Evaluation counters. + pub stats: ReturnLawEvalStats, +} + +/// Exact expected decoded return plus evaluation counters. +#[derive(Clone, Copy, Debug, PartialEq)] +pub(crate) struct ReturnLawExpectation { + /// Expected value under the normalized valid-label law. + pub value: f64, + /// Whether zero or non-finite mass forced the uniform fallback. + pub used_uniform_fallback: bool, + /// Evaluation counters. + pub stats: ReturnLawEvalStats, +} + +/// Predict and normalize the return-label distribution under `predictor`. +/// +/// With `ReturnPrefixUpdate::Training`, return-law descent deliberately uses +/// per-symbol rollback (`update`/`revert`) rather than +/// [`Predictor::begin_rollback_scope`]. Both policies preserve the same exact +/// trie semantics, but this evaluator opens one speculative branch per trie +/// edge, so scoped rollback would open one scope per edge rather than amortizing +/// a scope over a rollout. The built-in rate predictor's scopes are marker +/// based and measured cost-neutral here; per-symbol rollback remains the +/// clearer local contract for the return-law descent and matches the live +/// no-clone planning path. +/// +/// Cost model: this is a depth-first descent that mutates and rolls back the +/// predictor once per reached trie edge, i.e. `hypothetical_advances` undo +/// operations total. Distribution and scalar-expectation callers share the +/// same descent engine and differ only in how valid leaves are accumulated. +/// Leaf masses are accumulated directly in `f64`; current callers keep return +/// label widths small enough that valid mass does not exhaust linear floating +/// point range before the explicit zero/non-finite-mass fallback applies. +/// +/// This intentionally differs from MCTS rollout simulation, which can use +/// scoped predictor rollback because it opens one scope per rollout rather than +/// one scope per return-label trie edge. +#[cfg(test)] +pub(crate) fn predict_return_law( + predictor: &mut dyn Predictor, + codec: ReturnLabelCodec, + prefix_update: ReturnPrefixUpdate, + evaluator: ReturnLawEvaluator, +) -> ReturnLawDistribution { + if codec.bins() == 1 { + return ReturnLawDistribution { + probabilities: vec![1.0], + used_uniform_fallback: false, + stats: ReturnLawEvalStats { + valid_leaves: 1, + ..ReturnLawEvalStats::default() + }, + }; + } + + let mut masses = vec![0.0; codec.bins()]; + let stats = { + let mut descent = ReturnLawDescent::new(predictor, codec, prefix_update); + let mut sink = DistributionSink { + masses: &mut masses, + }; + descent.evaluate::(evaluator, &mut sink); + descent.stats + }; + let used_uniform_fallback = normalize_masses(&mut masses); + ReturnLawDistribution { + probabilities: masses, + used_uniform_fallback, + stats, + } +} + +/// Predict the normalized expected decoded return without materializing a law vector. +/// +/// For finite decoder values and finite nonzero valid mass, this is +/// mathematically equivalent to materializing the normalized label law and then +/// taking its dot product with `decode_label`, modulo floating-point +/// reassociation. The normal scalar path uses direct linear accumulation, which +/// is the intended regime for current finite-return alphabets. If any valid +/// leaf's linear mass underflows to exactly zero (only reachable for +/// pathologically deep return-label alphabets, and subsuming the degenerate +/// total-mass-zero case), the evaluator reruns the same balanced descent with +/// log-space accumulation so those negligible-but-nonzero leaves do not collapse +/// into a uniform midpoint fallback. +/// +/// If the decoder itself produces a non-finite value, this scalar path uses the +/// uniform-label fallback shape, because a non-finite decoded expectation +/// cannot be repaired by label-law normalization alone. +/// +/// Return-law descent deliberately uses balanced per-symbol rollback rather +/// than the scoped-simulation strategy used by MCTS rollouts: this evaluator +/// opens one speculative branch per return-label trie edge, so scoped rollback +/// would add per-edge scope management without reducing the number of +/// speculative updates. +pub(crate) fn predict_expected_return( + predictor: &mut dyn Predictor, + codec: ReturnLabelCodec, + prefix_update: ReturnPrefixUpdate, + evaluator: ReturnLawEvaluator, + mut decode_label: impl FnMut(u64) -> f64, +) -> ReturnLawExpectation { + if codec.bins() == 1 { + return ReturnLawExpectation { + value: decode_label(0), + used_uniform_fallback: false, + stats: ReturnLawEvalStats { + valid_leaves: 1, + ..ReturnLawEvalStats::default() + }, + }; + } + + let mut sink = LinearExpectedReturnSink { + decode_label: &mut decode_label, + weighted_sum: 0.0, + valid_mass: 0.0, + saw_zero_mass: false, + saw_non_finite_decode: false, + }; + let stats = { + let mut descent = ReturnLawDescent::new(predictor, codec, prefix_update); + descent.evaluate::(evaluator, &mut sink); + descent.stats + }; + + if sink.valid_mass.is_finite() + && sink.valid_mass > 0.0 + && sink.weighted_sum.is_finite() + && !sink.saw_zero_mass + && !sink.saw_non_finite_decode + { + return ReturnLawExpectation { + value: sink.weighted_sum / sink.valid_mass, + used_uniform_fallback: false, + stats, + }; + } + + // `valid_mass == 0.0` (every leaf underflowed) implies `saw_zero_mass`, so + // the per-leaf flag alone gates the log-space rerun. + if sink.saw_zero_mass && !sink.saw_non_finite_decode && sink.weighted_sum.is_finite() { + return predict_expected_return_log_fallback( + predictor, + codec, + prefix_update, + evaluator, + &mut decode_label, + ); + } + + uniform_expectation(codec, stats, &mut decode_label) +} + +fn predict_expected_return_log_fallback( + predictor: &mut dyn Predictor, + codec: ReturnLabelCodec, + prefix_update: ReturnPrefixUpdate, + evaluator: ReturnLawEvaluator, + decode_label: &mut F, +) -> ReturnLawExpectation +where + F: FnMut(u64) -> f64, +{ + let mut sink = LogExpectedReturnSink { + decode_label, + log_valid_mass: None, + log_positive_weighted_mass: None, + log_negative_weighted_mass: None, + saw_non_finite_decode: false, + }; + let stats = { + let mut descent = ReturnLawDescent::new(predictor, codec, prefix_update); + descent.evaluate::(evaluator, &mut sink); + descent.stats + }; + + let Some(log_valid_mass) = sink.log_valid_mass else { + return uniform_expectation(codec, stats, sink.decode_label); + }; + + if !log_valid_mass.is_finite() || sink.saw_non_finite_decode { + return uniform_expectation(codec, stats, sink.decode_label); + } + + let positive = sink + .log_positive_weighted_mass + .map_or(0.0, |log_sum| (log_sum - log_valid_mass).exp()); + let negative = sink + .log_negative_weighted_mass + .map_or(0.0, |log_sum| (log_sum - log_valid_mass).exp()); + + ReturnLawExpectation { + value: positive - negative, + used_uniform_fallback: false, + stats, + } +} + +fn uniform_expectation( + codec: ReturnLabelCodec, + stats: ReturnLawEvalStats, + decode_label: &mut impl FnMut(u64) -> f64, +) -> ReturnLawExpectation { + let uniform_sum: f64 = (0..codec.bins()) + .map(|label| decode_label(label as u64)) + .sum(); + ReturnLawExpectation { + value: uniform_sum / codec.bins() as f64, + used_uniform_fallback: true, + stats, + } +} + +/// Predict the normalized expected semantic label without materializing a law vector. +/// +/// This is the same trie evaluation as [`predict_expected_return`], specialized +/// for affine decoders of the form `offset + label * scale`. It avoids a +/// per-leaf affine evaluation by accumulating raw labels and letting callers +/// apply `offset + label * scale` once per action; the descent still pays the +/// trivial identity cast at each valid leaf. +pub(crate) fn predict_expected_label( + predictor: &mut dyn Predictor, + codec: ReturnLabelCodec, + prefix_update: ReturnPrefixUpdate, + evaluator: ReturnLawEvaluator, +) -> f64 { + let expectation = + predict_expected_return(predictor, codec, prefix_update, evaluator, |label| { + label as f64 + }); + expectation.value +} + +#[cfg(test)] +fn low_bits_mask(bits: usize) -> u64 { + if bits >= u64::BITS as usize { + u64::MAX + } else if bits == 0 { + 0 + } else { + (1u64 << bits) - 1 + } +} + +/// Compute an expectation from a normalized label law and semantic decoder. +#[cfg(test)] +pub(crate) fn expected_decoded_return( + distribution: &[f64], + mut decode_label: impl FnMut(u64) -> f64, +) -> f64 { + distribution + .iter() + .enumerate() + .map(|(label, probability)| decode_label(label as u64) * probability) + .sum() +} + +struct ReturnLawDescent<'a> { + predictor: &'a mut dyn Predictor, + codec: ReturnLabelCodec, + prefix_update: ReturnPrefixUpdate, + stats: ReturnLawEvalStats, +} + +impl<'a> ReturnLawDescent<'a> { + fn new( + predictor: &'a mut dyn Predictor, + codec: ReturnLabelCodec, + prefix_update: ReturnPrefixUpdate, + ) -> Self { + Self { + predictor, + codec, + prefix_update, + stats: ReturnLawEvalStats::default(), + } + } + + fn evaluate( + &mut self, + evaluator: ReturnLawEvaluator, + sink: &mut impl ReturnLawLeafSink, + ) { + match evaluator { + #[cfg(test)] + ReturnLawEvaluator::LeafByLeaf => self.descend_leaf_by_leaf::(sink), + ReturnLawEvaluator::SharedPrefix => { + self.descend_shared_prefix::(0, 0, 1.0, 0.0, sink); + } + } + } + + #[cfg(test)] + fn descend_leaf_by_leaf(&mut self, sink: &mut impl ReturnLawLeafSink) { + for label in 0..self.codec.bins() { + let mut mass = 1.0f64; + let mut log_mass = 0.0f64; + for depth in 0..self.codec.bits() { + let bit = self.codec.bit_at(label as u64, depth); + let q = self + .predictor + .predict_prob(bit) + .clamp(PROBABILITY_FLOOR, 1.0 - PROBABILITY_FLOOR); + self.stats.logical_queries = self.stats.logical_queries.saturating_add(1); + mass *= q; + if TRACK_LOG { + log_mass += q.ln(); + } + self.apply_hypothetical_bit(bit); + } + for _ in 0..self.codec.bits() { + self.revert_hypothetical_bit(); + } + sink.valid_leaf(label as u64, mass, log_mass, &mut self.stats); + } + } + + fn descend_shared_prefix( + &mut self, + depth: usize, + partial_value: u64, + prefix_mass: f64, + prefix_log_mass: f64, + sink: &mut impl ReturnLawLeafSink, + ) { + if depth == self.codec.bits() { + if partial_value < self.codec.bins() as u64 { + sink.valid_leaf(partial_value, prefix_mass, prefix_log_mass, &mut self.stats); + } else { + sink.invalid_leaf(&mut self.stats); + } + return; + } + + let p_one = self + .predictor + .predict_one() + .clamp(PROBABILITY_FLOOR, 1.0 - PROBABILITY_FLOOR); + self.stats.logical_queries = self.stats.logical_queries.saturating_add(1); + + let zero_value = self + .codec + .append_to_partial_value(partial_value, depth, false); + self.apply_hypothetical_bit(false); + let p_zero = 1.0 - p_one; + let zero_log_mass = if TRACK_LOG { + prefix_log_mass + p_zero.ln() + } else { + 0.0 + }; + self.descend_shared_prefix::( + depth + 1, + zero_value, + prefix_mass * p_zero, + zero_log_mass, + sink, + ); + self.revert_hypothetical_bit(); + + let one_value = self + .codec + .append_to_partial_value(partial_value, depth, true); + self.apply_hypothetical_bit(true); + let one_log_mass = if TRACK_LOG { + prefix_log_mass + p_one.ln() + } else { + 0.0 + }; + self.descend_shared_prefix::( + depth + 1, + one_value, + prefix_mass * p_one, + one_log_mass, + sink, + ); + self.revert_hypothetical_bit(); + } + + fn apply_hypothetical_bit(&mut self, bit: bool) { + match self.prefix_update { + ReturnPrefixUpdate::Training => self.predictor.update(bit), + #[cfg(test)] + ReturnPrefixUpdate::FrozenHistory => self.predictor.update_history(bit), + } + self.stats.hypothetical_advances = self.stats.hypothetical_advances.saturating_add(1); + } + + fn revert_hypothetical_bit(&mut self) { + match self.prefix_update { + ReturnPrefixUpdate::Training => self.predictor.revert(), + #[cfg(test)] + ReturnPrefixUpdate::FrozenHistory => self.predictor.pop_history(), + } + self.stats.rollbacks = self.stats.rollbacks.saturating_add(1); + } +} + +trait ReturnLawLeafSink { + fn valid_leaf(&mut self, label: u64, mass: f64, log_mass: f64, stats: &mut ReturnLawEvalStats); + + fn invalid_leaf(&mut self, stats: &mut ReturnLawEvalStats) { + stats.invalid_leaves = stats.invalid_leaves.saturating_add(1); + } +} + +#[cfg(test)] +struct DistributionSink<'a> { + masses: &'a mut [f64], +} + +#[cfg(test)] +impl ReturnLawLeafSink for DistributionSink<'_> { + fn valid_leaf( + &mut self, + label: u64, + mass: f64, + _log_mass: f64, + stats: &mut ReturnLawEvalStats, + ) { + // Defensive for test comparators: shared-prefix descent filters invalid + // codeword tails before calling `valid_leaf`, while leaf-by-leaf always + // iterates valid labels directly. + if let Some(slot) = self.masses.get_mut(label as usize) { + *slot = mass; + stats.valid_leaves = stats.valid_leaves.saturating_add(1); + } else { + self.invalid_leaf(stats); + } + } +} + +struct LinearExpectedReturnSink<'a, F> +where + F: FnMut(u64) -> f64, +{ + decode_label: &'a mut F, + weighted_sum: f64, + valid_mass: f64, + saw_zero_mass: bool, + saw_non_finite_decode: bool, +} + +impl ReturnLawLeafSink for LinearExpectedReturnSink<'_, F> +where + F: FnMut(u64) -> f64, +{ + fn valid_leaf( + &mut self, + label: u64, + mass: f64, + _log_mass: f64, + stats: &mut ReturnLawEvalStats, + ) { + if mass == 0.0 { + self.saw_zero_mass = true; + } + self.valid_mass += mass; + let decoded = (self.decode_label)(label); + if decoded.is_finite() { + self.weighted_sum += decoded * mass; + } else { + self.saw_non_finite_decode = true; + } + stats.valid_leaves = stats.valid_leaves.saturating_add(1); + } +} + +struct LogExpectedReturnSink<'a, F> +where + F: FnMut(u64) -> f64, +{ + decode_label: &'a mut F, + log_valid_mass: Option, + log_positive_weighted_mass: Option, + log_negative_weighted_mass: Option, + saw_non_finite_decode: bool, +} + +impl ReturnLawLeafSink for LogExpectedReturnSink<'_, F> +where + F: FnMut(u64) -> f64, +{ + fn valid_leaf( + &mut self, + label: u64, + _mass: f64, + log_mass: f64, + stats: &mut ReturnLawEvalStats, + ) { + self.log_valid_mass = Some(log_add_exp(self.log_valid_mass, log_mass)); + let decoded = (self.decode_label)(label); + if !decoded.is_finite() { + self.saw_non_finite_decode = true; + } else if decoded > 0.0 { + self.log_positive_weighted_mass = Some(log_add_exp( + self.log_positive_weighted_mass, + log_mass + decoded.ln(), + )); + } else if decoded < 0.0 { + self.log_negative_weighted_mass = Some(log_add_exp( + self.log_negative_weighted_mass, + log_mass + (-decoded).ln(), + )); + } + stats.valid_leaves = stats.valid_leaves.saturating_add(1); + } +} + +fn log_add_exp(current: Option, next: f64) -> f64 { + let Some(current) = current else { + return next; + }; + if current >= next { + current + (next - current).exp().ln_1p() + } else { + next + (current - next).exp().ln_1p() + } +} + +#[cfg(test)] +fn normalize_masses(masses: &mut [f64]) -> bool { + let sum: f64 = masses.iter().sum(); + if !sum.is_finite() || sum <= 0.0 { + let uniform = 1.0 / masses.len() as f64; + masses.fill(uniform); + return true; + } + for mass in masses { + *mass /= sum; + } + false +} + +#[cfg(test)] +mod tests { + use super::*; + + #[derive(Clone, Default)] + struct UniformPredictor { + history: Vec, + } + + impl Predictor for UniformPredictor { + fn update(&mut self, sym: bool) { + self.history.push(sym); + } + + fn update_history(&mut self, sym: bool) { + self.history.push(sym); + } + + fn revert(&mut self) { + self.history.pop(); + } + + fn pop_history(&mut self) { + self.history.pop(); + } + + fn predict_prob(&mut self, _sym: bool) -> f64 { + 0.5 + } + + fn model_name(&self) -> String { + "uniform-test".to_string() + } + + fn boxed_clone(&self) -> Box { + Box::new(self.clone()) + } + } + + #[derive(Clone, Default)] + struct ScopedCountingPredictor { + history: Vec, + scopes: Vec>, + begin_scope_calls: usize, + rollback_scope_calls: usize, + update_calls: usize, + revert_calls: usize, + } + + impl Predictor for ScopedCountingPredictor { + fn update(&mut self, sym: bool) { + self.update_calls = self.update_calls.saturating_add(1); + self.history.push(sym); + } + + fn revert(&mut self) { + self.revert_calls = self.revert_calls.saturating_add(1); + self.history.pop(); + } + + fn begin_rollback_scope(&mut self) { + self.begin_scope_calls = self.begin_scope_calls.saturating_add(1); + self.scopes.push(self.history.clone()); + } + + fn supports_rollback_scope(&self) -> bool { + true + } + + fn rollback_scope(&mut self) -> bool { + self.rollback_scope_calls = self.rollback_scope_calls.saturating_add(1); + let Some(history) = self.scopes.pop() else { + return false; + }; + self.history = history; + true + } + + fn predict_prob(&mut self, _sym: bool) -> f64 { + 0.5 + } + + fn model_name(&self) -> String { + "scoped-counting-test".to_string() + } + + fn boxed_clone(&self) -> Box { + Box::new(self.clone()) + } + } + + #[test] + fn value_monotone_prefixes_are_contiguous_for_power_of_two_labels() { + let codec = ReturnLabelCodec::value_monotone(8); + assert_eq!(codec.order(), ReturnLabelBitOrder::MsbFirst); + assert_eq!(codec.label_range_for_prefix(0, 1), Some((0, 3))); + assert_eq!(codec.label_range_for_prefix(1, 1), Some((4, 7))); + assert_eq!(codec.label_range_for_prefix(0, 2), Some((0, 1))); + assert_eq!(codec.label_range_for_prefix(2, 2), Some((2, 3))); + } + + #[test] + fn value_monotone_prefixes_stop_before_invalid_tail_for_non_power_of_two_labels() { + let codec = ReturnLabelCodec::value_monotone(6); + assert_eq!(codec.label_range_for_prefix(0, 1), Some((0, 3))); + assert_eq!(codec.label_range_for_prefix(1, 1), Some((4, 5))); + assert_eq!(codec.label_range_for_prefix(1, 2), Some((4, 5))); + assert_eq!(codec.label_range_for_prefix(3, 2), None); + } + + #[test] + fn lsb_first_prefixes_interleave_values() { + let codec = ReturnLabelCodec::lsb_first_for_test(8); + assert_eq!(codec.label_range_for_prefix(0, 1), Some((0, 6))); + assert_eq!(codec.label_range_for_prefix(1, 1), Some((1, 7))); + } + + #[test] + fn shared_prefix_matches_leaf_by_leaf_uniform_law_with_fewer_queries() { + let codec = ReturnLabelCodec::value_monotone(8); + let mut leaf_predictor = UniformPredictor::default(); + let mut shared_predictor = UniformPredictor::default(); + let leaf = predict_return_law( + &mut leaf_predictor, + codec, + ReturnPrefixUpdate::Training, + ReturnLawEvaluator::LeafByLeaf, + ); + let shared = predict_return_law( + &mut shared_predictor, + codec, + ReturnPrefixUpdate::Training, + ReturnLawEvaluator::SharedPrefix, + ); + assert_eq!(leaf.probabilities, shared.probabilities); + assert_eq!(leaf.stats.logical_queries, 24); + assert_eq!(shared.stats.logical_queries, 7); + assert_eq!(shared.stats.valid_leaves, 8); + } + + #[test] + fn expected_return_matches_distribution_expectation_for_power_of_two_labels() { + let codec = ReturnLabelCodec::value_monotone(16); + let mut distribution_predictor = UniformPredictor::default(); + let mut expectation_predictor = UniformPredictor::default(); + let distribution = predict_return_law( + &mut distribution_predictor, + codec, + ReturnPrefixUpdate::Training, + ReturnLawEvaluator::SharedPrefix, + ); + let expected_from_distribution = + expected_decoded_return(&distribution.probabilities, |label| (label * label) as f64); + let direct = predict_expected_return( + &mut expectation_predictor, + codec, + ReturnPrefixUpdate::Training, + ReturnLawEvaluator::SharedPrefix, + |label| (label * label) as f64, + ); + + assert!((direct.value - expected_from_distribution).abs() < 1e-14); + assert!(!direct.used_uniform_fallback); + assert_eq!(direct.stats, distribution.stats); + } + + #[test] + fn expected_return_matches_distribution_expectation_for_sparse_code_tail() { + let codec = ReturnLabelCodec::value_monotone(6); + let mut distribution_predictor = UniformPredictor::default(); + let mut expectation_predictor = UniformPredictor::default(); + let distribution = predict_return_law( + &mut distribution_predictor, + codec, + ReturnPrefixUpdate::Training, + ReturnLawEvaluator::SharedPrefix, + ); + let expected_from_distribution = + expected_decoded_return(&distribution.probabilities, |label| label as f64 + 0.25); + let direct = predict_expected_return( + &mut expectation_predictor, + codec, + ReturnPrefixUpdate::Training, + ReturnLawEvaluator::SharedPrefix, + |label| label as f64 + 0.25, + ); + + assert!((direct.value - expected_from_distribution).abs() < 1e-14); + assert!(!direct.used_uniform_fallback); + assert_eq!(direct.stats, distribution.stats); + assert_eq!(direct.stats.invalid_leaves, 2); + } + + #[test] + fn expected_return_matches_distribution_expectation_for_signed_decoder() { + let codec = ReturnLabelCodec::value_monotone(8); + let mut distribution_predictor = UniformPredictor::default(); + let mut expectation_predictor = UniformPredictor::default(); + let distribution = predict_return_law( + &mut distribution_predictor, + codec, + ReturnPrefixUpdate::Training, + ReturnLawEvaluator::SharedPrefix, + ); + let expected_from_distribution = + expected_decoded_return(&distribution.probabilities, |label| label as f64 - 3.5); + let direct = predict_expected_return( + &mut expectation_predictor, + codec, + ReturnPrefixUpdate::Training, + ReturnLawEvaluator::SharedPrefix, + |label| label as f64 - 3.5, + ); + + assert!((direct.value - expected_from_distribution).abs() < 1e-14); + assert!(!direct.used_uniform_fallback); + assert_eq!(direct.stats, distribution.stats); + } + + #[test] + fn log_add_exp_retains_tiny_terms_without_linear_underflow() { + let combined = log_add_exp(Some(-1000.0), -1000.0); + assert!((combined - (-1000.0 + std::f64::consts::LN_2)).abs() < 1e-12); + } + + #[test] + fn expected_return_uses_uniform_fallback_for_non_finite_decoder_sum() { + let codec = ReturnLabelCodec::value_monotone(4); + let mut predictor = UniformPredictor::default(); + let direct = predict_expected_return( + &mut predictor, + codec, + ReturnPrefixUpdate::Training, + ReturnLawEvaluator::SharedPrefix, + |label| { + if label == 0 { + f64::INFINITY + } else { + label as f64 + } + }, + ); + + assert!(direct.used_uniform_fallback); + assert!(direct.value.is_infinite()); + assert!(direct.value.is_sign_positive()); + assert_eq!(direct.stats.valid_leaves, 4); + } + + #[test] + fn shared_prefix_uses_per_symbol_rollbacks_for_return_law_training() { + let codec = ReturnLabelCodec::value_monotone(4); + let mut predictor = ScopedCountingPredictor::default(); + let law = predict_return_law( + &mut predictor, + codec, + ReturnPrefixUpdate::Training, + ReturnLawEvaluator::SharedPrefix, + ); + + assert_eq!(law.probabilities, vec![0.25; 4]); + assert_eq!(law.stats.logical_queries, 3); + assert_eq!(law.stats.hypothetical_advances, 6); + assert_eq!(law.stats.rollbacks, 6); + assert_eq!(law.stats.valid_leaves, 4); + assert_eq!(predictor.update_calls, 6); + assert_eq!(predictor.revert_calls, 6); + assert_eq!(predictor.begin_scope_calls, 0); + assert_eq!(predictor.rollback_scope_calls, 0); + assert!(predictor.history.is_empty()); + assert!(predictor.scopes.is_empty()); + } +} diff --git a/crates/infotheory/src/aixi/test_envs.rs b/crates/infotheory/src/aixi/test_envs.rs new file mode 100644 index 00000000..be9d93ac --- /dev/null +++ b/crates/infotheory/src/aixi/test_envs.rs @@ -0,0 +1,126 @@ +#![allow(dead_code)] + +use infotheory::aixi::common::{Action, PerceptVal, RandomGenerator, Reward}; +use infotheory::aixi::environment::Environment; + +#[derive(Default)] +pub struct DeterministicBinaryEnv { + cycle: usize, + last_action: Action, + obs: PerceptVal, + rew: Reward, +} + +impl DeterministicBinaryEnv { + pub fn new() -> Self { + Self::default() + } +} + +impl Environment for DeterministicBinaryEnv { + fn perform_action(&mut self, action: Action) { + self.obs = if self.cycle == 0 { + 0 + } else { + (self.last_action + 1) % 2 + }; + self.rew = if action == self.obs { 1 } else { 0 }; + self.last_action = action; + self.cycle += 1; + } + + fn get_observation(&self) -> PerceptVal { + self.obs + } + + fn get_reward(&self) -> Reward { + self.rew + } + + fn is_finished(&self) -> bool { + false + } + + fn get_observation_bits(&self) -> usize { + 1 + } + + fn get_reward_bits(&self) -> usize { + 1 + } + + fn get_action_bits(&self) -> usize { + 1 + } + + fn min_reward(&self) -> Reward { + 0 + } + + fn max_reward(&self) -> Reward { + 1 + } +} + +pub struct SeededCoinFlipEnv { + p: f64, + obs: PerceptVal, + rew: Reward, + rng: RandomGenerator, +} + +impl SeededCoinFlipEnv { + pub fn new(p: f64) -> Self { + let mut env = Self { + p, + obs: 0, + rew: 0, + rng: RandomGenerator::from_seed(1), + }; + env.obs = env.next_observation(); + env + } + + fn next_observation(&mut self) -> PerceptVal { + if self.rng.gen_bool(self.p) { 1 } else { 0 } + } +} + +impl Environment for SeededCoinFlipEnv { + fn perform_action(&mut self, action: Action) { + self.obs = self.next_observation(); + self.rew = if action == self.obs { 1 } else { 0 }; + } + + fn get_observation(&self) -> PerceptVal { + self.obs + } + + fn get_reward(&self) -> Reward { + self.rew + } + + fn is_finished(&self) -> bool { + false + } + + fn get_observation_bits(&self) -> usize { + 1 + } + + fn get_reward_bits(&self) -> usize { + 1 + } + + fn get_action_bits(&self) -> usize { + 1 + } + + fn min_reward(&self) -> Reward { + 0 + } + + fn max_reward(&self) -> Reward { + 1 + } +} diff --git a/crates/infotheory/src/aixi/vm_nyx.rs b/crates/infotheory/src/aixi/vm_nyx.rs new file mode 100644 index 00000000..c7ec32fc --- /dev/null +++ b/crates/infotheory/src/aixi/vm_nyx.rs @@ -0,0 +1,3440 @@ +//! High-performance VM-backed AIXI environment using nyx-lite (Firecracker). +//! +//! This module provides a VM environment implementation built on top of nyx-lite, +//! enabling high-frequency snapshot-based resets for fast experimentation (hardware and +//! guest behavior dependent). +//! +//! ## Architecture +//! +//! The environment uses Firecracker's KVM-based microVM with nyx-lite's incremental +//! snapshot and reset capabilities. Communication with the guest occurs via: +//! +//! 1. **Shared Memory**: Zero-copy data transfer between host and guest +//! 2. **Hypercalls**: Control plane communication (snapshot, done, etc.) +//! 3. **Serial PTY**: Optional console I/O for simpler protocols +//! +//! ## Design Principles +//! +//! - **Universal**: Not biased towards any specific use case (fuzzing, etc.) +//! - **High Performance**: Leverages incremental snapshots and dirty page tracking +//! - **Configurable**: Pluggable reward policies, action sources, observation modes +//! - **Information-Theoretic**: Built-in support for entropy-based metrics + +use crate::aixi::common::{Action, ActionAlphabet, PerceptVal, RandomGenerator, Reward}; +use crate::aixi::environment::Environment; +use crate::api::{ + CompiledRateBackend, RateBackend, empirical_entropy_bytes, try_cross_entropy_rate_backend, + try_entropy_rate_backend, +}; +#[cfg(feature = "backend-ctw")] +use crate::backends::ctw::{ContextTree, FacContextTree, ctw_symbol_bit_msb}; +#[cfg(feature = "backend-rosa")] +use crate::backends::rosaplus::RosaPlus; +#[cfg(feature = "backend-zpaq")] +use crate::backends::zpaq_rate::ZpaqRateModel; +#[cfg(feature = "backend-rwkv")] +use crate::coders::softmax_pdf_inplace; +use crate::error::{InfotheoryError, InfotheoryResult}; +#[cfg(feature = "backend-mamba")] +use crate::mambazip; +#[cfg(feature = "backend-mamba")] +use crate::mambazip::Compressor as MambaCompressor; +use crate::mixture::OnlineBytePredictor; +#[cfg(feature = "backend-rwkv")] +use crate::rwkvzip::Compressor; +use crate::spec::{ + AssetBinding, AssetRef, EnvironmentSpec, ResolvedAssetBinding, SharedMemoryPolicySpec, + SpecEnvironment, VmActionFilterSpec, VmEnvironmentSpec, VmFuzzMutatorSpec, + VmObservationPolicySpec, VmObservationStreamModeSpec, VmPayloadEncodingSpec, + VmRewardPolicySpec, VmRewardShapingSpec, VmRuntimeActionSourceSpec, VmTraceSpec, +}; +use serde_json::Value; +use std::borrow::Cow; +use std::fs::OpenOptions; +use std::io::Write; +use std::path::Path; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +// Re-export nyx-lite types for external use +pub use nyx_lite::mem::SharedMemoryRegion; +pub use nyx_lite::snapshot::NyxSnapshot; +pub use nyx_lite::{ExitReason, NyxVM, SharedMemoryPolicy}; + +// ============================================================================ +// Encoding Types +// ============================================================================ + +/// Payload encoding for wire protocol. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub enum PayloadEncoding { + /// Treat payloads as UTF-8/text bytes. + Utf8, + /// Treat payloads as hexadecimal text. + Hex, +} + +impl PayloadEncoding { + /// Decode a wire payload string into raw bytes using this encoding. + pub fn decode(self, s: &str) -> anyhow::Result> { + match self { + Self::Utf8 => Ok(s.as_bytes().to_vec()), + Self::Hex => hex_decode(s), + } + } + + /// Encode raw bytes for transport over the configured wire protocol. + pub fn encode(self, bytes: &[u8]) -> String { + match self { + Self::Utf8 => String::from_utf8_lossy(bytes).to_string(), + Self::Hex => hex_encode(bytes), + } + } +} + +impl std::str::FromStr for PayloadEncoding { + type Err = &'static str; + + fn from_str(s: &str) -> Result { + match s { + "utf8" => Ok(Self::Utf8), + "hex" => Ok(Self::Hex), + _ => Err("unknown payload encoding"), + } + } +} + +fn hex_decode(s: &str) -> anyhow::Result> { + let mut out = Vec::with_capacity(s.len() / 2); + let mut buf = 0u8; + let mut high = true; + for c in s.bytes() { + let v = match c { + b'0'..=b'9' => c - b'0', + b'a'..=b'f' => c - b'a' + 10, + b'A'..=b'F' => c - b'A' + 10, + b' ' | b'\n' | b'\r' | b'\t' => continue, + _ => return Err(anyhow::anyhow!("invalid hex byte: {}", c as char)), + }; + if high { + buf = v << 4; + high = false; + } else { + buf |= v; + out.push(buf); + high = true; + } + } + if !high { + return Err(anyhow::anyhow!("hex string has odd length")); + } + Ok(out) +} + +fn resolve_relative_path(base: &Path, path: &str) -> String { + let p = Path::new(path); + if p.is_absolute() { + path.to_string() + } else { + base.join(p).to_string_lossy().to_string() + } +} + +fn rewrite_firecracker_config_paths(config_path: &str, raw_json: &str) -> anyhow::Result { + let base_dir = Path::new(config_path) + .parent() + .unwrap_or_else(|| Path::new(".")); + let mut v: Value = serde_json::from_str(raw_json)?; + + if let Some(boot) = v.get_mut("boot-source") { + if let Some(path_val) = boot.get_mut("kernel_image_path") + && let Some(path_str) = path_val.as_str() + { + let resolved = resolve_relative_path(base_dir, path_str); + *path_val = Value::String(resolved); + } + if let Some(path_val) = boot.get_mut("initrd_path") + && let Some(path_str) = path_val.as_str() + { + let resolved = resolve_relative_path(base_dir, path_str); + *path_val = Value::String(resolved); + } + } + + if let Some(drives) = v.get_mut("drives").and_then(|d| d.as_array_mut()) { + for drive in drives { + if let Some(path_val) = drive.get_mut("path_on_host") + && let Some(path_str) = path_val.as_str() + { + let resolved = resolve_relative_path(base_dir, path_str); + *path_val = Value::String(resolved); + } + } + } + + Ok(serde_json::to_string(&v)?) +} + +fn hex_encode(bytes: &[u8]) -> String { + let mut s = String::with_capacity(bytes.len() * 2); + for b in bytes { + s.push(hex_digit(b >> 4)); + s.push(hex_digit(b & 0x0F)); + } + s +} + +fn hex_digit(v: u8) -> char { + match v { + 0..=9 => (b'0' + v) as char, + _ => (b'a' + (v - 10)) as char, + } +} + +// ============================================================================ +// Guest Communication Protocol +// ============================================================================ + +/// Hypercall identifiers (must match guest implementation). +/// These are exported for use by custom guest programs. +#[allow(dead_code)] +pub const HYPERCALL_EXECDONE: u64 = 0x656e6f6463657865; // "execdone" +/// Guest requested host-side snapshot operation. +#[allow(dead_code)] +pub const HYPERCALL_SNAPSHOT: u64 = 0x746f687370616e73; // "snapshot" +/// Guest announced nyx-lite protocol/version handshake. +#[allow(dead_code)] +pub const HYPERCALL_NYX_LITE: u64 = 0x6574696c2d78796e; // "nyx-lite" +/// Guest requested shared memory initialization/refresh. +#[allow(dead_code)] +pub const HYPERCALL_SHAREMEM: u64 = 0x6d656d6572616873; // "sharemem" +/// Guest emitted a debug-print hypercall payload. +#[allow(dead_code)] +pub const HYPERCALL_DBGPRINT: u64 = 0x746e697270676264; // "dbgprint" + +const SHARED_ACTION_LEN_OFFSET: u64 = 0; +const SHARED_RESP_LEN_OFFSET: u64 = 8; +const SHARED_PAYLOAD_OFFSET: u64 = 16; + +/// Protocol configuration for structured communication. +#[derive(Clone, Debug)] +#[non_exhaustive] +pub struct NyxProtocolConfig { + /// Prefix for action messages. + pub action_prefix: String, + /// Suffix for action messages. + pub action_suffix: String, + /// Prefix for observation responses. + pub obs_prefix: String, + /// Prefix for reward responses. + pub rew_prefix: String, + /// Prefix for done indicator. + pub done_prefix: String, + /// Prefix for data payloads. + pub data_prefix: String, + /// Wire encoding for payloads. + pub wire_encoding: PayloadEncoding, +} + +impl Default for NyxProtocolConfig { + fn default() -> Self { + Self { + action_prefix: "ACT ".to_string(), + action_suffix: "\n".to_string(), + obs_prefix: "OBS ".to_string(), + rew_prefix: "REW ".to_string(), + done_prefix: "DONE ".to_string(), + data_prefix: "DATA ".to_string(), + wire_encoding: PayloadEncoding::Hex, + } + } +} + +// ============================================================================ +// Action Configuration +// ============================================================================ + +/// A single action specification. +#[derive(Clone, Debug)] +#[non_exhaustive] +pub struct NyxActionSpec { + /// Optional human-readable name. + pub name: Option, + /// Raw payload bytes to send. + pub payload: Vec, +} + +impl NyxActionSpec { + /// Create an action specification with no explicit name. + pub fn new(payload: Vec) -> Self { + Self { + name: None, + payload, + } + } + + /// Create an action specification with a human-readable name. + pub fn named(name: impl Into, payload: Vec) -> Self { + Self { + name: Some(name.into()), + payload, + } + } +} + +impl Default for NyxActionSpec { + fn default() -> Self { + Self::new(Vec::new()) + } +} + +/// Fuzzing mutator types. +#[derive(Clone, Debug)] +#[non_exhaustive] +pub enum FuzzMutator { + /// Flip one random bit. + FlipBit, + /// Flip one full byte. + FlipByte, + /// Insert a random byte at a random position. + InsertByte, + /// Delete one random byte. + DeleteByte, + /// Splice bytes from an existing seed input. + SpliceSeed, + /// Replace the working input with a seed input. + ResetSeed, + /// Apply a short sequence of random mutations. + Havoc, +} + +/// Fuzzing configuration for action generation. +#[derive(Clone, Debug)] +#[non_exhaustive] +pub struct NyxFuzzConfig { + /// Corpus used for seed/reset/splice operations. + pub seeds: Vec>, + /// Mutator set available for action generation. + pub mutators: Vec, + /// Minimum generated action length. + pub min_len: usize, + /// Maximum generated action length. + pub max_len: usize, + /// Optional dictionary tokens for insertion/splicing. + pub dictionary: Vec>, + /// Deterministic RNG seed for mutation sampling. + pub rng_seed: u64, +} + +impl NyxFuzzConfig { + /// Create fuzzing configuration with sensible defaults. + pub fn new(seeds: Vec>) -> Self { + Self { + seeds, + mutators: vec![FuzzMutator::Havoc], + min_len: 1, + max_len: 4096, + dictionary: Vec::new(), + rng_seed: 0, + } + } +} + +impl Default for NyxFuzzConfig { + fn default() -> Self { + Self::new(Vec::new()) + } +} + +/// Source of actions for the environment. +#[derive(Clone, Debug)] +#[non_exhaustive] +pub enum NyxActionSource { + /// Fixed set of action payloads. + Literal(Vec), + /// Mutation-based action generation. + Fuzz(NyxFuzzConfig), +} + +// ============================================================================ +// Observation Configuration +// ============================================================================ + +/// How observations are derived from guest output. +#[derive(Clone, Copy, Debug)] +#[non_exhaustive] +pub enum NyxObservationPolicy { + /// Parse structured OBS/REW/DONE messages from guest. + FromGuest, + /// Hash raw output to derive observation. + OutputHash, + /// Use raw output bytes as observation stream. + RawOutput, + /// Use shared memory contents as observation. + SharedMemory, +} + +/// Stream normalization mode. +#[derive(Clone, Copy, Debug)] +#[non_exhaustive] +pub enum NyxObservationStreamMode { + /// Pad short streams, truncate long ones. + PadTruncate, + /// Only pad short streams. + Pad, + /// Only truncate long streams. + Truncate, +} + +// ============================================================================ +// Reward Configuration +// ============================================================================ + +/// How rewards are computed. +#[derive(Clone)] +#[non_exhaustive] +pub enum NyxRewardPolicy { + /// Parse reward from guest response. + FromGuest, + /// Pattern matching on output. + Pattern { + /// Substring/pattern tested against guest output. + pattern: String, + /// Reward returned when the pattern does not match. + base_reward: i64, + /// Additional reward added when the pattern matches. + bonus_reward: i64, + }, + /// Custom reward function (callback-based). + Custom(Arc Reward + Send + Sync>), +} + +/// Optional reward shaping (additive to base reward). +/// +/// Algorithmic configuration for the entropy estimator (such as ROSA's +/// `max_order`) lives inside the active `stats_backend`'s +/// [`crate::api::RateBackend`] variant. +#[derive(Clone, Debug)] +#[non_exhaustive] +pub enum NyxRewardShaping { + /// Entropy reduction vs baseline. + EntropyReduction { + /// Reference bytes used as baseline data distribution. + baseline_bytes: Vec, + /// Scaling factor applied to the shaping term. + scale: f64, + /// Optional additive bonus when guest crashes. + crash_bonus: Option, + /// Optional additive bonus when guest times out. + timeout_bonus: Option, + }, + /// Entropy of trace data (online learning). + TraceEntropy { + /// Scaling factor applied to the shaping term. + scale: f64, + /// If true, normalize by trace length. + normalize: bool, + }, +} + +impl std::fmt::Debug for NyxRewardPolicy { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::FromGuest => write!(f, "FromGuest"), + Self::Pattern { + pattern, + base_reward, + bonus_reward, + } => f + .debug_struct("Pattern") + .field("pattern", pattern) + .field("base_reward", base_reward) + .field("bonus_reward", bonus_reward) + .finish(), + Self::Custom(_) => write!(f, "Custom()"), + } + } +} + +// ============================================================================ +// Action Filtering +// ============================================================================ + +/// Information-theoretic action filtering. +/// +/// Algorithmic configuration for the entropy estimator (such as ROSA's +/// `max_order`) lives inside the active `stats_backend`'s +/// [`crate::api::RateBackend`] variant. +#[derive(Clone, Debug)] +#[non_exhaustive] +pub struct NyxActionFilter { + /// Minimum entropy threshold. + pub min_entropy: Option, + /// Maximum entropy threshold. + pub max_entropy: Option, + /// Minimum intrinsic dependence. + pub min_intrinsic_dependence: Option, + /// Minimum novelty (cross-entropy vs prior). + pub min_novelty: Option, + /// Prior corpus for novelty computation. + pub novelty_prior: Option>, + /// Reward to assign when action is rejected. + pub reject_reward: Option, +} + +impl NyxActionFilter { + /// Create an action filter with no active constraints. + pub fn new() -> Self { + Self::default() + } +} + +impl Default for NyxActionFilter { + fn default() -> Self { + Self { + min_entropy: None, + max_entropy: None, + min_intrinsic_dependence: None, + min_novelty: None, + novelty_prior: None, + reject_reward: None, + } + } +} + +// ============================================================================ +// Trace Configuration +// ============================================================================ + +/// Configuration for trace collection and analysis. +#[derive(Clone, Debug)] +#[non_exhaustive] +pub struct NyxTraceConfig { + /// Shared memory region name for trace data. + pub shared_region_name: Option, + /// Maximum bytes to collect per step. + pub max_bytes: usize, + /// Reset trace model on episode boundary. + pub reset_on_episode: bool, +} + +impl NyxTraceConfig { + /// Create trace configuration with defaults used by the CLI parser. + pub fn new() -> Self { + Self::default() + } +} + +impl Default for NyxTraceConfig { + fn default() -> Self { + Self { + shared_region_name: Some("trace".to_string()), + max_bytes: 1_000_000, + reset_on_episode: false, + } + } +} + +// ============================================================================ +// Main Configuration +// ============================================================================ + +/// Complete configuration for the nyx-lite VM environment. +#[derive(Clone)] +#[non_exhaustive] +pub struct NyxVmConfig { + /// Path to Firecracker JSON config. + pub firecracker_config: String, + /// Instance ID for the VM. + pub instance_id: String, + + // Shared memory configuration + /// Name of the shared memory region for communication. + pub shared_region_name: String, + /// Size of the shared memory region. + pub shared_region_size: usize, + /// Shared memory policy (snapshot vs preserve). + pub shared_memory_policy: SharedMemoryPolicy, + + // Timing configuration + /// Timeout for each step. + pub step_timeout: Duration, + /// Timeout for initial boot. + pub boot_timeout: Duration, + + // Episode configuration + /// Number of steps per episode. + pub episode_steps: usize, + /// Cost subtracted from reward each step. + pub step_cost: i64, + + // Observation configuration + /// Observation derivation policy. + pub observation_policy: NyxObservationPolicy, + /// Bits per observation symbol. + pub observation_bits: usize, + /// Number of observation symbols per action. + pub observation_stream_len: usize, + /// Stream normalization mode. + pub observation_stream_mode: NyxObservationStreamMode, + /// Padding byte for short streams. + pub observation_pad_byte: u8, + + // Reward configuration + /// Bits for reward encoding. + pub reward_bits: usize, + /// Reward computation policy. + pub reward_policy: NyxRewardPolicy, + /// Optional reward shaping (additive; non-canonical). + pub reward_shaping: Option, + + // Action configuration + /// Source of actions. + pub action_source: NyxActionSource, + /// Optional action filter. + pub action_filter: Option, + + // Protocol configuration + /// Wire protocol for structured communication. + pub protocol: NyxProtocolConfig, + + // Statistics backend + /// Backend for entropy estimation. + pub stats_backend: RateBackend, + + // Trace configuration + /// Optional trace collection. + pub trace: Option, + + // Debug mode + /// Enable verbose VM/protocol diagnostics. + pub debug_mode: bool, + + // Crash logging + /// Path to log crashes/interesting behaviors (JSONL format). + pub crash_log: Option, +} + +fn default_vm_stats_backend() -> RateBackend { + // Keep the VM default explicit so `vm` can be combined with a narrow + // backend slice instead of inheriting the crate-wide implicit default. + RateBackend::Ctw { depth: 20 } +} + +impl Default for NyxVmConfig { + fn default() -> Self { + Self { + firecracker_config: String::new(), + instance_id: "aixi-nyx".to_string(), + shared_region_name: "shared".to_string(), + shared_region_size: 4096, + shared_memory_policy: SharedMemoryPolicy::Snapshot, + step_timeout: Duration::from_millis(100), + boot_timeout: Duration::from_secs(30), + episode_steps: 100, + step_cost: 0, + observation_policy: NyxObservationPolicy::SharedMemory, + observation_bits: 8, + observation_stream_len: 64, + observation_stream_mode: NyxObservationStreamMode::PadTruncate, + observation_pad_byte: 0, + reward_bits: 8, + reward_policy: NyxRewardPolicy::FromGuest, + reward_shaping: None, + action_source: NyxActionSource::Literal(vec![]), + action_filter: None, + protocol: NyxProtocolConfig::default(), + stats_backend: default_vm_stats_backend(), + trace: None, + debug_mode: false, + crash_log: None, + } + } +} + +impl NyxVmConfig { + fn validate_runtime_invariants(&self) -> InfotheoryResult<()> { + if self.firecracker_config.trim().is_empty() { + return Err(InfotheoryError::invalid_backend_config( + "firecracker_config path must be set", + )); + } + if self.episode_steps == 0 { + return Err(InfotheoryError::invalid_backend_config( + "episode_steps must be > 0", + )); + } + if matches!(self.observation_policy, NyxObservationPolicy::RawOutput) + && self.observation_stream_len == 0 + { + return Err(InfotheoryError::invalid_backend_config( + "observation_stream_len must be > 0 for RawOutput policy", + )); + } + if matches!( + self.reward_shaping, + Some(NyxRewardShaping::TraceEntropy { .. }) + ) && self.trace.is_none() + { + return Err(InfotheoryError::invalid_backend_config( + "vm_trace must be configured for vm_reward_shaping.mode=trace-entropy", + )); + } + + Ok(()) + } + + /// Validate this VM configuration for direct runtime construction. + pub fn validate(&self) -> InfotheoryResult<()> { + self.validate_runtime_invariants() + } + + /// Validate that this VM configuration is representable by canonical spec documents. + pub fn validate_canonical_spec_compatibility(&self) -> InfotheoryResult<()> { + self.validate_runtime_invariants()?; + + let encoding = self.protocol.wire_encoding; + let firecracker_asset = "firecracker_config".to_string(); + let mut assets = vec![AssetBinding { + id: firecracker_asset.clone(), + path: self.firecracker_config.clone(), + }]; + let reward_shaping = match &self.reward_shaping { + Some(NyxRewardShaping::EntropyReduction { + baseline_bytes: _, + scale, + crash_bonus, + timeout_bonus, + }) => { + let asset_id = "reward_shaping_baseline".to_string(); + assets.push(AssetBinding { + id: asset_id.clone(), + path: "inline://reward_shaping_baseline".to_string(), + }); + Some(VmRewardShapingSpec::EntropyReduction { + baseline_asset: asset_id, + scale: *scale, + crash_bonus: *crash_bonus, + timeout_bonus: *timeout_bonus, + }) + } + Some(NyxRewardShaping::TraceEntropy { scale, normalize }) => { + Some(VmRewardShapingSpec::TraceEntropy { + scale: *scale, + normalize: *normalize, + }) + } + None => None, + }; + let action_filter = self.action_filter.as_ref().map(|filter| { + let novelty_prior_asset = filter.novelty_prior.as_ref().map(|_| { + let asset_id = "action_filter_novelty_prior".to_string(); + assets.push(AssetBinding { + id: asset_id.clone(), + path: "inline://action_filter_novelty_prior".to_string(), + }); + asset_id + }); + VmActionFilterSpec { + min_entropy: filter.min_entropy, + max_entropy: filter.max_entropy, + min_intrinsic_dependence: filter.min_intrinsic_dependence, + min_novelty: filter.min_novelty, + novelty_prior_asset, + reject_reward: filter.reject_reward, + } + }); + let action_source = match &self.action_source { + NyxActionSource::Literal(actions) => VmRuntimeActionSourceSpec::Literal { + names: actions.iter().map(|action| action.name.clone()).collect(), + payloads: actions + .iter() + .map(|action| encoding.encode(&action.payload)) + .collect(), + encoding: match encoding { + PayloadEncoding::Utf8 => VmPayloadEncodingSpec::Utf8, + PayloadEncoding::Hex => VmPayloadEncodingSpec::Hex, + }, + }, + NyxActionSource::Fuzz(fuzz) => VmRuntimeActionSourceSpec::Fuzz { + seeds: fuzz + .seeds + .iter() + .map(|seed| encoding.encode(seed)) + .collect(), + encoding: match encoding { + PayloadEncoding::Utf8 => VmPayloadEncodingSpec::Utf8, + PayloadEncoding::Hex => VmPayloadEncodingSpec::Hex, + }, + mutators: fuzz + .mutators + .iter() + .map(|mutator| match mutator { + FuzzMutator::FlipBit => VmFuzzMutatorSpec::FlipBit, + FuzzMutator::FlipByte => VmFuzzMutatorSpec::FlipByte, + FuzzMutator::InsertByte => VmFuzzMutatorSpec::InsertByte, + FuzzMutator::DeleteByte => VmFuzzMutatorSpec::DeleteByte, + FuzzMutator::SpliceSeed => VmFuzzMutatorSpec::SpliceSeed, + FuzzMutator::ResetSeed => VmFuzzMutatorSpec::ResetSeed, + FuzzMutator::Havoc => VmFuzzMutatorSpec::Havoc, + }) + .collect(), + min_len: fuzz.min_len, + max_len: fuzz.max_len, + dictionary: fuzz + .dictionary + .iter() + .map(|entry| encoding.encode(entry)) + .collect(), + rng_seed: fuzz.rng_seed, + }, + }; + let reward_policy = match &self.reward_policy { + NyxRewardPolicy::FromGuest => VmRewardPolicySpec::FromGuest, + NyxRewardPolicy::Pattern { + pattern, + base_reward, + bonus_reward, + } => VmRewardPolicySpec::Pattern { + pattern: pattern.clone(), + base_reward: *base_reward, + bonus_reward: *bonus_reward, + }, + NyxRewardPolicy::Custom(_) => { + return Err(InfotheoryError::invalid_backend_config( + "custom Nyx reward callbacks are not representable in canonical specs", + )); + } + }; + let environment = EnvironmentSpec::NyxVm(VmEnvironmentSpec { + firecracker_config_asset: firecracker_asset, + instance_id: self.instance_id.clone(), + shared_region_name: self.shared_region_name.clone(), + shared_region_size: self.shared_region_size, + shared_memory_policy: match self.shared_memory_policy { + SharedMemoryPolicy::Preserve => SharedMemoryPolicySpec::Preserve, + SharedMemoryPolicy::Snapshot => SharedMemoryPolicySpec::Snapshot, + }, + step_timeout_ms: self.step_timeout.as_millis() as u64, + boot_timeout_ms: self.boot_timeout.as_millis() as u64, + episode_steps: self.episode_steps, + step_cost: self.step_cost, + observation_policy: match self.observation_policy { + NyxObservationPolicy::FromGuest => VmObservationPolicySpec::FromGuest, + NyxObservationPolicy::OutputHash => VmObservationPolicySpec::OutputHash, + NyxObservationPolicy::RawOutput => VmObservationPolicySpec::RawOutput, + NyxObservationPolicy::SharedMemory => VmObservationPolicySpec::SharedMemory, + }, + observation_bits: self.observation_bits, + observation_stream_len: self.observation_stream_len, + observation_stream_mode: match self.observation_stream_mode { + NyxObservationStreamMode::PadTruncate => VmObservationStreamModeSpec::PadTruncate, + NyxObservationStreamMode::Pad => VmObservationStreamModeSpec::Pad, + NyxObservationStreamMode::Truncate => VmObservationStreamModeSpec::Truncate, + }, + observation_pad_byte: self.observation_pad_byte, + reward_bits: self.reward_bits, + reward_policy, + reward_shaping, + action_source, + action_filter, + action_prefix: self.protocol.action_prefix.clone(), + action_suffix: self.protocol.action_suffix.clone(), + obs_prefix: self.protocol.obs_prefix.clone(), + rew_prefix: self.protocol.rew_prefix.clone(), + done_prefix: self.protocol.done_prefix.clone(), + data_prefix: self.protocol.data_prefix.clone(), + wire_encoding: match self.protocol.wire_encoding { + PayloadEncoding::Utf8 => VmPayloadEncodingSpec::Utf8, + PayloadEncoding::Hex => VmPayloadEncodingSpec::Hex, + }, + stats_backend: self.stats_backend.clone(), + trace: self.trace.as_ref().map(|trace| VmTraceSpec { + shared_region_name: trace.shared_region_name.clone(), + max_bytes: trace.max_bytes, + reset_on_episode: trace.reset_on_episode, + }), + debug_mode: self.debug_mode, + crash_log: self.crash_log.clone(), + }); + environment + .validate_in(&assets, &SpecEnvironment::default()) + .map(|_| ()) + .map_err(|err| InfotheoryError::invalid_backend_config(err.to_string())) + } + + /// Build a runtime VM configuration from a canonical planner environment spec. + pub fn from_environment_spec( + spec: &VmEnvironmentSpec, + resolved_assets: &[ResolvedAssetBinding], + ) -> InfotheoryResult { + let wire_encoding = match spec.wire_encoding { + VmPayloadEncodingSpec::Utf8 => PayloadEncoding::Utf8, + VmPayloadEncodingSpec::Hex => PayloadEncoding::Hex, + }; + let reward_policy = match &spec.reward_policy { + VmRewardPolicySpec::FromGuest => NyxRewardPolicy::FromGuest, + VmRewardPolicySpec::Pattern { + pattern, + base_reward, + bonus_reward, + } => NyxRewardPolicy::Pattern { + pattern: pattern.clone(), + base_reward: *base_reward, + bonus_reward: *bonus_reward, + }, + }; + let reward_shaping = match &spec.reward_shaping { + Some(VmRewardShapingSpec::EntropyReduction { + baseline_asset, + scale, + crash_bonus, + timeout_bonus, + }) => Some(NyxRewardShaping::EntropyReduction { + baseline_bytes: read_resolved_asset_bytes(resolved_assets, baseline_asset)?, + scale: *scale, + crash_bonus: *crash_bonus, + timeout_bonus: *timeout_bonus, + }), + Some(VmRewardShapingSpec::TraceEntropy { scale, normalize }) => { + Some(NyxRewardShaping::TraceEntropy { + scale: *scale, + normalize: *normalize, + }) + } + None => None, + }; + let action_source = match &spec.action_source { + VmRuntimeActionSourceSpec::Literal { + names, + payloads, + encoding, + } => { + let encoding = match encoding { + VmPayloadEncodingSpec::Utf8 => PayloadEncoding::Utf8, + VmPayloadEncodingSpec::Hex => PayloadEncoding::Hex, + }; + let mut actions = Vec::with_capacity(payloads.len()); + for (index, payload) in payloads.iter().enumerate() { + actions.push(NyxActionSpec { + name: names.get(index).cloned().flatten(), + payload: encoding.decode(payload).map_err(|err| { + InfotheoryError::invalid_backend_config(format!( + "invalid literal action payload: {err}" + )) + })?, + }); + } + NyxActionSource::Literal(actions) + } + VmRuntimeActionSourceSpec::Fuzz { + seeds, + encoding, + mutators, + min_len, + max_len, + dictionary, + rng_seed, + } => { + let encoding = match encoding { + VmPayloadEncodingSpec::Utf8 => PayloadEncoding::Utf8, + VmPayloadEncodingSpec::Hex => PayloadEncoding::Hex, + }; + NyxActionSource::Fuzz(NyxFuzzConfig { + seeds: seeds + .iter() + .map(|seed| { + encoding.decode(seed).map_err(|err| { + InfotheoryError::invalid_backend_config(format!( + "invalid VM fuzz seed: {err}" + )) + }) + }) + .collect::, _>>()?, + mutators: mutators + .iter() + .map(|mutator| match mutator { + VmFuzzMutatorSpec::FlipBit => Ok(FuzzMutator::FlipBit), + VmFuzzMutatorSpec::FlipByte => Ok(FuzzMutator::FlipByte), + VmFuzzMutatorSpec::InsertByte => Ok(FuzzMutator::InsertByte), + VmFuzzMutatorSpec::DeleteByte => Ok(FuzzMutator::DeleteByte), + VmFuzzMutatorSpec::SpliceSeed => Ok(FuzzMutator::SpliceSeed), + VmFuzzMutatorSpec::ResetSeed => Ok(FuzzMutator::ResetSeed), + VmFuzzMutatorSpec::Havoc => Ok(FuzzMutator::Havoc), + }) + .collect::, InfotheoryError>>()?, + min_len: *min_len, + max_len: *max_len, + dictionary: dictionary + .iter() + .map(|entry| { + encoding.decode(entry).map_err(|err| { + InfotheoryError::invalid_backend_config(format!( + "invalid VM fuzz dictionary entry: {err}" + )) + }) + }) + .collect::, _>>()?, + rng_seed: *rng_seed, + }) + } + }; + let action_filter = spec + .action_filter + .as_ref() + .map(|filter| -> InfotheoryResult { + Ok(NyxActionFilter { + min_entropy: filter.min_entropy, + max_entropy: filter.max_entropy, + min_intrinsic_dependence: filter.min_intrinsic_dependence, + min_novelty: filter.min_novelty, + novelty_prior: filter + .novelty_prior_asset + .as_ref() + .map(|id| read_resolved_asset_bytes(resolved_assets, id)) + .transpose()?, + reject_reward: filter.reject_reward, + }) + }) + .transpose()?; + + let config = Self { + firecracker_config: resolved_asset_path( + resolved_assets, + &spec.firecracker_config_asset, + )? + .to_string_lossy() + .into_owned(), + instance_id: spec.instance_id.clone(), + shared_region_name: spec.shared_region_name.clone(), + shared_region_size: spec.shared_region_size, + shared_memory_policy: match spec.shared_memory_policy { + SharedMemoryPolicySpec::Preserve => SharedMemoryPolicy::Preserve, + SharedMemoryPolicySpec::Snapshot => SharedMemoryPolicy::Snapshot, + }, + step_timeout: Duration::from_millis(spec.step_timeout_ms), + boot_timeout: Duration::from_millis(spec.boot_timeout_ms), + episode_steps: spec.episode_steps, + step_cost: spec.step_cost, + observation_policy: match spec.observation_policy { + VmObservationPolicySpec::FromGuest => NyxObservationPolicy::FromGuest, + VmObservationPolicySpec::OutputHash => NyxObservationPolicy::OutputHash, + VmObservationPolicySpec::RawOutput => NyxObservationPolicy::RawOutput, + VmObservationPolicySpec::SharedMemory => NyxObservationPolicy::SharedMemory, + }, + observation_bits: spec.observation_bits, + observation_stream_len: spec.observation_stream_len, + observation_stream_mode: match spec.observation_stream_mode { + VmObservationStreamModeSpec::PadTruncate => NyxObservationStreamMode::PadTruncate, + VmObservationStreamModeSpec::Pad => NyxObservationStreamMode::Pad, + VmObservationStreamModeSpec::Truncate => NyxObservationStreamMode::Truncate, + }, + observation_pad_byte: spec.observation_pad_byte, + reward_bits: spec.reward_bits, + reward_policy, + reward_shaping, + action_source, + action_filter, + protocol: NyxProtocolConfig { + action_prefix: spec.action_prefix.clone(), + action_suffix: spec.action_suffix.clone(), + obs_prefix: spec.obs_prefix.clone(), + rew_prefix: spec.rew_prefix.clone(), + done_prefix: spec.done_prefix.clone(), + data_prefix: spec.data_prefix.clone(), + wire_encoding, + }, + stats_backend: spec.stats_backend.clone(), + trace: spec.trace.as_ref().map(|trace| NyxTraceConfig { + shared_region_name: trace.shared_region_name.clone(), + max_bytes: trace.max_bytes, + reset_on_episode: trace.reset_on_episode, + }), + debug_mode: spec.debug_mode, + crash_log: spec.crash_log.clone(), + }; + config.validate()?; + Ok(config) + } +} + +fn resolved_asset_path<'a>( + resolved_assets: &'a [ResolvedAssetBinding], + id: &str, +) -> InfotheoryResult<&'a Path> { + let binding = resolved_assets + .iter() + .find(|binding| binding.id == id) + .ok_or_else(|| { + InfotheoryError::invalid_backend_config(format!( + "planner_run references unknown asset id '{id}'" + )) + })?; + match &binding.asset { + AssetRef::Filesystem(path) => Ok(path.as_path()), + } +} + +fn read_resolved_asset_bytes( + resolved_assets: &[ResolvedAssetBinding], + id: &str, +) -> InfotheoryResult> { + let path = resolved_asset_path(resolved_assets, id)?; + std::fs::read(path).map_err(|err| { + InfotheoryError::invalid_backend_config(format!( + "failed to read asset '{}': {err}", + path.display() + )) + }) +} + +// ============================================================================ +// Step Result +// ============================================================================ + +/// Result of a single environment step. +#[derive(Clone, Debug)] +pub struct NyxStepResult { + /// Exit reason from the VM. + pub exit_reason: NyxExitKind, + /// Raw output data from guest. + pub output: Vec, + /// Parsed observation (if any). + pub parsed_obs: Option, + /// Parsed reward (if any). + pub parsed_rew: Option, + /// Done flag. + pub done: bool, + /// Trace data (if collected). + pub trace_data: Vec, + /// Shared memory contents snapshot. + pub shared_memory: Vec, +} + +/// Simplified exit reason categories. +#[derive(Clone, Debug)] +pub enum NyxExitKind { + /// Guest terminated normally with an application-defined code. + ExecDone(u64), + /// Step timed out before a terminal signal/response. + Timeout, + /// VM reported a shutdown event. + Shutdown, + /// Raw hypercall event with integer arguments. + Hypercall { + /// Hypercall identifier/magic value. + code: u64, + /// Hypercall argument 1. + arg1: u64, + /// Hypercall argument 2. + arg2: u64, + /// Hypercall argument 3. + arg3: u64, + /// Hypercall argument 4. + arg4: u64, + }, + /// Debug string emitted by guest/host bridge. + DebugPrint(String), + /// Breakpoint/trap-like stop event. + Breakpoint, + /// Uncategorized exit event represented as text. + Other(String), +} + +impl From for NyxExitKind { + fn from(reason: ExitReason) -> Self { + match reason { + ExitReason::ExecDone(code) => Self::ExecDone(code), + ExitReason::Timeout => Self::Timeout, + ExitReason::Shutdown => Self::Shutdown, + ExitReason::Hypercall(r8, r9, r10, r11, r12) => Self::Hypercall { + code: r8, + arg1: r9, + arg2: r10, + arg3: r11, + arg4: r12, + }, + ExitReason::DebugPrint(s) => Self::DebugPrint(s), + ExitReason::Breakpoint => Self::Breakpoint, + ExitReason::RequestSnapshot => Self::Other("RequestSnapshot".to_string()), + ExitReason::SharedMem(name, _, _) => Self::Other(format!("SharedMem({})", name)), + ExitReason::SingleStep => Self::Other("SingleStep".to_string()), + ExitReason::Interrupted => Self::Other("Interrupted".to_string()), + ExitReason::HWBreakpoint(n) => Self::Other(format!("HWBreakpoint({})", n)), + ExitReason::BadMemoryAccess(_) => Self::Other("BadMemoryAccess".to_string()), + } + } +} + +// ============================================================================ +// Trace Model +// ============================================================================ + +/// Predictive model for trace-based reward computation. +enum TraceModel { + #[cfg(feature = "backend-rosa")] + Rosa { model: RosaPlus, max_order: i64 }, + // `max_order` is preserved here so that `reset()` can rebuild a fresh + // `RosaPlus` with the same `max_order` configured by the active backend + // variant; it is read once at construction from `RateBackendPlan::RosaPlus`. + #[cfg(feature = "backend-ctw")] + Ctw { tree: ContextTree }, + #[cfg(feature = "backend-ctw")] + FacCtw { + tree: FacContextTree, + bits_per_symbol: usize, + msb_first: bool, + }, + #[cfg(feature = "backend-mamba")] + Mamba { + compressor: MambaCompressor, + primed: bool, + }, + #[cfg(feature = "backend-rwkv")] + Rwkv7 { + compressor: Compressor, + primed: bool, + }, + #[cfg(feature = "backend-zpaq")] + Zpaq { model: ZpaqRateModel }, + Mixture { + backend: CompiledRateBackend, + model: crate::mixture::RateBackendPredictor, + }, +} + +impl TraceModel { + fn predictor_backed(backend: CompiledRateBackend) -> anyhow::Result { + let mut model = crate::runtime::build_rate_backend_predictor(&backend, 2f64.powi(-24)) + .map_err(|e| anyhow::anyhow!("predictor-backed init failed: {e}"))?; + model + .begin_stream(None) + .map_err(|e| anyhow::anyhow!("predictor-backed stream init failed: {e}"))?; + Ok(TraceModel::Mixture { backend, model }) + } + + fn new(backend: &CompiledRateBackend) -> anyhow::Result { + #[allow(unreachable_patterns)] + match crate::runtime::rate_backend_trace_model_strategy(backend) { + #[cfg(feature = "backend-rosa")] + crate::runtime::TraceModelStrategy::Rosa => { + let crate::spec::core::RateBackendPlan::RosaPlus { max_order } = backend.plan() + else { + unreachable!("rosa trace strategy used with non-rosa backend") + }; + let mut model = RosaPlus::new(*max_order, false, 0, 42); + model.build_lm_full_bytes_no_finalize_endpos(); + Ok(TraceModel::Rosa { + model, + max_order: *max_order, + }) + } + crate::runtime::TraceModelStrategy::PredictorBacked => { + TraceModel::predictor_backed(backend.clone()) + } + #[cfg(feature = "backend-ctw")] + crate::runtime::TraceModelStrategy::Ctw => { + let crate::spec::core::RateBackendPlan::Ctw { depth } = backend.plan() else { + unreachable!("trace-model strategy mismatch for ctw"); + }; + Ok(TraceModel::Ctw { + tree: ContextTree::new(*depth), + }) + } + #[cfg(feature = "backend-ctw")] + crate::runtime::TraceModelStrategy::FacCtw => { + let crate::spec::core::RateBackendPlan::FacCtw { + base_depth, + num_percept_bits: _, + encoding_bits, + msb_first, + } = backend.plan() + else { + unreachable!("trace-model strategy mismatch for fac-ctw"); + }; + let bits_per_symbol = *encoding_bits; + Ok(TraceModel::FacCtw { + tree: FacContextTree::new(*base_depth, bits_per_symbol), + bits_per_symbol, + msb_first: *msb_first, + }) + } + #[cfg(feature = "backend-zpaq")] + crate::runtime::TraceModelStrategy::Zpaq => { + let crate::spec::core::RateBackendPlan::Zpaq { method } = backend.plan() else { + unreachable!("trace-model strategy mismatch for zpaq"); + }; + Ok(TraceModel::Zpaq { + model: ZpaqRateModel::new(method.clone(), 2f64.powi(-24)), + }) + } + #[cfg(feature = "backend-mamba")] + crate::runtime::TraceModelStrategy::Mamba => { + let crate::spec::core::RateBackendPlan::Mamba { parsed_method, .. } = + backend.plan() + else { + unreachable!("trace-model strategy mismatch for mamba"); + }; + let compressor = MambaCompressor::new_from_method_spec(parsed_method) + .map_err(|e| anyhow::anyhow!("invalid mamba method for vm trace model: {e}"))?; + Ok(TraceModel::Mamba { + compressor, + primed: false, + }) + } + #[cfg(feature = "backend-rwkv")] + crate::runtime::TraceModelStrategy::Rwkv7 => { + let crate::spec::core::RateBackendPlan::Rwkv7 { parsed_method, .. } = + backend.plan() + else { + unreachable!("trace-model strategy mismatch for rwkv7"); + }; + let compressor = Compressor::new_from_method_spec(parsed_method) + .map_err(|e| anyhow::anyhow!("invalid rwkv7 method for vm trace model: {e}"))?; + Ok(TraceModel::Rwkv7 { + compressor, + primed: false, + }) + } + _ => unreachable!("trace-model strategy requires an unavailable backend feature"), + } + } + + fn reset(&mut self) -> anyhow::Result<()> { + match self { + #[cfg(feature = "backend-rosa")] + TraceModel::Rosa { model, max_order } => { + let mut fresh = RosaPlus::new(*max_order, false, 0, 42); + fresh.build_lm_full_bytes_no_finalize_endpos(); + *model = fresh; + } + #[cfg(feature = "backend-ctw")] + TraceModel::Ctw { tree } => tree.clear(), + #[cfg(feature = "backend-ctw")] + TraceModel::FacCtw { tree, .. } => tree.clear(), + #[cfg(feature = "backend-mamba")] + TraceModel::Mamba { compressor, primed } => { + compressor.state.reset(); + *primed = false; + } + #[cfg(feature = "backend-rwkv")] + TraceModel::Rwkv7 { compressor, primed } => { + compressor.state.reset(); + *primed = false; + } + #[cfg(feature = "backend-zpaq")] + TraceModel::Zpaq { model } => { + model.reset(); + } + TraceModel::Mixture { backend, model } => { + *model = crate::runtime::build_rate_backend_predictor(backend, 2f64.powi(-24)) + .map_err(|e| anyhow::anyhow!("mixture model reset failed: {e}"))?; + model + .begin_stream(None) + .map_err(|e| anyhow::anyhow!("mixture stream init failed: {e}"))?; + } + } + Ok(()) + } + + /// Update the model with new data and return the surprise (bits). + fn update_and_score(&mut self, data: &[u8]) -> f64 { + if data.is_empty() { + return 0.0; + } + match self { + #[cfg(feature = "backend-rosa")] + TraceModel::Rosa { model, .. } => { + let mut bits = 0.0; + for &b in data { + let p = model.prob_for_last(b as u32).max(1e-12); + bits -= p.log2(); + model.train_byte(b); + } + bits + } + #[cfg(feature = "backend-ctw")] + TraceModel::Ctw { tree } => { + let log_before = tree.get_log_block_probability(); + for &b in data { + for i in (0..8).rev() { + tree.update(((b >> i) & 1) == 1); + } + } + let log_after = tree.get_log_block_probability(); + let log_delta = log_after - log_before; + -log_delta / std::f64::consts::LN_2 + } + #[cfg(feature = "backend-ctw")] + TraceModel::FacCtw { + tree, + bits_per_symbol, + msb_first, + } => { + let log_before = tree.get_log_block_probability(); + for &b in data { + for i in 0..*bits_per_symbol { + let bit = if *msb_first { + ctw_symbol_bit_msb(b, *bits_per_symbol, i) + } else { + ((b >> i) & 1) == 1 + }; + tree.update(bit, i); + } + } + let log_after = tree.get_log_block_probability(); + let log_delta = log_after - log_before; + -log_delta / std::f64::consts::LN_2 + } + #[cfg(feature = "backend-mamba")] + TraceModel::Mamba { compressor, primed } => { + if !*primed { + let bias = compressor.online_bias_snapshot(); + let logits = + compressor + .model + .forward(&mut compressor.scratch, 0, &mut compressor.state); + mambazip::Compressor::logits_to_pdf( + logits, + bias.as_deref(), + &mut compressor.pdf_buffer, + ); + *primed = true; + } + let mut bits = 0.0; + for &b in data { + let p = compressor.pdf_buffer[b as usize].max(1e-12); + bits -= p.log2(); + let bias = compressor.online_bias_snapshot(); + let logits = compressor.model.forward( + &mut compressor.scratch, + b as u32, + &mut compressor.state, + ); + mambazip::Compressor::logits_to_pdf( + logits, + bias.as_deref(), + &mut compressor.pdf_buffer, + ); + } + bits + } + #[cfg(feature = "backend-rwkv")] + TraceModel::Rwkv7 { compressor, primed } => { + if !*primed { + let vocab_size = compressor.vocab_size(); + let logits = + compressor + .model + .forward(&mut compressor.scratch, 0, &mut compressor.state); + softmax_pdf_inplace(logits, vocab_size, &mut compressor.pdf_buffer); + *primed = true; + } + let mut bits = 0.0; + let vocab_size = compressor.vocab_size(); + for &b in data { + let p = compressor.pdf_buffer[b as usize].max(1e-12); + bits -= p.log2(); + let logits = compressor.model.forward( + &mut compressor.scratch, + b as u32, + &mut compressor.state, + ); + softmax_pdf_inplace(logits, vocab_size, &mut compressor.pdf_buffer); + } + bits + } + #[cfg(feature = "backend-zpaq")] + TraceModel::Zpaq { model } => model.update_and_score(data), + TraceModel::Mixture { model, .. } => { + let mut bits = 0.0; + for &b in data { + let logp = model.log_prob(b); + bits -= logp / std::f64::consts::LN_2; + model.update(b); + } + bits + } + } + } +} + +// ============================================================================ +// Fuzz State +// ============================================================================ + +struct FuzzState { + current: Vec, + rng: RandomGenerator, +} + +// ============================================================================ +// NyxVmEnvironment +// ============================================================================ + +/// High-performance VM environment using nyx-lite. +pub struct NyxVmEnvironment { + /// Configuration. + config: NyxVmConfig, + /// Compiled entropy/scoring backend used by VM reward logic. + compiled_stats_backend: CompiledRateBackend, + /// The nyx-lite VM instance. + vm: NyxVM, + /// Base snapshot for episode resets. + base_snapshot: Option>, + /// Shared memory virtual address in guest. + shared_vaddr: Option, + /// CR3 used when shared memory was registered. + shared_cr3: Option, + /// Trace model for entropy-based rewards. + trace_model: Option, + /// Baseline entropy for entropy reduction rewards. + baseline_entropy: Option, + /// Effective reward shaping policy (additive). + reward_shaping: Option, + /// Fuzzing state. + fuzz_state: Option, + + // Current step state + /// Current observation. + obs: PerceptVal, + /// Current reward. + rew: Reward, + /// Current observation stream. + obs_stream: Vec, + /// Step within current episode. + step_in_episode: usize, + /// Whether the environment needs reset. + needs_reset: bool, + /// Whether the VM has been initialized. + initialized: bool, +} + +impl NyxVmEnvironment { + /// Creates a new NyxVmEnvironment with the given configuration. + pub fn new(config: NyxVmConfig) -> anyhow::Result { + config.validate().map_err(anyhow::Error::msg)?; + + // Load Firecracker config and resolve relative paths + let fc_config_raw = std::fs::read_to_string(&config.firecracker_config) + .map_err(|e| anyhow::anyhow!("Failed to read firecracker config: {}", e))?; + let fc_config = + rewrite_firecracker_config_paths(&config.firecracker_config, &fc_config_raw) + .map_err(|e| anyhow::anyhow!("Failed to parse firecracker config: {}", e))?; + + // Create the VM + let vm = NyxVM::new(config.instance_id.clone(), &fc_config); + + // Initialize reward shaping + let reward_shaping = config.reward_shaping.clone(); + + let compiled_stats_backend = config + .stats_backend + .compile() + .map_err(|err| anyhow::anyhow!("invalid vm stats_backend: {err}"))?; + + // Initialize trace model if needed + let trace_model = match &reward_shaping { + Some(NyxRewardShaping::TraceEntropy { .. }) => Some( + TraceModel::new(&compiled_stats_backend) + .map_err(|err| anyhow::anyhow!("failed to initialize trace model: {err}"))?, + ), + _ => None, + }; + + // Compute baseline entropy if needed + let baseline_entropy = match &reward_shaping { + Some(NyxRewardShaping::EntropyReduction { baseline_bytes, .. }) => { + let h = try_entropy_rate_backend(baseline_bytes, &compiled_stats_backend).map_err( + |err| { + anyhow::anyhow!( + "validated vm stats_backend failed to score baseline entropy: {err}" + ) + }, + )?; + Some(h) + } + _ => None, + }; + + // Initialize fuzz state if needed + let fuzz_state = match &config.action_source { + NyxActionSource::Fuzz(fuzz) => { + if fuzz.seeds.is_empty() { + return Err(anyhow::anyhow!("Fuzz mode requires at least one seed")); + } + if fuzz.mutators.is_empty() { + return Err(anyhow::anyhow!("Fuzz mode requires at least one mutator")); + } + let seed = fuzz.seeds[0].clone(); + Some(FuzzState { + current: seed, + rng: RandomGenerator::from_seed(fuzz.rng_seed), + }) + } + NyxActionSource::Literal(actions) => { + if actions.is_empty() { + return Err(anyhow::anyhow!("Literal mode requires at least one action")); + } + None + } + }; + + let mut env = Self { + config, + compiled_stats_backend, + vm, + base_snapshot: None, + shared_vaddr: None, + shared_cr3: None, + trace_model, + baseline_entropy, + reward_shaping, + fuzz_state, + obs: 0, + rew: 0, + obs_stream: Vec::new(), + step_in_episode: 0, + needs_reset: true, + initialized: false, + }; + + // Boot and initialize + env.initialize()?; + + Ok(env) + } + + /// Initializes the VM by booting to the snapshot point. + fn initialize(&mut self) -> anyhow::Result<()> { + if self.initialized { + return Ok(()); + } + + if self.config.debug_mode { + eprintln!("[NyxVm] Booting VM..."); + } + + // Run until we get the shared memory registration + let start = Instant::now(); + loop { + if start.elapsed() > self.config.boot_timeout { + return Err(anyhow::anyhow!("Boot timeout waiting for shared memory")); + } + + let exit = self.vm.run(Duration::from_secs(1)); + match exit { + ExitReason::SharedMem(name, vaddr, size) => { + if self.config.debug_mode { + eprintln!( + "[NyxVm] Shared memory registered: {} @ {:#x} ({} bytes)", + name, vaddr, size + ); + } + if name.trim_end_matches('\0') == self.config.shared_region_name { + self.shared_vaddr = Some(vaddr); + self.shared_cr3 = Some(self.vm.sregs().cr3); + // Register the shared region with the configured policy + let _ = self.vm.register_shared_region_current( + vaddr, + size, + self.config.shared_memory_policy, + ); + break; + } + } + ExitReason::DebugPrint(msg) => { + if self.config.debug_mode { + eprintln!("[NyxVm] Guest: {}", msg); + } + } + ExitReason::Shutdown => { + return Err(anyhow::anyhow!("VM shut down during boot")); + } + _ => { + if self.config.debug_mode { + eprintln!("[NyxVm] Boot exit: {:?}", exit); + } + // Continue waiting + } + } + } + + // Continue running until snapshot request + loop { + if start.elapsed() > self.config.boot_timeout { + return Err(anyhow::anyhow!("Boot timeout waiting for snapshot request")); + } + + let exit = self.vm.run(Duration::from_secs(1)); + match exit { + ExitReason::RequestSnapshot => { + if self.config.debug_mode { + eprintln!("[NyxVm] Taking base snapshot..."); + } + self.base_snapshot = Some(self.vm.take_base_snapshot()); + break; + } + ExitReason::DebugPrint(msg) => { + if self.config.debug_mode { + eprintln!("[NyxVm] Guest: {}", msg); + } + } + ExitReason::Shutdown => { + return Err(anyhow::anyhow!("VM shut down before snapshot")); + } + _ => { + if self.config.debug_mode { + eprintln!("[NyxVm] Snapshot wait exit: {:?}", exit); + } + // Continue waiting + } + } + } + + if self.config.debug_mode { + eprintln!("[NyxVm] Initialization complete"); + } + + self.initialized = true; + self.needs_reset = false; + Ok(()) + } + + /// Resets to the base snapshot. + pub fn reset(&mut self) -> anyhow::Result<()> { + let snapshot = self + .base_snapshot + .as_ref() + .ok_or_else(|| anyhow::anyhow!("No base snapshot available"))? + .clone(); + + self.vm.apply_snapshot(&snapshot); + + // Reset trace model if configured + if let Some(trace_cfg) = &self.config.trace + && trace_cfg.reset_on_episode + && let Some(model) = &mut self.trace_model + { + model + .reset() + .map_err(|err| anyhow::anyhow!("failed to reset trace model: {err}"))?; + } + + self.step_in_episode = 0; + self.needs_reset = false; + + Ok(()) + } + + /// Writes action data to shared memory. + fn write_action_to_shared_memory(&mut self, payload: &[u8]) -> anyhow::Result<()> { + let vaddr = self + .shared_vaddr + .ok_or_else(|| anyhow::anyhow!("Shared memory not initialized"))?; + let cr3 = self + .shared_cr3 + .ok_or_else(|| anyhow::anyhow!("Shared memory CR3 not initialized"))?; + let process = self.vm.process_memory(cr3); + + // Ensure guest has cleared the previous message length to avoid races. + let wait_start = Instant::now(); + loop { + let cur_len = process + .read_u64(vaddr + SHARED_ACTION_LEN_OFFSET) + .unwrap_or(0); + if cur_len == 0 { + break; + } + if wait_start.elapsed() > self.config.step_timeout { + return Err(anyhow::anyhow!("shared buffer busy (len={cur_len})")); + } + std::thread::yield_now(); + } + + // Write length as first 8 bytes (u64 LE) + let len = payload.len() as u64; + process + .write_u64(vaddr + SHARED_ACTION_LEN_OFFSET, len) + .map_err(|e| anyhow::anyhow!("write len failed: {e}"))?; + let _ = process.write_u64(vaddr + SHARED_RESP_LEN_OFFSET, 0); + + // Write payload starting at offset 8 + let max_len = self + .config + .shared_region_size + .saturating_sub(SHARED_PAYLOAD_OFFSET as usize); + let write_len = payload.len().min(max_len); + if write_len > 0 { + let _ = process + .write_bytes(vaddr + SHARED_PAYLOAD_OFFSET, &payload[..write_len]) + .map_err(|e| anyhow::anyhow!("write payload failed: {e}"))?; + } + + if self.config.debug_mode { + let verify = process + .read_u64(vaddr + SHARED_ACTION_LEN_OFFSET) + .unwrap_or(0) as usize; + eprintln!( + "[NyxVm] Wrote action len={}, verified len={}", + write_len, verify + ); + } + + Ok(()) + } + + /// Reads response from shared memory. + fn read_shared_memory(&self) -> Vec { + let Some(vaddr) = self.shared_vaddr else { + return Vec::new(); + }; + let Some(cr3) = self.shared_cr3 else { + return Vec::new(); + }; + let process = self.vm.process_memory(cr3); + + // Read length from first 8 bytes + let len = process + .read_u64(vaddr + SHARED_RESP_LEN_OFFSET) + .unwrap_or(0) as usize; + let max_len = self + .config + .shared_region_size + .saturating_sub(SHARED_PAYLOAD_OFFSET as usize); + let read_len = len.min(max_len); + + if read_len == 0 { + return Vec::new(); + } + + let mut buf = vec![0u8; read_len]; + let _ = process.read_bytes(vaddr + SHARED_PAYLOAD_OFFSET, &mut buf); + buf + } + + fn clear_shared_length(&self) { + let (Some(vaddr), Some(cr3)) = (self.shared_vaddr, self.shared_cr3) else { + return; + }; + let process = self.vm.process_memory(cr3); + let _ = process.write_u64(vaddr + SHARED_ACTION_LEN_OFFSET, 0); + let _ = process.write_u64(vaddr + SHARED_RESP_LEN_OFFSET, 0); + } + + /// Runs a single step, returning detailed results. + pub fn run_step(&mut self, payload: &[u8]) -> anyhow::Result { + // Write action to shared memory + self.write_action_to_shared_memory(payload)?; + + // Run the VM until we get a meaningful exit + let start = Instant::now(); + let mut output = Vec::new(); + let mut trace_data = Vec::new(); + let mut parsed_obs = None; + let mut parsed_rew = None; + let mut done = false; + let exit_kind; + let collect_output = + matches!( + self.config.observation_policy, + NyxObservationPolicy::OutputHash | NyxObservationPolicy::RawOutput + ) || matches!(self.config.reward_policy, NyxRewardPolicy::Pattern { .. }) + || matches!( + self.reward_shaping, + Some(NyxRewardShaping::EntropyReduction { .. }) + ); + + loop { + let remaining = self + .config + .step_timeout + .checked_sub(start.elapsed()) + .unwrap_or(Duration::ZERO); + + if remaining.is_zero() { + exit_kind = NyxExitKind::Timeout; + break; + } + + let exit = self.vm.run(remaining); + match exit { + ExitReason::ExecDone(code) => { + exit_kind = NyxExitKind::ExecDone(code); + done = true; + break; + } + ExitReason::Timeout => { + if self.config.debug_mode { + eprintln!("[NyxVm] Step timeout"); + } + exit_kind = NyxExitKind::Timeout; + break; + } + ExitReason::Shutdown => { + if self.config.debug_mode { + eprintln!("[NyxVm] VM shutdown during step"); + } + exit_kind = NyxExitKind::Shutdown; + done = true; + break; + } + ExitReason::DebugPrint(msg) => { + if self.config.debug_mode { + eprintln!("[NyxVm] Guest: {}", msg); + } + // Accumulate debug output + if collect_output { + output.extend_from_slice(msg.as_bytes()); + } + // Continue running + } + ExitReason::Hypercall(r8, r9, r10, r11, r12) => { + exit_kind = NyxExitKind::Hypercall { + code: r8, + arg1: r9, + arg2: r10, + arg3: r11, + arg4: r12, + }; + // Attempt to parse structured response + if let Some(obs) = Self::try_parse_u64(r9) { + parsed_obs = Some(obs); + } + if let Some(rew) = Self::try_parse_i64(r10) { + parsed_rew = Some(rew); + } + break; + } + ExitReason::Breakpoint => { + if self.config.debug_mode { + eprintln!("[NyxVm] Breakpoint exit during step"); + } + exit_kind = NyxExitKind::Breakpoint; + break; + } + _ => { + // Continue for other exits + } + } + } + + // Read shared memory contents (only if needed) + let need_shared_memory = matches!( + self.config.observation_policy, + NyxObservationPolicy::SharedMemory + ) || matches!( + self.config.reward_policy, + NyxRewardPolicy::Pattern { .. } + ) || matches!( + self.reward_shaping, + Some(NyxRewardShaping::EntropyReduction { .. }) + ) || self.config.trace.is_some(); + let shared_memory = if need_shared_memory { + self.read_shared_memory() + } else { + Vec::new() + }; + + // Clear shared length to avoid host/guest races on the next step. + self.clear_shared_length(); + + // Collect trace data if configured + if let Some(trace_cfg) = &self.config.trace + && trace_cfg.shared_region_name.is_some() + { + // Read from trace shared memory region (implementation-specific) + // For now, use main shared memory as fallback + trace_data = shared_memory.clone(); + if trace_data.len() > trace_cfg.max_bytes { + trace_data.truncate(trace_cfg.max_bytes); + } + } + + Ok(NyxStepResult { + exit_reason: exit_kind, + output, + parsed_obs, + parsed_rew, + done, + trace_data, + shared_memory, + }) + } + + fn try_parse_u64(val: u64) -> Option { + // Hypercall args are already u64 + Some(val) + } + + fn try_parse_i64(val: u64) -> Option { + Some(val as i64) + } + + /// Gets the action payload for the given action index. + fn get_action_payload(&mut self, action: Action) -> anyhow::Result> { + match &self.config.action_source { + NyxActionSource::Literal(actions) => { + let idx = action as usize; + if idx >= actions.len() { + return Err(anyhow::anyhow!("Action index out of range")); + } + Ok(Cow::Borrowed(actions[idx].payload.as_slice())) + } + NyxActionSource::Fuzz(fuzz) => { + let state = self + .fuzz_state + .as_mut() + .ok_or_else(|| anyhow::anyhow!("Fuzz state missing"))?; + let idx = action as usize % fuzz.mutators.len(); + let mut input = state.current.clone(); + let mutator = &fuzz.mutators[idx]; + apply_mutator(mutator, &mut input, fuzz, &mut state.rng); + if input.len() < fuzz.min_len { + input.resize(fuzz.min_len, 0); + } + if input.len() > fuzz.max_len { + input.truncate(fuzz.max_len); + } + state.current = input.clone(); + Ok(Cow::Owned(input)) + } + } + } + + /// Applies action filtering, returning reject reward if filtered. + fn filter_action(&self, payload: &[u8]) -> anyhow::Result> { + let Some(filter) = self.config.action_filter.as_ref() else { + return Ok(None); + }; + if payload.is_empty() { + return Ok(filter.reject_reward); + } + + let (entropy, intrinsic, novelty) = self.compute_filter_metrics(payload, filter)?; + + if let Some(min_entropy) = filter.min_entropy + && entropy < min_entropy + { + return Ok(filter.reject_reward); + } + if let Some(max_entropy) = filter.max_entropy + && entropy > max_entropy + { + return Ok(filter.reject_reward); + } + if let Some(min_intrinsic) = filter.min_intrinsic_dependence + && intrinsic < min_intrinsic + { + return Ok(filter.reject_reward); + } + if let Some(min_novelty) = filter.min_novelty + && filter.novelty_prior.is_some() + && novelty < min_novelty + { + return Ok(filter.reject_reward); + } + Ok(None) + } + + fn wrap_action_payload(&self, payload: &[u8]) -> Vec { + let p = &self.config.protocol; + let mut wrapped = p.action_prefix.clone().into_bytes(); + wrapped.extend_from_slice(p.wire_encoding.encode(payload).as_bytes()); + wrapped.extend_from_slice(p.action_suffix.as_bytes()); + wrapped + } + + fn compute_filter_metrics( + &self, + payload: &[u8], + filter: &NyxActionFilter, + ) -> anyhow::Result<(f64, f64, f64)> { + let h_marg = empirical_entropy_bytes(payload); + let h_rate = + try_entropy_rate_backend(payload, &self.compiled_stats_backend).map_err(|err| { + anyhow::anyhow!("vm stats backend failed to score payload entropy: {err}") + })?; + + let intrinsic = if h_marg < 1e-9 { + 0.0 + } else { + ((h_marg - h_rate) / h_marg).clamp(0.0, 1.0) + }; + + let novelty = if let Some(ref prior) = filter.novelty_prior { + try_cross_entropy_rate_backend(payload, prior, &self.compiled_stats_backend) + .map_err(|err| anyhow::anyhow!("vm stats backend failed to score novelty: {err}"))? + } else { + 0.0 + }; + + Ok((h_rate, intrinsic, novelty)) + } + + /// Computes reward from step result. + fn compute_reward(&mut self, result: &NyxStepResult) -> anyhow::Result { + let base_reward = match &self.config.reward_policy { + NyxRewardPolicy::FromGuest => result.parsed_rew.unwrap_or(0), + NyxRewardPolicy::Pattern { + pattern, + base_reward, + bonus_reward, + } => { + let text = String::from_utf8_lossy(&result.output); + let shared_text = String::from_utf8_lossy(&result.shared_memory); + if text.contains(pattern) || shared_text.contains(pattern) { + base_reward + bonus_reward + } else { + *base_reward + } + } + NyxRewardPolicy::Custom(f) => f(result), + }; + + let shaping_reward = if let Some(shaping) = self.reward_shaping.clone() { + self.compute_reward_shaping(&shaping, result)? + } else { + 0 + }; + + let mut reward = base_reward.saturating_add(shaping_reward); + + reward = reward.saturating_sub(self.config.step_cost); + let min_reward = self.min_reward(); + let max_reward = self.max_reward(); + Ok(reward.clamp(min_reward, max_reward)) + } + + fn compute_reward_shaping( + &mut self, + shaping: &NyxRewardShaping, + result: &NyxStepResult, + ) -> anyhow::Result { + Ok(match shaping { + NyxRewardShaping::EntropyReduction { + scale, + crash_bonus, + timeout_bonus, + .. + } => { + let mut base_reward = { + let data = if result.shared_memory.is_empty() { + &result.output + } else { + &result.shared_memory + }; + let h_obs = try_entropy_rate_backend(data, &self.compiled_stats_backend) + .map_err(|err| { + anyhow::anyhow!( + "vm stats backend failed to score observation entropy: {err}" + ) + })?; + let h_base = self.baseline_entropy.unwrap_or(0.0); + let er = (h_base - h_obs) * scale; + er.round() as i64 + }; + + // Add bonuses for interesting behaviors (bugs/crashes) + match &result.exit_reason { + NyxExitKind::Shutdown | NyxExitKind::Breakpoint => { + if let Some(bonus) = crash_bonus { + base_reward = base_reward.saturating_add(*bonus); + } + } + NyxExitKind::Timeout => { + if let Some(bonus) = timeout_bonus { + base_reward = base_reward.saturating_add(*bonus); + } + } + _ => {} + } + + base_reward + } + NyxRewardShaping::TraceEntropy { + scale, normalize, .. + } => { + let data = &result.trace_data; + let bits = match self.trace_model.as_mut() { + Some(model) => model.update_and_score(data), + None => 0.0, + }; + let bits = if *normalize && !data.is_empty() { + bits / data.len() as f64 + } else { + bits + }; + (bits * scale).round() as i64 + } + }) + } + + fn mask_observation(&self, value: u64) -> u64 { + let bits = self.config.observation_bits; + if bits >= 64 { + value + } else if bits == 0 { + 0 + } else { + value & ((1u64 << bits) - 1) + } + } + + fn build_observation_stream(&self, result: &NyxStepResult) -> Vec { + let mut observations = match self.config.observation_policy { + NyxObservationPolicy::FromGuest => { + if let Some(obs) = result.parsed_obs { + vec![self.mask_observation(obs)] + } else { + vec![self.hash_observation(&result.shared_memory)] + } + } + NyxObservationPolicy::OutputHash => { + vec![self.hash_observation(&result.output)] + } + NyxObservationPolicy::RawOutput => { + result.output.iter().map(|b| *b as PerceptVal).collect() + } + NyxObservationPolicy::SharedMemory => result + .shared_memory + .iter() + .map(|b| *b as PerceptVal) + .collect(), + }; + + if observations.is_empty() { + observations.push(0); + } + + self.normalize_observation_stream(&mut observations); + observations + } + + fn hash_observation(&self, data: &[u8]) -> PerceptVal { + let h = robust_hash_bytes(data); + self.mask_observation(h) + } + + fn normalize_observation_stream(&self, observations: &mut Vec) { + let mask = if self.config.observation_bits >= 64 { + u64::MAX + } else if self.config.observation_bits == 0 { + 0 + } else { + (1u64 << self.config.observation_bits) - 1 + }; + + for obs in observations.iter_mut() { + *obs &= mask; + } + + let target = self.config.observation_stream_len; + if target == 0 { + return; + } + + if observations.len() > target { + match self.config.observation_stream_mode { + NyxObservationStreamMode::Truncate | NyxObservationStreamMode::PadTruncate => { + observations.truncate(target); + } + NyxObservationStreamMode::Pad => {} + } + } else if observations.len() < target { + match self.config.observation_stream_mode { + NyxObservationStreamMode::Pad | NyxObservationStreamMode::PadTruncate => { + let pad = self.config.observation_pad_byte as PerceptVal; + observations.resize(target, pad); + } + NyxObservationStreamMode::Truncate => {} + } + } + } + + fn action_count(&self) -> usize { + match &self.config.action_source { + NyxActionSource::Literal(actions) => actions.len(), + NyxActionSource::Fuzz(fuzz) => fuzz.mutators.len(), + } + } + + /// Direct access to the underlying NyxVM for advanced use cases. + pub fn vm(&self) -> &NyxVM { + &self.vm + } + + /// Mutable access to the underlying NyxVM. + pub fn vm_mut(&mut self) -> &mut NyxVM { + &mut self.vm + } + + /// Takes a new snapshot at the current state. + pub fn take_snapshot(&mut self) -> Arc { + self.vm.take_snapshot() + } + + /// Applies a specific snapshot. + pub fn apply_snapshot(&mut self, snapshot: &Arc) { + self.vm.apply_snapshot(snapshot); + } + + /// Resets trace model. + pub fn reset_trace_model(&mut self) -> anyhow::Result<()> { + if let Some(model) = &mut self.trace_model { + model + .reset() + .map_err(|err| anyhow::anyhow!("failed to reset trace model: {err}"))?; + } + Ok(()) + } + + /// Logs crashes and interesting behaviors to file. + fn log_crash(&self, action_payload: &[u8], result: &NyxStepResult, reward: i64) { + let Some(log_path) = &self.config.crash_log else { + return; + }; + + // Only log interesting exits + let is_interesting = matches!( + result.exit_reason, + NyxExitKind::Shutdown | NyxExitKind::Breakpoint | NyxExitKind::Timeout + ); + + if !is_interesting { + return; + } + + let log_entry = serde_json::json!({ + "timestamp": std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs(), + "exit_reason": format!("{:?}", result.exit_reason), + "action_payload": hex_encode(action_payload), + "action_payload_str": String::from_utf8_lossy(action_payload), + "output": String::from_utf8_lossy(&result.output), + "shared_memory": hex_encode(&result.shared_memory), + "reward": reward, + "parsed_obs": result.parsed_obs, + "parsed_rew": result.parsed_rew, + }); + + // Append to JSONL file + if let Ok(mut file) = OpenOptions::new().create(true).append(true).open(log_path) + && let Ok(json_str) = serde_json::to_string(&log_entry) + { + let _ = writeln!(file, "{}", json_str); + } + } +} + +// ============================================================================ +// Environment Trait Implementation +// ============================================================================ + +impl Environment for NyxVmEnvironment { + fn perform_action(&mut self, action: Action) { + if self.needs_reset + && let Err(e) = self.reset() + && self.config.debug_mode + { + eprintln!("[NyxVm] Reset failed: {}", e); + } + + let payload = match self.get_action_payload(action) { + Ok(payload) => payload.into_owned(), + Err(e) => { + if self.config.debug_mode { + eprintln!("[NyxVm] Invalid action: {}", e); + } + self.obs = 0; + self.rew = self.min_reward(); + self.obs_stream.clear(); + self.obs_stream.push(0); + self.step_in_episode = (self.step_in_episode + 1) % self.config.episode_steps; + if self.step_in_episode == 0 { + self.needs_reset = true; + } + return; + } + }; + + // Check action filter + match self.filter_action(&payload) { + Ok(Some(reject_reward)) => { + self.obs = 0; + self.rew = reject_reward.clamp(self.min_reward(), self.max_reward()); + self.obs_stream.clear(); + self.obs_stream.push(0); + self.step_in_episode = (self.step_in_episode + 1) % self.config.episode_steps; + if self.step_in_episode == 0 { + self.needs_reset = true; + } + return; + } + Ok(None) => {} + Err(e) => { + if self.config.debug_mode { + eprintln!("[NyxVm] Action filter scoring failed: {}", e); + } + self.obs = 0; + self.rew = self.min_reward(); + self.obs_stream.clear(); + self.obs_stream.push(0); + self.step_in_episode = (self.step_in_episode + 1) % self.config.episode_steps; + if self.step_in_episode == 0 { + self.needs_reset = true; + } + return; + } + } + + // Run the step + let wrapped_payload = self.wrap_action_payload(&payload); + let result = match self.run_step(&wrapped_payload) { + Ok(result) => result, + Err(e) => { + if self.config.debug_mode { + eprintln!("[NyxVm] Step failed: {}", e); + } + self.obs = 0; + self.rew = self.min_reward(); + self.obs_stream.clear(); + self.obs_stream.push(0); + self.step_in_episode = (self.step_in_episode + 1) % self.config.episode_steps; + if self.step_in_episode == 0 { + self.needs_reset = true; + } + return; + } + }; + + // Process results + self.obs_stream = self.build_observation_stream(&result); + self.obs = self.obs_stream.first().copied().unwrap_or(0); + self.rew = match self.compute_reward(&result) { + Ok(reward) => reward, + Err(e) => { + if self.config.debug_mode { + eprintln!("[NyxVm] Reward computation failed: {}", e); + } + self.min_reward() + } + }; + + // Log crashes and interesting behaviors + self.log_crash(&payload, &result, self.rew); + + if self.config.debug_mode { + eprintln!( + "[NyxVm] Action={} Obs={} Rew={} Done={:?} Exit={:?}", + action, self.obs, self.rew, result.done, result.exit_reason + ); + } + + self.step_in_episode = (self.step_in_episode + 1) % self.config.episode_steps; + if self.step_in_episode == 0 || result.done { + self.needs_reset = true; + } + } + + fn get_observation(&self) -> PerceptVal { + self.obs + } + + fn drain_observations(&mut self) -> Vec { + if self.obs_stream.is_empty() { + vec![self.obs] + } else { + std::mem::take(&mut self.obs_stream) + } + } + + fn get_reward(&self) -> Reward { + self.rew + } + + fn is_finished(&self) -> bool { + false + } + + fn get_observation_bits(&self) -> usize { + self.config.observation_bits + } + + fn get_reward_bits(&self) -> usize { + self.config.reward_bits + } + + fn get_action_bits(&self) -> usize { + let n = self.action_count(); + if n <= 1 { + return 1; + } + (n as f64).log2().ceil() as usize + } + + fn get_num_actions(&self) -> ActionAlphabet { + ActionAlphabet::try_from_usize(self.action_count()) + .expect("vm environment must expose a non-empty action alphabet") + } + + fn max_reward(&self) -> Reward { + let bits = self.config.reward_bits; + if bits >= 64 { + i64::MAX + } else if bits == 0 { + 0 + } else { + (1i64 << (bits - 1)) - 1 + } + } + + fn min_reward(&self) -> Reward { + let bits = self.config.reward_bits; + if bits >= 64 { + i64::MIN + } else if bits == 0 { + 0 + } else { + -(1i64 << (bits - 1)) + } + } +} + +// ============================================================================ +// Helper Functions +// ============================================================================ + +fn robust_hash_bytes(data: &[u8]) -> u64 { + let mut h = 0u64; + for &b in data { + h = h.rotate_left(7) ^ (b as u64); + } + h +} + +fn apply_mutator( + mutator: &FuzzMutator, + input: &mut Vec, + fuzz: &NyxFuzzConfig, + rng: &mut RandomGenerator, +) { + match mutator { + FuzzMutator::FlipBit => { + if input.is_empty() { + input.push(0); + } + let idx = rng.gen_range(input.len()); + let bit = rng.gen_range(8); + input[idx] ^= 1u8 << bit; + } + FuzzMutator::FlipByte => { + if input.is_empty() { + input.push(0); + } + let idx = rng.gen_range(input.len()); + input[idx] ^= rng.next_u64() as u8; + } + FuzzMutator::InsertByte => { + let idx = if input.is_empty() { + 0 + } else { + rng.gen_range(input.len() + 1) + }; + let byte = if !fuzz.dictionary.is_empty() { + let d = rng.gen_range(fuzz.dictionary.len()); + let entry = &fuzz.dictionary[d]; + if entry.is_empty() { + 0 + } else { + entry[rng.gen_range(entry.len())] + } + } else { + rng.next_u64() as u8 + }; + input.insert(idx, byte); + } + FuzzMutator::DeleteByte => { + if input.len() > 1 { + let idx = rng.gen_range(input.len()); + input.remove(idx); + } + } + FuzzMutator::SpliceSeed => { + if fuzz.seeds.is_empty() { + return; + } + let seed = &fuzz.seeds[rng.gen_range(fuzz.seeds.len())]; + if input.is_empty() { + input.extend_from_slice(seed); + } else if !seed.is_empty() { + let cut = rng.gen_range(input.len()); + let seed_cut = rng.gen_range(seed.len()); + let mut out = Vec::new(); + out.extend_from_slice(&input[..cut]); + out.extend_from_slice(&seed[seed_cut..]); + *input = out; + } + } + FuzzMutator::ResetSeed => { + if fuzz.seeds.is_empty() { + return; + } + *input = fuzz.seeds[rng.gen_range(fuzz.seeds.len())].clone(); + } + FuzzMutator::Havoc => { + let flips = 1 + rng.gen_range(8); + for _ in 0..flips { + if input.is_empty() { + input.push(0); + } + let idx = rng.gen_range(input.len()); + input[idx] ^= rng.next_u64() as u8; + } + } + } +} + +// ============================================================================ +// Tests +// ============================================================================ + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_hex_encoding() { + let data = b"hello"; + let encoded = hex_encode(data); + assert_eq!(encoded, "68656c6c6f"); + let decoded = hex_decode(&encoded).unwrap(); + assert_eq!(decoded, data); + } + + #[test] + fn test_robust_hash() { + let data1 = b"test data"; + let data2 = b"test data"; + let data3 = b"different"; + + assert_eq!(robust_hash_bytes(data1), robust_hash_bytes(data2)); + assert_ne!(robust_hash_bytes(data1), robust_hash_bytes(data3)); + } + + #[test] + fn test_payload_encoding() { + let utf8 = PayloadEncoding::Utf8; + let hex = PayloadEncoding::Hex; + + let data = b"test"; + assert_eq!(utf8.encode(data), "test"); + assert_eq!(hex.encode(data), "74657374"); + + assert_eq!(utf8.decode("test").unwrap(), data); + assert_eq!(hex.decode("74657374").unwrap(), data); + } + + fn fuzz_cfg_with_seed(rng_seed: u64) -> NyxFuzzConfig { + NyxFuzzConfig { + seeds: vec![b"seed-alpha".to_vec(), b"seed-beta".to_vec()], + mutators: vec![ + FuzzMutator::FlipBit, + FuzzMutator::FlipByte, + FuzzMutator::InsertByte, + FuzzMutator::DeleteByte, + FuzzMutator::SpliceSeed, + FuzzMutator::ResetSeed, + FuzzMutator::Havoc, + ], + min_len: 1, + max_len: 32, + dictionary: vec![b"DICT".to_vec(), b"TOK".to_vec()], + rng_seed, + } + } + + fn fuzz_payload_sequence(config: &NyxFuzzConfig, steps: usize) -> Vec> { + let mut current = config.seeds[0].clone(); + let mut rng = RandomGenerator::from_seed(config.rng_seed); + let mut out = Vec::with_capacity(steps); + for _ in 0..steps { + let mut input = current.clone(); + let idx = rng.gen_range(config.mutators.len()); + let mutator = &config.mutators[idx]; + apply_mutator(mutator, &mut input, config, &mut rng); + current = input.clone(); + out.push(input); + } + out + } + + #[test] + fn fuzz_mutation_sequence_is_reproducible_for_identical_rng_seed() { + let a = fuzz_payload_sequence(&fuzz_cfg_with_seed(77), 64); + let b = fuzz_payload_sequence(&fuzz_cfg_with_seed(77), 64); + assert_eq!(a, b); + } + + #[test] + fn fuzz_mutation_sequence_changes_for_different_rng_seed() { + let a = fuzz_payload_sequence(&fuzz_cfg_with_seed(77), 64); + let b = fuzz_payload_sequence(&fuzz_cfg_with_seed(78), 64); + assert_ne!(a, b); + } + + #[cfg(feature = "vm")] + #[test] + fn validate_allows_custom_reward_callbacks_for_runtime_configs() { + let mut config = NyxVmConfig::default(); + config.firecracker_config = "dummy-firecracker.json".to_string(); + config.reward_policy = NyxRewardPolicy::Custom(Arc::new(|_| 0)); + config + .validate() + .expect("custom reward callbacks should remain valid for direct runtime configs"); + } + + #[cfg(feature = "vm")] + #[test] + fn validate_canonical_spec_compatibility_rejects_custom_reward_callbacks() { + let mut config = NyxVmConfig::default(); + config.firecracker_config = "dummy-firecracker.json".to_string(); + config.reward_policy = NyxRewardPolicy::Custom(Arc::new(|_| 0)); + let err = config + .validate_canonical_spec_compatibility() + .expect_err("custom reward callbacks are not canonical"); + assert!(matches!( + err, + InfotheoryError::InvalidBackendConfig(message) + if message.contains("not representable in canonical specs") + )); + } + + #[cfg(feature = "vm")] + #[test] + fn from_environment_spec_builds_runtime_vm_config_without_legacy_json() { + use std::time::{SystemTime, UNIX_EPOCH}; + + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|duration| duration.as_nanos()) + .unwrap_or(0); + let root = std::env::temp_dir().join(format!( + "infotheory-vm-spec-runtime-{}-{nanos}", + std::process::id() + )); + std::fs::create_dir_all(&root).expect("temp dir"); + + let firecracker_path = root.join("firecracker.json"); + let baseline_path = root.join("baseline.bin"); + let novelty_path = root.join("novelty.bin"); + std::fs::write(&firecracker_path, b"{\"boot-source\":{}}").expect("firecracker config"); + std::fs::write(&baseline_path, b"baseline-bytes").expect("baseline asset"); + std::fs::write(&novelty_path, b"novelty-bytes").expect("novelty asset"); + + let spec = VmEnvironmentSpec { + firecracker_config_asset: "firecracker".to_string(), + instance_id: "vm-test".to_string(), + shared_region_name: "shared".to_string(), + shared_region_size: 4096, + shared_memory_policy: SharedMemoryPolicySpec::Snapshot, + step_timeout_ms: 125, + boot_timeout_ms: 1_250, + episode_steps: 8, + step_cost: -1, + observation_policy: VmObservationPolicySpec::OutputHash, + observation_bits: 8, + observation_stream_len: 16, + observation_stream_mode: VmObservationStreamModeSpec::PadTruncate, + observation_pad_byte: 0x7f, + reward_bits: 8, + reward_policy: VmRewardPolicySpec::Pattern { + pattern: "win".to_string(), + base_reward: 1, + bonus_reward: 4, + }, + reward_shaping: Some(VmRewardShapingSpec::EntropyReduction { + baseline_asset: "baseline".to_string(), + scale: 0.25, + crash_bonus: Some(5), + timeout_bonus: Some(6), + }), + action_source: VmRuntimeActionSourceSpec::Literal { + names: vec![Some("hi".to_string())], + payloads: vec!["6869".to_string()], + encoding: VmPayloadEncodingSpec::Hex, + }, + action_filter: Some(VmActionFilterSpec { + min_entropy: Some(0.1), + max_entropy: Some(2.0), + min_intrinsic_dependence: Some(0.05), + min_novelty: Some(0.2), + novelty_prior_asset: Some("novelty".to_string()), + reject_reward: Some(-3), + }), + action_prefix: "ACT ".to_string(), + action_suffix: "\n".to_string(), + obs_prefix: "OBS ".to_string(), + rew_prefix: "REW ".to_string(), + done_prefix: "DONE ".to_string(), + data_prefix: "DATA ".to_string(), + wire_encoding: VmPayloadEncodingSpec::Utf8, + stats_backend: RateBackend::Ctw { depth: 8 }, + trace: Some(VmTraceSpec { + shared_region_name: Some("trace".to_string()), + max_bytes: 256, + reset_on_episode: true, + }), + debug_mode: true, + crash_log: Some("/tmp/vm-crash.jsonl".to_string()), + }; + let assets = vec![ + ResolvedAssetBinding { + id: "firecracker".to_string(), + asset: AssetRef::Filesystem(firecracker_path.clone()), + }, + ResolvedAssetBinding { + id: "baseline".to_string(), + asset: AssetRef::Filesystem(baseline_path.clone()), + }, + ResolvedAssetBinding { + id: "novelty".to_string(), + asset: AssetRef::Filesystem(novelty_path.clone()), + }, + ]; + + let config = NyxVmConfig::from_environment_spec(&spec, &assets) + .expect("canonical VM spec should build runtime config"); + + assert_eq!( + config.firecracker_config, + firecracker_path.display().to_string() + ); + assert_eq!(config.crash_log.as_deref(), Some("/tmp/vm-crash.jsonl")); + assert!(matches!( + config.observation_policy, + NyxObservationPolicy::OutputHash + )); + assert!(matches!( + config.observation_stream_mode, + NyxObservationStreamMode::PadTruncate + )); + assert!(matches!( + config.reward_policy, + NyxRewardPolicy::Pattern { + ref pattern, + base_reward: 1, + bonus_reward: 4, + } if pattern == "win" + )); + assert!(matches!( + config.protocol.wire_encoding, + PayloadEncoding::Utf8 + )); + assert!(matches!( + config.stats_backend, + RateBackend::Ctw { depth: 8 } + )); + match &config.action_source { + NyxActionSource::Literal(actions) => { + assert_eq!(actions.len(), 1); + assert_eq!(actions[0].name.as_deref(), Some("hi")); + assert_eq!(actions[0].payload, b"hi"); + } + other => panic!("expected literal actions, got {other:?}"), + } + match &config.reward_shaping { + Some(NyxRewardShaping::EntropyReduction { baseline_bytes, .. }) => { + assert_eq!(baseline_bytes, b"baseline-bytes"); + } + other => panic!("expected entropy-reduction shaping, got {other:?}"), + } + match &config.action_filter { + Some(filter) => { + assert_eq!(filter.novelty_prior.as_deref(), Some(&b"novelty-bytes"[..])); + } + None => panic!("expected action filter"), + } + + let _ = std::fs::remove_file(firecracker_path); + let _ = std::fs::remove_file(baseline_path); + let _ = std::fs::remove_file(novelty_path); + let _ = std::fs::remove_dir(root); + } + + #[cfg(feature = "vm")] + #[test] + fn from_environment_spec_accepts_vm_alias_names() { + use std::time::{SystemTime, UNIX_EPOCH}; + + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|duration| duration.as_nanos()) + .unwrap_or(0); + let root = std::env::temp_dir().join(format!( + "infotheory-vm-spec-aliases-{}-{nanos}", + std::process::id() + )); + std::fs::create_dir_all(&root).expect("temp dir"); + + let firecracker_path = root.join("firecracker.json"); + std::fs::write(&firecracker_path, b"{\"boot-source\":{}}").expect("firecracker config"); + + let spec = VmEnvironmentSpec { + firecracker_config_asset: "firecracker".to_string(), + instance_id: "vm-test".to_string(), + shared_region_name: "shared".to_string(), + shared_region_size: 4096, + shared_memory_policy: SharedMemoryPolicySpec::Snapshot, + step_timeout_ms: 125, + boot_timeout_ms: 1_250, + episode_steps: 8, + step_cost: -1, + observation_policy: VmObservationPolicySpec::OutputHash, + observation_bits: 8, + observation_stream_len: 16, + observation_stream_mode: VmObservationStreamModeSpec::PadTruncate, + observation_pad_byte: 0x00, + reward_bits: 8, + reward_policy: VmRewardPolicySpec::FromGuest, + reward_shaping: None, + action_source: VmRuntimeActionSourceSpec::Fuzz { + seeds: vec!["seed".to_string()], + encoding: VmPayloadEncodingSpec::Utf8, + mutators: vec![VmFuzzMutatorSpec::FlipBit, VmFuzzMutatorSpec::SpliceSeed], + min_len: 1, + max_len: 8, + dictionary: vec!["dict".to_string()], + rng_seed: 7, + }, + action_filter: None, + action_prefix: "ACT ".to_string(), + action_suffix: "\n".to_string(), + obs_prefix: "OBS ".to_string(), + rew_prefix: "REW ".to_string(), + done_prefix: "DONE ".to_string(), + data_prefix: "DATA ".to_string(), + wire_encoding: VmPayloadEncodingSpec::Utf8, + stats_backend: RateBackend::Ctw { depth: 8 }, + trace: None, + debug_mode: false, + crash_log: None, + }; + let assets = vec![ResolvedAssetBinding { + id: "firecracker".to_string(), + asset: AssetRef::Filesystem(firecracker_path.clone()), + }]; + + let config = + NyxVmConfig::from_environment_spec(&spec, &assets).expect("aliases should parse"); + assert!(matches!( + config.observation_policy, + NyxObservationPolicy::OutputHash + )); + assert!(matches!( + config.observation_stream_mode, + NyxObservationStreamMode::PadTruncate + )); + assert!(matches!( + config.protocol.wire_encoding, + PayloadEncoding::Utf8 + )); + match &config.action_source { + NyxActionSource::Fuzz(fuzz) => { + assert_eq!(fuzz.seeds, vec![b"seed".to_vec()]); + assert!(matches!(fuzz.mutators[0], FuzzMutator::FlipBit)); + assert!(matches!(fuzz.mutators[1], FuzzMutator::SpliceSeed)); + } + other => panic!("expected fuzz action source, got {other:?}"), + } + + let _ = std::fs::remove_file(firecracker_path); + let _ = std::fs::remove_dir(root); + } + + #[cfg(feature = "all-backends")] + #[test] + fn trace_model_supports_predictor_backed_backends() { + use crate::api::{ + CalibratedSpec, CalibrationContextKind, MixtureExpertSpec, MixtureKind, MixtureSpec, + ParticleSpec, + }; + + let backends = vec![ + RateBackend::Match { + hash_bits: 20, + min_len: 4, + max_len: 255, + base_mix: 0.02, + confidence_scale: 1.0, + }, + RateBackend::SparseMatch { + hash_bits: 19, + min_len: 3, + max_len: 64, + gap_min: 1, + gap_max: 2, + base_mix: 0.05, + confidence_scale: 1.0, + }, + RateBackend::Ppmd { + order: 8, + memory_mb: 8, + }, + RateBackend::Calibrated { + spec: Arc::new(CalibratedSpec { + base: RateBackend::Ctw { depth: 8 }, + context: CalibrationContextKind::Text, + bins: 33, + learning_rate: 0.02, + bias_clip: 4.0, + }), + }, + RateBackend::Particle { + spec: Arc::new(ParticleSpec { + num_particles: 4, + num_cells: 4, + cell_dim: 8, + ..ParticleSpec::default() + }), + }, + RateBackend::Mixture { + spec: Arc::new(MixtureSpec::new( + MixtureKind::Bayes, + vec![MixtureExpertSpec { + name: Some("ctw".to_string()), + log_prior: 0.0, + backend: RateBackend::Ctw { depth: 8 }, + }], + )), + }, + ]; + + for backend in backends { + let compiled = backend.compile().expect("compiled trace backend"); + let mut model = TraceModel::new(&compiled).expect("trace model should initialize"); + let bits = model.update_and_score(b"trace payload"); + assert!(bits.is_finite() && bits >= 0.0, "bits={bits}"); + model.reset().expect("trace model should reset"); + let bits_after_reset = model.update_and_score(b"trace payload"); + assert!( + bits_after_reset.is_finite() && bits_after_reset >= 0.0, + "bits_after_reset={bits_after_reset}" + ); + } + } + + /// Scores `data` against an existing [`FacContextTree`] using the same + /// bit-extraction logic as `TraceModel::FacCtw::update_and_score`, then + /// returns the surprise in bits (negative log-prob delta / ln 2). + /// + /// The tree is mutated (updated) exactly as `update_and_score` would do, + /// so callers can chain multiple calls on the same tree to simulate + /// the VM's incremental scoring pattern. + #[cfg(feature = "backend-ctw")] + fn fac_ctw_oracle_score_on_tree( + tree: &mut crate::backends::ctw::FacContextTree, + bits_per_symbol: usize, + msb_first: bool, + data: &[u8], + ) -> f64 { + use crate::backends::ctw::ctw_symbol_bit_msb; + let log_before = tree.get_log_block_probability(); + for &b in data { + for i in 0..bits_per_symbol { + let bit = if msb_first { + ctw_symbol_bit_msb(b, bits_per_symbol, i) + } else { + ((b >> i) & 1) == 1 + }; + tree.update(bit, i); + } + } + let log_after = tree.get_log_block_probability(); + -(log_after - log_before) / std::f64::consts::LN_2 + } + + /// Computes the expected `update_and_score` result by driving a *fresh* + /// [`FacContextTree`] directly with the same bit-extraction logic used + /// inside `TraceModel::FacCtw::update_and_score`. + /// + /// This is the single-shot reference oracle used by parity tests. + /// For incremental (multi-chunk) scenarios use [`fac_ctw_oracle_score_on_tree`] + /// with a persistent tree. + #[cfg(feature = "backend-ctw")] + fn fac_ctw_oracle_score( + base_depth: usize, + bits_per_symbol: usize, + msb_first: bool, + data: &[u8], + ) -> f64 { + use crate::backends::ctw::FacContextTree; + let mut tree = FacContextTree::new(base_depth, bits_per_symbol); + fac_ctw_oracle_score_on_tree(&mut tree, bits_per_symbol, msb_first, data) + } + + /// Asserts that `TraceModel::FacCtw` with the given parameters scores + /// `data` identically (bit-exact `f64`) to the reference oracle, and that + /// `reset()` restores the model so a second pass yields the same score. + #[cfg(feature = "backend-ctw")] + fn assert_fac_ctw_trace_parity( + base_depth: usize, + encoding_bits: usize, + msb_first: Option, + data: &[u8], + ) { + // The plan resolves msb_first via `unwrap_or(encoding_bits == 8)`. + let resolved_msb_first = msb_first.unwrap_or(encoding_bits == 8); + + let backend = RateBackend::FacCtw { + base_depth, + num_percept_bits: encoding_bits, + encoding_bits, + msb_first, + }; + let compiled = backend + .compile() + .expect("FacCtw backend should compile cleanly"); + + let mut model = + TraceModel::new(&compiled).expect("TraceModel::FacCtw should initialize without error"); + + // ── First pass: trace model vs. oracle ───────────────────────────── + let trace_bits = model.update_and_score(data); + let oracle_bits = fac_ctw_oracle_score(base_depth, encoding_bits, resolved_msb_first, data); + + assert!( + trace_bits.is_finite() && trace_bits >= 0.0, + "trace model bits must be finite and non-negative; got {trace_bits} \ + (base_depth={base_depth}, encoding_bits={encoding_bits}, msb_first={msb_first:?})" + ); + assert_eq!( + trace_bits.to_bits(), + oracle_bits.to_bits(), + "TraceModel::FacCtw score must match FacContextTree oracle exactly \ + (base_depth={base_depth}, encoding_bits={encoding_bits}, msb_first={msb_first:?}); \ + trace={trace_bits}, oracle={oracle_bits}" + ); + + // ── Reset then second pass: scores must be identical to first pass ── + // This catches msb_first / bits_per_symbol state not being properly + // preserved across reset(), or the tree not being fully cleared. + model + .reset() + .expect("TraceModel::FacCtw reset should succeed"); + let trace_bits_after_reset = model.update_and_score(data); + + assert_eq!( + trace_bits_after_reset.to_bits(), + oracle_bits.to_bits(), + "TraceModel::FacCtw score after reset must equal the fresh-model score \ + (base_depth={base_depth}, encoding_bits={encoding_bits}, msb_first={msb_first:?}); \ + after_reset={trace_bits_after_reset}, expected={oracle_bits}" + ); + } + + /// Regression test: `TraceModel::FacCtw` with 8-bit symbols and MSB-first + /// ordering must use `ctw_symbol_bit_msb` to decompose each byte, not + /// the legacy LSB path. This is the primary regression target for the + /// branch that wired `msb_first=true` and `raw encoding_bits` into the + /// trace model. + #[cfg(feature = "backend-ctw")] + #[test] + fn trace_model_fac_ctw_msb_first_8bit_parity() { + // Use a non-trivial payload with varied bit patterns to exercise the + // full 8-bit MSB decomposition path. + let data = b"trace-model regression: fac-ctw msb path"; + assert_fac_ctw_trace_parity( + /*base_depth=*/ 6, + /*encoding_bits=*/ 8, + /*msb_first=*/ Some(true), + data, + ); + } + + /// Regression test: `TraceModel::FacCtw` with 8-bit symbols and explicit + /// LSB-first ordering must use `(b >> i) & 1`. Verifies the `msb_first` + /// flag is correctly threaded through from the compiled plan into the + /// update loop and is distinct from the MSB path above (the scores for the + /// same data must differ, proving the two paths are not identical). + #[cfg(feature = "backend-ctw")] + #[test] + fn trace_model_fac_ctw_lsb_first_8bit_parity() { + let data = b"trace-model regression: fac-ctw lsb path"; + assert_fac_ctw_trace_parity( + /*base_depth=*/ 6, + /*encoding_bits=*/ 8, + /*msb_first=*/ Some(false), + data, + ); + + // Sanity: the two orderings must produce distinct scores for non-palindromic + // bit patterns, confirming the flag actually controls bit extraction. + let oracle_msb = fac_ctw_oracle_score(6, 8, true, data); + let oracle_lsb = fac_ctw_oracle_score(6, 8, false, data); + assert_ne!( + oracle_msb.to_bits(), + oracle_lsb.to_bits(), + "MSB-first and LSB-first FacCtw must differ on non-palindromic data" + ); + } + + /// Regression test: `TraceModel::FacCtw` with a sub-byte `encoding_bits` + /// (4 bits per symbol) and MSB-first ordering, verifying that + /// `ctw_symbol_bit_msb` correctly addresses the low-4 bits of each byte. + /// + /// `num_percept_bits` is kept equal to `encoding_bits` here; see + /// [`trace_model_fac_ctw_encoding_bits_drives_width_not_num_percept_bits`] + /// for the dedicated guard that `TraceModel` uses `encoding_bits` (not + /// `num_percept_bits`) as the per-symbol bit width. + #[cfg(feature = "backend-ctw")] + #[test] + fn trace_model_fac_ctw_sub_byte_4bit_msb_parity() { + // Bytes whose lower nibble and upper nibble differ, so that LSB vs MSB + // ordering produces different bit sequences. + let data = &[0xA3u8, 0x5C, 0xF1, 0x7E, 0x29, 0xB4]; + assert_fac_ctw_trace_parity( + /*base_depth=*/ 4, + /*encoding_bits=*/ 4, + /*msb_first=*/ Some(true), + data, + ); + } + + /// Regression test: default `msb_first=None` with 8-bit symbols must + /// resolve to MSB-first. The rule `unwrap_or(encoding_bits == 8)` evaluates + /// to `true` for 8-bit symbols; this verifies that the `Option` → + /// `bool` resolution in `compile_rate_plan_fac_ctw` propagates end-to-end + /// through `TraceModel::new` into the update loop. + /// + /// The `assert_fac_ctw_trace_parity` call already exercises the full + /// compile → `TraceModel::new` → `update_and_score` path against the oracle + /// with the resolved `bool`. The additional assertion below confirms that + /// `None` and `Some(true)` produce bit-identical `TraceModel` scores on the + /// same data, ruling out any partial or inverted propagation. + #[cfg(feature = "backend-ctw")] + #[test] + fn trace_model_fac_ctw_default_msb_resolution_8bit() { + let data = b"default-msb resolution smoke test"; + + // None + encoding_bits=8 → resolved msb_first = true. + assert_fac_ctw_trace_parity( + /*base_depth=*/ 5, /*encoding_bits=*/ 8, /*msb_first=*/ None, data, + ); + + // Confirm: two TraceModels — one with None, one with Some(true) — must + // score the same data identically. This catches inversions or partial + // propagation that assert_fac_ctw_trace_parity (oracle-based) would miss + // if the oracle itself used the wrong convention. + let backend_none = RateBackend::FacCtw { + base_depth: 5, + num_percept_bits: 8, + encoding_bits: 8, + msb_first: None, + }; + let backend_explicit = RateBackend::FacCtw { + base_depth: 5, + num_percept_bits: 8, + encoding_bits: 8, + msb_first: Some(true), + }; + let mut model_none = + TraceModel::new(&backend_none.compile().expect("fac-ctw None compile")) + .expect("TraceModel::new (None)"); + let mut model_explicit = TraceModel::new( + &backend_explicit + .compile() + .expect("fac-ctw Some(true) compile"), + ) + .expect("TraceModel::new (Some(true))"); + assert_eq!( + model_none.update_and_score(data).to_bits(), + model_explicit.update_and_score(data).to_bits(), + "msb_first=None with encoding_bits=8 must produce the same score as Some(true)" + ); + } + + /// Regression test: default `msb_first=None` with a sub-byte `encoding_bits` + /// (4 bits) must resolve to LSB-first. The rule `unwrap_or(encoding_bits == 8)` + /// evaluates to `false` for any width other than 8; this verifies end-to-end + /// propagation for the sub-byte default case. + /// + /// An additional assertion confirms that `None` and `Some(false)` produce + /// bit-identical `TraceModel` scores, ruling out any inversion. + #[cfg(feature = "backend-ctw")] + #[test] + fn trace_model_fac_ctw_default_lsb_resolution_4bit() { + let data = &[0xA3u8, 0x5C, 0xF1, 0x7E, 0x29, 0xB4]; + + // None + encoding_bits=4 → resolved msb_first = false (LSB). + assert_fac_ctw_trace_parity( + /*base_depth=*/ 4, /*encoding_bits=*/ 4, /*msb_first=*/ None, data, + ); + + // Confirm: None score == Some(false) score via two TraceModel instances. + let backend_none = RateBackend::FacCtw { + base_depth: 4, + num_percept_bits: 4, + encoding_bits: 4, + msb_first: None, + }; + let backend_explicit = RateBackend::FacCtw { + base_depth: 4, + num_percept_bits: 4, + encoding_bits: 4, + msb_first: Some(false), + }; + let mut model_none = + TraceModel::new(&backend_none.compile().expect("fac-ctw None/4-bit compile")) + .expect("TraceModel::new (None/4-bit)"); + let mut model_explicit = TraceModel::new( + &backend_explicit + .compile() + .expect("fac-ctw Some(false)/4-bit compile"), + ) + .expect("TraceModel::new (Some(false)/4-bit)"); + assert_eq!( + model_none.update_and_score(data).to_bits(), + model_explicit.update_and_score(data).to_bits(), + "msb_first=None with encoding_bits=4 must produce the same score as Some(false)" + ); + } + + /// Regression guard: `TraceModel::FacCtw` must use `encoding_bits` as the + /// per-symbol bit width, not `num_percept_bits`. + /// + /// `TraceModel::new` explicitly patterns `num_percept_bits: _` and assigns + /// `bits_per_symbol = *encoding_bits`. A regression back to `num_percept_bits` + /// would cause 4-bit vs 8-bit symbol decomposition, producing a different bit + /// count and failing the oracle assertion (oracle is wired to `encoding_bits`). + #[cfg(feature = "backend-ctw")] + #[test] + fn trace_model_fac_ctw_encoding_bits_drives_width_not_num_percept_bits() { + // num_percept_bits=8 (AIXI percept cardinality) diverges from + // encoding_bits=4 (VM trace / rate-byte symbol width). + let data = &[0xA3u8, 0x5C, 0xF1, 0x7E, 0x29, 0xB4]; + let base_depth: usize = 4; + let encoding_bits: usize = 4; + + let backend = RateBackend::FacCtw { + base_depth, + num_percept_bits: 8, // intentionally differs from encoding_bits + encoding_bits, + msb_first: Some(true), + }; + let compiled = backend.compile().expect("fac-ctw compile"); + let mut model = TraceModel::new(&compiled).expect("TraceModel::new"); + + let trace_bits = model.update_and_score(data); + + // Oracle uses encoding_bits=4 (as the trace model must). + let oracle_4bit = fac_ctw_oracle_score(base_depth, encoding_bits, true, data); + assert_eq!( + trace_bits.to_bits(), + oracle_4bit.to_bits(), + "TraceModel must use encoding_bits={encoding_bits} as symbol width, not num_percept_bits=8; \ + trace={trace_bits}, oracle_4bit={oracle_4bit}" + ); + + // Confirm the test is meaningful: an oracle with 8-bit width produces a + // *different* score, so the assert above would catch a num_percept_bits regression. + let oracle_8bit = fac_ctw_oracle_score(base_depth, 8, true, data); + assert_ne!( + oracle_4bit.to_bits(), + oracle_8bit.to_bits(), + "4-bit and 8-bit FacCtw oracles must differ on this data (test is non-trivial)" + ); + } + + /// Regression test: `TraceModel::FacCtw` must produce the correct incremental + /// surprise when `update_and_score` is called multiple times on the same + /// persistent model — the normal VM usage pattern for trace-entropy shaping. + /// + /// Each call must score only the *new* bytes against the model already updated + /// by all prior calls; the oracle maintains a matching persistent + /// [`FacContextTree`] using [`fac_ctw_oracle_score_on_tree`]. + #[cfg(feature = "backend-ctw")] + #[test] + fn trace_model_fac_ctw_incremental_scoring_parity() { + use crate::backends::ctw::FacContextTree; + + let base_depth: usize = 5; + let encoding_bits: usize = 8; + let msb_first = true; + + let backend = RateBackend::FacCtw { + base_depth, + num_percept_bits: encoding_bits, + encoding_bits, + msb_first: Some(msb_first), + }; + let compiled = backend.compile().expect("fac-ctw compile"); + let mut model = TraceModel::new(&compiled).expect("TraceModel::new"); + + // Two distinct chunks sharing context (realistic VM trace pattern). + let chunk_a: &[u8] = b"incremental trace chunk A"; + let chunk_b: &[u8] = b"incremental trace chunk B -- different continuation"; + + // ── Trace model: two sequential updates ──────────────────────────── + let trace_bits_a = model.update_and_score(chunk_a); + let trace_bits_b = model.update_and_score(chunk_b); + + // ── Oracle: persistent tree updated through A then B ─────────────── + let mut oracle_tree = FacContextTree::new(base_depth, encoding_bits); + let oracle_bits_a = + fac_ctw_oracle_score_on_tree(&mut oracle_tree, encoding_bits, msb_first, chunk_a); + let oracle_bits_b = + fac_ctw_oracle_score_on_tree(&mut oracle_tree, encoding_bits, msb_first, chunk_b); + + assert_eq!( + trace_bits_a.to_bits(), + oracle_bits_a.to_bits(), + "incremental: first chunk score must match oracle; \ + trace={trace_bits_a}, oracle={oracle_bits_a}" + ); + assert_eq!( + trace_bits_b.to_bits(), + oracle_bits_b.to_bits(), + "incremental: second chunk score must match oracle after first chunk is consumed; \ + trace={trace_bits_b}, oracle={oracle_bits_b}" + ); + } + + /// Edge case: `update_and_score` on empty data must return exactly `0.0` + /// without mutating the model. The production guard is the top-level + /// `if data.is_empty() { return 0.0; }` in `update_and_score`. + #[cfg(feature = "backend-ctw")] + #[test] + fn trace_model_fac_ctw_empty_input_returns_zero() { + let backend = RateBackend::FacCtw { + base_depth: 4, + num_percept_bits: 8, + encoding_bits: 8, + msb_first: Some(true), + }; + let compiled = backend.compile().expect("fac-ctw compile"); + let mut model = TraceModel::new(&compiled).expect("TraceModel::new"); + + let bits_empty = model.update_and_score(b""); + assert_eq!( + bits_empty.to_bits(), + 0.0f64.to_bits(), + "empty input must return exactly 0.0" + ); + + // Confirm the model is unmodified: scoring non-empty data after an empty + // call must match a fresh oracle (no phantom state from the empty update). + let data = b"post-empty data"; + let bits_after = model.update_and_score(data); + let oracle_bits = fac_ctw_oracle_score(4, 8, true, data); + assert_eq!( + bits_after.to_bits(), + oracle_bits.to_bits(), + "model must be unmodified after empty update; \ + bits_after={bits_after}, oracle={oracle_bits}" + ); + } +} diff --git a/crates/infotheory/src/aixi/warmstart.rs b/crates/infotheory/src/aixi/warmstart.rs new file mode 100644 index 00000000..f21c26a0 --- /dev/null +++ b/crates/infotheory/src/aixi/warmstart.rs @@ -0,0 +1,4274 @@ +//! Warm-start exact finite-horizon objective controller for AIXI-family runs. + +use crate::aixi::common::{ + Action, ActionAlphabet, PerceptVal, RandomGenerator, Reward, RewardEncodingError, + resolve_random_seed, validate_reward_encoding_bounds, +}; +use crate::aixi::model::{Predictor, PredictorBuildError, build_aiqi_predictor}; +use crate::aixi::planner_agent::PlannerActionProvenance; +use crate::aixi::planner_spec::{PlannerInterfaceConfig, build_default_planner_run_spec}; +use crate::aixi::return_law::{ + ReturnLabelCodec, ReturnLawEvaluator, ReturnPrefixUpdate, predict_expected_label, +}; +use crate::aixi::warmstart_contract::{ + TaskFingerprint, WARMSTART_STANDALONE_OBSERVATION_ADAPTER_SPEC_REF, + WARMSTART_STANDALONE_SCALAR_REPRESENTATION, WARMSTART_TEACHER_CONTRACT_SCHEMA_VERSION, + observation_key_mode_name, standalone_exact_reward_encoding_certificate_hash, + standalone_observation_adapter_content_crc32, warmstart_exact_jh_planner_task_fingerprint, +}; +use crate::api::{BitStreamSemantics, RateBackend, validate_rate_backend}; +use crate::spec::{ + AssetBinding, BuiltinEnvironmentSpec, CompiledPlannerController, CompiledPlannerRunSpec, + ControllerSpec, EnvironmentSpec, PlannerRunSpec, SpecError, WarmStartExactJhControllerSpec, +}; +use serde_json::{Value, json}; +use std::cmp::Ordering; +use std::collections::{BTreeMap, BTreeSet}; +use std::error::Error; +use std::fmt; +use std::fs::File; +use std::io::{BufRead, BufReader}; +use std::num::NonZeroUsize; +use std::path::Path; + +/// One observed environment transition in a same-task warm-start trace. +#[derive(Clone, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub struct WarmStartExactJhTransition { + /// Action selected by the teacher/controller. + pub action: Action, + /// Observation stream emitted after the action. + pub observations: Vec, + /// Exact integer reward emitted after the action. + pub reward: Reward, +} + +impl WarmStartExactJhTransition { + /// Construct one warm-start teacher transition. + pub fn new(action: Action, observations: Vec, reward: Reward) -> Self { + Self { + action, + observations, + reward, + } + } +} + +/// Same-task trace used to initialize a warm-start exact-J_H controller. +#[derive(Clone, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub struct WarmStartExactJhTeacherTrace { + /// Chronological transition sequence. + pub transitions: Vec, +} + +impl WarmStartExactJhTeacherTrace { + /// Construct a same-task teacher trace from chronological transitions. + pub fn new(transitions: Vec) -> Self { + Self { transitions } + } +} + +/// Validates standalone planner-run provenance hashes against canonical standalone declarations. +/// +/// Used by the corresponding method on the private WarmStartExactJhRuntimeConfig +/// and by `validate_warmstart_teacher_against_compiled_planner_run`. +pub fn validate_standalone_warmstart_provenance( + contract: &WarmStartExactJhTeacherContract, + observation_bits: usize, + observation_stream_len: usize, + reward_bits: usize, +) -> Result<(), WarmStartExactJhError> { + if contract.observation_adapter_spec_ref != WARMSTART_STANDALONE_OBSERVATION_ADAPTER_SPEC_REF { + return Err(WarmStartExactJhError::InvalidTeacherDataset { + reason: format!( + "teacher observation_adapter_spec_ref '{}' does not match standalone direct-percept adapter declaration '{}'", + contract.observation_adapter_spec_ref, + WARMSTART_STANDALONE_OBSERVATION_ADAPTER_SPEC_REF + ), + }); + } + let expected_adapter_crc = standalone_observation_adapter_content_crc32( + observation_bits, + observation_stream_len, + reward_bits, + ) + .map_err(|err| WarmStartExactJhError::InvalidTeacherDataset { + reason: format!("failed to compute standalone observation adapter content hash: {err}"), + })?; + if contract.observation_adapter_content_crc32 != expected_adapter_crc { + return Err(WarmStartExactJhError::InvalidTeacherDataset { + reason: format!( + "teacher observation_adapter_content_crc32 '{}' does not match canonical standalone adapter spec '{}'", + contract.observation_adapter_content_crc32, expected_adapter_crc + ), + }); + } + if contract.scalar_representation != WARMSTART_STANDALONE_SCALAR_REPRESENTATION { + return Err(WarmStartExactJhError::InvalidTeacherDataset { + reason: format!( + "teacher scalar_representation '{}' does not match standalone nonnegative integer declaration '{}'", + contract.scalar_representation, WARMSTART_STANDALONE_SCALAR_REPRESENTATION + ), + }); + } + let expected_reward_cert = standalone_exact_reward_encoding_certificate_hash(reward_bits) + .map_err(|err| WarmStartExactJhError::InvalidTeacherDataset { + reason: format!( + "failed to compute standalone exact reward encoding certificate hash: {err}" + ), + })?; + if contract.exact_reward_encoding_certificate != expected_reward_cert { + return Err(WarmStartExactJhError::InvalidTeacherDataset { + reason: format!( + "teacher exact_reward_encoding_certificate '{}' does not match canonical standalone reward encoder certificate '{}'", + contract.exact_reward_encoding_certificate, expected_reward_cert + ), + }); + } + Ok(()) +} + +/// Expected warm-start teacher contract fields for comparison against a parsed contract. +pub(crate) struct WarmStartTeacherContractExpectation<'a> { + /// Expected schema version. + pub schema_version: u64, + /// Expected planner task fingerprint. + pub task_fingerprint: TaskFingerprint, + /// Expected action alphabet size. + pub action_alphabet_size: usize, + /// Expected observation bit width. + pub observation_bits: usize, + /// Expected observation stream length. + pub observation_stream_len: usize, + /// Expected observation key mode label. + pub observation_key_mode: &'a str, + /// Expected reward bit width. + pub reward_bits: usize, + /// Expected return horizon. + pub return_horizon: usize, + /// Expected delayed-label phase period. + pub label_phase_period: usize, + /// Whether standalone planner-run provenance hashes must match. + pub validate_standalone_provenance: bool, +} + +/// Validate a parsed teacher contract against an explicit field expectation. +pub(crate) fn validate_warmstart_teacher_contract_against_expectation( + contract: &WarmStartExactJhTeacherContract, + expected: &WarmStartTeacherContractExpectation<'_>, +) -> Result<(), WarmStartExactJhError> { + if contract.schema_version != expected.schema_version { + return Err(WarmStartExactJhError::InvalidTeacherDataset { + reason: format!("teacher schema_version must be {}", expected.schema_version), + }); + } + if contract.task_fingerprint != expected.task_fingerprint { + return Err(WarmStartExactJhError::InvalidTeacherDataset { + reason: format!( + "teacher task_fingerprint '{}' does not match current planner_run '{}'", + contract.task_fingerprint, expected.task_fingerprint + ), + }); + } + if contract.action_alphabet_size != expected.action_alphabet_size { + return Err(WarmStartExactJhError::InvalidTeacherDataset { + reason: format!( + "teacher action_alphabet_size {} does not match configured {}", + contract.action_alphabet_size, expected.action_alphabet_size + ), + }); + } + if contract.observation_bits != expected.observation_bits { + return Err(WarmStartExactJhError::InvalidTeacherDataset { + reason: format!( + "teacher observation_bits {} does not match configured {}", + contract.observation_bits, expected.observation_bits + ), + }); + } + if contract.observation_stream_len != expected.observation_stream_len { + return Err(WarmStartExactJhError::InvalidTeacherDataset { + reason: format!( + "teacher observation_stream_len {} does not match configured {}", + contract.observation_stream_len, expected.observation_stream_len + ), + }); + } + if contract.observation_key_mode != expected.observation_key_mode { + return Err(WarmStartExactJhError::InvalidTeacherDataset { + reason: format!( + "teacher observation_key_mode '{}' does not match configured planner interface '{}'", + contract.observation_key_mode, expected.observation_key_mode + ), + }); + } + if contract.reward_bits != expected.reward_bits { + return Err(WarmStartExactJhError::InvalidTeacherDataset { + reason: format!( + "teacher reward_bits {} does not match configured {}", + contract.reward_bits, expected.reward_bits + ), + }); + } + if contract.return_horizon != expected.return_horizon { + return Err(WarmStartExactJhError::InvalidTeacherDataset { + reason: format!( + "teacher return_horizon {} does not match configured {}", + contract.return_horizon, expected.return_horizon + ), + }); + } + if contract.label_phase_period != expected.label_phase_period { + return Err(WarmStartExactJhError::InvalidTeacherDataset { + reason: format!( + "teacher label_phase_period {} does not match configured {}", + contract.label_phase_period, expected.label_phase_period + ), + }); + } + if expected.validate_standalone_provenance { + validate_standalone_warmstart_provenance( + contract, + expected.observation_bits, + expected.observation_stream_len, + expected.reward_bits, + )?; + } + Ok(()) +} + +/// Validates teacher [`WarmStartExactJhTeacherContract::schema_version`] and +/// [`WarmStartExactJhTeacherContract::task_fingerprint`] against a compiled planner run. +/// +/// Used by [`validate_warmstart_teacher_against_compiled_planner_run`] (standalone CLI / assets) +/// and direct fingerprint probes. The tuner bridge validates complete teacher datasets through +/// [`validate_warmstart_teacher_dataset_for_compiled_planner_run`], which includes this +/// fingerprint check via the compiled runtime contract and then validates trace payloads. +/// On mismatch the reason string includes +/// `current planner_run ''` for stable integration-test probing. +pub fn validate_warmstart_teacher_planner_task_fingerprint( + compiled: &CompiledPlannerRunSpec, + contract: &WarmStartExactJhTeacherContract, +) -> Result<(), WarmStartExactJhError> { + if contract.schema_version != WARMSTART_TEACHER_CONTRACT_SCHEMA_VERSION { + return Err(WarmStartExactJhError::InvalidTeacherDataset { + reason: format!( + "teacher schema_version must be {WARMSTART_TEACHER_CONTRACT_SCHEMA_VERSION}" + ), + }); + } + let task_fingerprint = + warmstart_exact_jh_planner_task_fingerprint(compiled).map_err(|err| { + WarmStartExactJhError::InvalidTeacherDataset { + reason: format!("failed to compute planner task fingerprint: {err}"), + } + })?; + if contract.task_fingerprint != task_fingerprint { + return Err(WarmStartExactJhError::InvalidTeacherDataset { + reason: format!( + "teacher task_fingerprint '{}' does not match current planner_run '{}'", + contract.task_fingerprint, task_fingerprint + ), + }); + } + Ok(()) +} + +/// Validates a parsed teacher contract against a compiled standalone [`PlannerRunSpec`] (CLI / asset loader). +/// +/// This is the single authoritative check for filesystem-loaded teachers before runtime construction. +pub fn validate_warmstart_teacher_against_compiled_planner_run( + compiled: &CompiledPlannerRunSpec, + contract: &WarmStartExactJhTeacherContract, +) -> Result<(), WarmStartExactJhError> { + let interface = compiled.interface(); + let (return_horizon, label_phase_period, planner_simulations_per_step) = + match compiled.controller() { + CompiledPlannerController::AiqiWarmstartExactJh { + return_horizon, + label_phase_period, + planner_simulations_per_step, + .. + } => ( + *return_horizon, + *label_phase_period, + *planner_simulations_per_step, + ), + _ => { + return Err(WarmStartExactJhError::InvalidTeacherDataset { + reason: + "warm-start teacher contract can only be validated for aiqi_warmstart_exact_jh" + .to_string(), + }); + } + }; + let task_fingerprint = + warmstart_exact_jh_planner_task_fingerprint(compiled).map_err(|err| { + WarmStartExactJhError::InvalidTeacherDataset { + reason: format!("failed to compute planner task fingerprint: {err}"), + } + })?; + validate_warmstart_teacher_contract_against_expectation( + contract, + &WarmStartTeacherContractExpectation { + schema_version: WARMSTART_TEACHER_CONTRACT_SCHEMA_VERSION, + task_fingerprint, + action_alphabet_size: interface.agent_actions.get(), + observation_bits: interface.observation_bits, + observation_stream_len: interface.observation_stream_len.max(1), + observation_key_mode: observation_key_mode_name(interface.observation_key_mode), + reward_bits: interface.reward_bits, + return_horizon, + label_phase_period, + validate_standalone_provenance: true, + }, + )?; + if planner_simulations_per_step == 0 { + return Err(WarmStartExactJhError::PlannerSimulationsZero); + } + if planner_simulations_per_step != 1 { + return Err(WarmStartExactJhError::PlannerSimulationsUnsupported { + configured: planner_simulations_per_step, + }); + } + Ok(()) +} + +/// Validate a complete warm-start teacher dataset against a compiled planner run. +/// +/// This is the authoritative ingestion/export gate for teacher assets. It checks +/// the same runtime contract used by [`WarmStartExactJhAgent`]: controller +/// compatibility, task fingerprint, provenance policy, transition bounds, exact +/// finite-horizon label encodability, and that every trace contributes at least +/// one complete \(H\)-step label. +pub fn validate_warmstart_teacher_dataset_for_compiled_planner_run( + compiled: &CompiledPlannerRunSpec, + teacher: &WarmStartExactJhTeacherDataset, +) -> Result<(), WarmStartExactJhError> { + let config = WarmStartExactJhRuntimeConfig::from_compiled(compiled)?; + config.validate_teacher_contract(&teacher.contract)?; + let predictor = match compiled.controller() { + CompiledPlannerController::AiqiWarmstartExactJh { predictor, .. } => predictor, + _ => return Err(WarmStartExactJhError::ControllerKindMismatch), + }; + if !predictor.supports_frozen_conditioning() { + return Err(WarmStartExactJhError::UnsupportedRateBackend { + reason: "warm-start exact-J_H strict mode requires frozen context conditioning; configured rate_backend does not provide strict frozen conditioning", + }); + } + + let mut total_labels: usize = 0; + for (trace_index, trace) in teacher.traces.iter().enumerate() { + let trace_len = trace.transitions.len(); + if trace_len < config.return_horizon { + return Err(WarmStartExactJhError::InvalidTeacherDataset { + reason: format!( + "traces[{trace_index}] contains {trace_len} transitions but return_horizon is {}", + config.return_horizon + ), + }); + } + for (step_index, transition) in trace.transitions.iter().enumerate() { + validate_runtime_transition( + &config, + transition.action, + &transition.observations, + transition.reward, + ) + .map_err(|err| WarmStartExactJhError::InvalidTeacherDataset { + reason: format!( + "traces[{trace_index}].transitions[{step_index}] violates runtime contract: {err}" + ), + })?; + } + let labels = exact_return_labels_for_trace(&config, &trace.transitions)?; + let label_count = labels.iter().filter(|label| label.is_some()).count(); + if label_count == 0 { + return Err(WarmStartExactJhError::InvalidTeacherDataset { + reason: format!("traces[{trace_index}] did not contain any complete H-step labels"), + }); + } + total_labels = total_labels.saturating_add(label_count); + } + + if total_labels == 0 { + return Err(WarmStartExactJhError::InvalidTeacherDataset { + reason: "teacher dataset did not contain any complete H-step labels".to_string(), + }); + } + Ok(()) +} + +/// Validate one reconstructed teacher transition against a warm-start bridge contract. +pub fn validate_warmstart_teacher_transition_against_contract( + contract: &WarmStartExactJhTeacherContract, + action: Action, + observations: &[PerceptVal], + reward: Reward, +) -> Result<(), WarmStartExactJhError> { + let agent_actions = + ActionAlphabet::try_from_usize(contract.action_alphabet_size).map_err(|_| { + WarmStartExactJhError::InvalidTeacherDataset { + reason: "teacher contract action_alphabet_size is zero".to_string(), + } + })?; + if action as usize >= contract.action_alphabet_size { + return Err(WarmStartExactJhError::ActionOutOfRange { + action, + agent_actions, + }); + } + if observations.len() != contract.observation_stream_len { + return Err(WarmStartExactJhError::ObservationStreamLengthMismatch { + expected: contract.observation_stream_len, + actual: observations.len(), + }); + } + let obs_max = max_value_for_bits(contract.observation_bits); + for &observation in observations { + if observation > obs_max { + return Err(WarmStartExactJhError::ObservationValueOutOfRange { + observation, + observation_bits: contract.observation_bits, + maximum: obs_max, + }); + } + } + let max_reward = max_value_for_bits(contract.reward_bits) as i64; + if reward < 0 || reward > max_reward { + return Err(WarmStartExactJhError::RewardOutOfRange { + reward, + min_reward: 0, + max_reward, + }); + } + Ok(()) +} + +/// Same-task teacher dataset for [`WarmStartExactJhAgent`]. +#[derive(Clone, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub struct WarmStartExactJhTeacherDataset { + /// Versioned same-task contract metadata for the teacher traces. + pub contract: WarmStartExactJhTeacherContract, + /// Canonical teacher traces. Each trace is treated as an independent + /// same-task rollout, and dataset construction sorts and deduplicates this + /// set by transition content. + pub traces: Vec, +} + +impl WarmStartExactJhTeacherDataset { + /// Construct a teacher dataset from its contract and trace set. + /// + /// The supplied traces may be in arbitrary order and may contain + /// duplicates; the dataset stores the canonical deterministic set. + pub fn new( + contract: WarmStartExactJhTeacherContract, + mut traces: Vec, + ) -> Self { + canonicalize_warmstart_teacher_traces(&mut traces); + Self { contract, traces } + } +} + +/// Versioned same-task contract attached to a warm-start teacher dataset. +#[derive(Clone, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub struct WarmStartExactJhTeacherContract { + /// Teacher dataset schema version. Version 1 is the v1 exact-J_H contract. + pub schema_version: u64, + /// Fingerprint of the exact tuning task that produced the traces. + pub task_fingerprint: TaskFingerprint, + /// Number of actions in the compiled planner alphabet. + pub action_alphabet_size: usize, + /// Observation bit width. + pub observation_bits: usize, + /// Number of observation symbols per step. + pub observation_stream_len: usize, + /// Observation keying mode used by the planner-visible history. + pub observation_key_mode: String, + /// Observation adapter declaration reference used to encode raw tuner observations. + pub observation_adapter_spec_ref: String, + /// Content hash of the concrete observation adapter schema. + pub observation_adapter_content_crc32: String, + /// Reward bit width. + pub reward_bits: usize, + /// Return horizon used to compute exact labels. + pub return_horizon: usize, + /// Delayed-label phase period. + pub label_phase_period: usize, + /// Scalar representation declaration used by the exact reward encoder. + pub scalar_representation: String, + /// Hash or ref for the verified exact reward encoder. + pub exact_reward_encoding_certificate: String, +} + +impl WarmStartExactJhTeacherDataset { + /// Parse a JSON teacher dataset. + pub fn from_json_slice(bytes: &[u8]) -> Result { + let value = serde_json::from_slice::(bytes).map_err(|err| { + WarmStartExactJhError::InvalidTeacherDataset { + reason: format!("invalid teacher JSON: {err}"), + } + })?; + Self::from_json_value(&value) + } + + /// Parse a JSON teacher dataset value. + pub fn from_json_value(value: &Value) -> Result { + let object = + value + .as_object() + .ok_or_else(|| WarmStartExactJhError::InvalidTeacherDataset { + reason: "teacher dataset must be a JSON object".to_string(), + })?; + ensure_teacher_fields( + object, + &["schema_version", "contract", "traces"], + "teacher dataset", + )?; + let schema_version = object + .get("schema_version") + .and_then(Value::as_u64) + .ok_or_else(|| WarmStartExactJhError::InvalidTeacherDataset { + reason: format!( + "teacher dataset requires schema_version={WARMSTART_TEACHER_CONTRACT_SCHEMA_VERSION}" + ), + })?; + if schema_version != WARMSTART_TEACHER_CONTRACT_SCHEMA_VERSION { + return Err(WarmStartExactJhError::InvalidTeacherDataset { + reason: format!( + "teacher dataset schema_version must be {WARMSTART_TEACHER_CONTRACT_SCHEMA_VERSION}, got {schema_version}" + ), + }); + } + let contract = parse_teacher_contract(object, schema_version)?; + let traces_value = + object + .get("traces") + .ok_or_else(|| WarmStartExactJhError::InvalidTeacherDataset { + reason: "teacher dataset requires a 'traces' array".to_string(), + })?; + let traces_array = traces_value.as_array().ok_or_else(|| { + WarmStartExactJhError::InvalidTeacherDataset { + reason: "teacher dataset field 'traces' must be an array".to_string(), + } + })?; + let mut traces = Vec::with_capacity(traces_array.len()); + for (trace_index, trace_value) in traces_array.iter().enumerate() { + traces.push(parse_teacher_trace(trace_value, trace_index)?); + } + if traces.is_empty() { + return Err(WarmStartExactJhError::InvalidTeacherDataset { + reason: "teacher dataset must contain at least one trace".to_string(), + }); + } + canonicalize_warmstart_teacher_traces(&mut traces); + Ok(Self { contract, traces }) + } + + /// Convert this teacher dataset to its JSON representation. + pub fn to_json_value(&self) -> Value { + json!({ + "schema_version": WARMSTART_TEACHER_CONTRACT_SCHEMA_VERSION, + "contract": teacher_contract_to_json_value(&self.contract), + "traces": self.traces.iter().map(teacher_trace_to_json_value).collect::>(), + }) + } + + /// Count labels constructible from this dataset for the supplied horizon. + pub fn label_count_for_horizon(&self, return_horizon: usize) -> usize { + if return_horizon == 0 { + return 0; + } + self.traces + .iter() + .map(|trace| { + trace + .transitions + .len() + .saturating_add(1) + .saturating_sub(return_horizon) + }) + .sum() + } +} + +fn teacher_contract_to_json_value(contract: &WarmStartExactJhTeacherContract) -> Value { + json!({ + "task_fingerprint": contract.task_fingerprint.to_string(), + "action_alphabet_size": contract.action_alphabet_size, + "observation_bits": contract.observation_bits, + "observation_stream_len": contract.observation_stream_len, + "observation_key_mode": contract.observation_key_mode, + "observation_adapter_spec_ref": contract.observation_adapter_spec_ref, + "observation_adapter_content_crc32": contract.observation_adapter_content_crc32, + "reward_bits": contract.reward_bits, + "return_horizon": contract.return_horizon, + "label_phase_period": contract.label_phase_period, + "scalar_representation": contract.scalar_representation, + "exact_reward_encoding_certificate": contract.exact_reward_encoding_certificate, + }) +} + +fn teacher_trace_to_json_value(trace: &WarmStartExactJhTeacherTrace) -> Value { + json!({ + "transitions": trace.transitions.iter().map(|transition| { + json!({ + "action": transition.action, + "observations": transition.observations, + "reward": transition.reward, + }) + }).collect::>(), + }) +} + +fn compare_warmstart_teacher_traces( + left: &WarmStartExactJhTeacherTrace, + right: &WarmStartExactJhTeacherTrace, +) -> Ordering { + let mut left_iter = left.transitions.iter(); + let mut right_iter = right.transitions.iter(); + loop { + match (left_iter.next(), right_iter.next()) { + (Some(left_transition), Some(right_transition)) => { + let ordering = left_transition + .action + .cmp(&right_transition.action) + .then_with(|| { + left_transition + .observations + .as_slice() + .cmp(right_transition.observations.as_slice()) + }) + .then_with(|| left_transition.reward.cmp(&right_transition.reward)); + if !ordering.is_eq() { + return ordering; + } + } + (None, Some(_)) => return Ordering::Less, + (Some(_), None) => return Ordering::Greater, + (None, None) => return Ordering::Equal, + } + } +} + +fn canonicalize_warmstart_teacher_traces(traces: &mut Vec) { + traces.sort_by(compare_warmstart_teacher_traces); + traces.dedup_by(|right, left| compare_warmstart_teacher_traces(left, right).is_eq()); +} + +fn insert_warmstart_teacher_trace_canonical( + traces: &mut Vec, + trace: WarmStartExactJhTeacherTrace, +) -> Option { + match traces.binary_search_by(|existing| compare_warmstart_teacher_traces(existing, &trace)) { + Ok(_) => None, + Err(index) => { + traces.insert(index, trace); + Some(index) + } + } +} + +/// Merge one teacher trace into `traces`, preserving deterministic lexicographic order. +pub fn merge_warmstart_teacher_trace_deterministic( + traces: &mut Vec, + trace: WarmStartExactJhTeacherTrace, +) -> bool { + canonicalize_warmstart_teacher_traces(traces); + insert_warmstart_teacher_trace_canonical(traces, trace).is_some() +} + +/// Merge teacher traces into `traces`, returning the number and payload of inserted traces. +/// +/// Existing and incoming traces are canonicalized as a deterministic set before +/// insertion, so callers may pass traces in arbitrary order. +pub fn merge_warmstart_teacher_traces_deterministic( + traces: &mut Vec, + incoming: I, +) -> (usize, Vec) +where + I: IntoIterator, +{ + canonicalize_warmstart_teacher_traces(traces); + let mut inserted = Vec::new(); + for trace in incoming { + if let Some(index) = insert_warmstart_teacher_trace_canonical(traces, trace) { + inserted.push(traces[index].clone()); + } + } + (inserted.len(), inserted) +} + +/// Records normalized warm-start action/percept telemetry into one teacher trace. +#[derive(Clone, Debug, Default)] +pub struct WarmStartExactJhTraceRecorder { + actions: BTreeMap, + percepts: BTreeMap, Reward)>, +} + +impl WarmStartExactJhTraceRecorder { + /// Construct an empty trace recorder. + pub fn new() -> Self { + Self::default() + } + + /// Record an action at step `step`. + pub fn record_action( + &mut self, + step: usize, + action: Action, + ) -> Result<(), WarmStartExactJhError> { + if self.actions.insert(step, action).is_some() { + return Err(invalid_telemetry(format!( + "duplicate action record for step {step}" + ))); + } + Ok(()) + } + + /// Record a post-action percept at step `step`. + pub fn record_percept( + &mut self, + step: usize, + observations: &[PerceptVal], + reward: Reward, + ) -> Result<(), WarmStartExactJhError> { + if self + .percepts + .insert(step, (observations.to_vec(), reward)) + .is_some() + { + return Err(invalid_telemetry(format!( + "duplicate percept record for step {step}" + ))); + } + Ok(()) + } + + /// Convert the recorded telemetry into a validated teacher trace. + pub fn into_teacher_trace( + self, + contract: &WarmStartExactJhTeacherContract, + return_horizon: usize, + ) -> Result { + if self.actions.len() != self.percepts.len() { + return Err(invalid_telemetry( + "action/percept record counts do not match", + )); + } + let action_steps = self.actions.keys().copied().collect::>(); + let percept_steps = self.percepts.keys().copied().collect::>(); + if action_steps != percept_steps { + return Err(invalid_telemetry("action/percept step sets do not match")); + } + ensure_dense_step_set(&action_steps, "recorded action/percept")?; + let mut transitions = Vec::with_capacity(self.actions.len()); + for (step, action) in self.actions { + let (observations, reward) = self + .percepts + .get(&step) + .ok_or_else(|| invalid_telemetry(format!("missing percept for step {step}")))?; + validate_warmstart_teacher_transition_against_contract( + contract, + action, + observations, + *reward, + )?; + transitions.push(WarmStartExactJhTransition { + action, + observations: observations.clone(), + reward: *reward, + }); + } + if transitions.len() < return_horizon { + return Err(invalid_telemetry(format!( + "trace contains {} transitions but return_horizon is {return_horizon}", + transitions.len() + ))); + } + Ok(WarmStartExactJhTeacherTrace { transitions }) + } +} + +/// Build a normalized JSONL action record. +pub fn warmstart_jsonl_action_record( + step: usize, + action: Action, + provenance: PlannerActionProvenance, +) -> Value { + json!({ + "kind": "action", + "t": step, + "action": action, + "provenance": provenance.as_str(), + }) +} + +/// Build a normalized JSONL percept record. +pub fn warmstart_jsonl_percept_record( + step: usize, + observations: &[PerceptVal], + reward: Reward, +) -> Value { + json!({ + "kind": "percept", + "t": step, + "observations": observations, + "reward": reward, + }) +} + +fn invalid_telemetry(reason: impl Into) -> WarmStartExactJhError { + WarmStartExactJhError::InvalidTelemetry { + reason: reason.into(), + } +} + +fn ensure_dense_step_set( + steps: &BTreeSet, + label: &str, +) -> Result<(), WarmStartExactJhError> { + let Some(&first) = steps.first() else { + return Ok(()); + }; + let mut previous = first; + for &step in steps.iter().skip(1) { + let expected = previous.checked_add(1).ok_or_else(|| { + invalid_telemetry(format!( + "{label} step set cannot be dense after usize::MAX step {previous}" + )) + })?; + if step != expected { + return Err(invalid_telemetry(format!( + "{label} step set is not contiguous: expected step {expected} before step {step}" + ))); + } + previous = step; + } + Ok(()) +} + +fn ensure_jsonl_fields( + object: &serde_json::Map, + allowed: &[&str], + line_number: usize, +) -> Result<(), WarmStartExactJhError> { + for key in object.keys() { + if !allowed.contains(&key.as_str()) { + return Err(invalid_telemetry(format!( + "line {line_number}: unknown field '{key}'" + ))); + } + } + Ok(()) +} + +fn jsonl_required_u64( + value: &Value, + field: &str, + line_number: usize, +) -> Result { + value.get(field).and_then(Value::as_u64).ok_or_else(|| { + invalid_telemetry(format!("line {line_number}: missing u64 field '{field}'")) + }) +} + +fn jsonl_required_i64( + value: &Value, + field: &str, + line_number: usize, +) -> Result { + value.get(field).and_then(Value::as_i64).ok_or_else(|| { + invalid_telemetry(format!("line {line_number}: missing i64 field '{field}'")) + }) +} + +fn jsonl_required_observations( + value: &Value, + line_number: usize, +) -> Result, WarmStartExactJhError> { + let observations = value + .get("observations") + .and_then(Value::as_array) + .ok_or_else(|| { + invalid_telemetry(format!("line {line_number}: missing observations array")) + })?; + observations + .iter() + .map(|item| { + item.as_u64().ok_or_else(|| { + invalid_telemetry(format!("line {line_number}: observation must be a u64")) + }) + }) + .collect() +} + +#[derive(Clone, Debug)] +struct JsonlActionRecord { + action: Action, + line_number: usize, +} + +#[derive(Clone, Debug)] +struct JsonlPerceptRecord { + observations: Vec, + reward: Reward, + line_number: usize, +} + +#[derive(Clone, Debug, Default)] +struct JsonlStepRecords { + action: Option, + percept: Option, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum JsonlTraceConvention { + ActionThenPostPercept, + DecisionPerceptThenAction, +} + +fn infer_jsonl_trace_convention( + steps: &BTreeMap, +) -> Result { + let mut saw_action_then_percept = false; + let mut saw_percept_then_action = false; + for records in steps.values() { + let (Some(action), Some(percept)) = (&records.action, &records.percept) else { + continue; + }; + match action.line_number.cmp(&percept.line_number) { + std::cmp::Ordering::Less => saw_action_then_percept = true, + std::cmp::Ordering::Greater => saw_percept_then_action = true, + std::cmp::Ordering::Equal => { + return Err(invalid_telemetry( + "action and percept records cannot originate from the same JSONL line", + )); + } + } + } + match (saw_action_then_percept, saw_percept_then_action) { + (true, false) => Ok(JsonlTraceConvention::ActionThenPostPercept), + (false, true) => Ok(JsonlTraceConvention::DecisionPerceptThenAction), + (true, true) => Err(invalid_telemetry( + "mixed JSONL action/percept conventions in one trace", + )), + (false, false) => Err(invalid_telemetry( + "cannot infer JSONL action/percept convention", + )), + } +} + +fn push_validated_jsonl_transition( + transitions: &mut Vec, + contract: &WarmStartExactJhTeacherContract, + action_step: usize, + action: Action, + percept: &JsonlPerceptRecord, +) -> Result<(), WarmStartExactJhError> { + validate_warmstart_teacher_transition_against_contract( + contract, + action, + &percept.observations, + percept.reward, + ) + .map_err(|err| { + invalid_telemetry(format!( + "action step {action_step} paired with percept line {} violates contract: {err}", + percept.line_number + )) + })?; + transitions.push(WarmStartExactJhTransition { + action, + observations: percept.observations.clone(), + reward: percept.reward, + }); + Ok(()) +} + +fn jsonl_steps_into_teacher_trace( + steps: BTreeMap, + contract: &WarmStartExactJhTeacherContract, + return_horizon: usize, +) -> Result { + let convention = infer_jsonl_trace_convention(&steps)?; + let mut transitions = Vec::new(); + match convention { + JsonlTraceConvention::ActionThenPostPercept => { + let all_steps = steps.keys().copied().collect::>(); + ensure_dense_step_set(&all_steps, "action-then-percept JSONL")?; + for (step, records) in &steps { + let action = records.action.as_ref().ok_or_else(|| { + invalid_telemetry(format!("missing action record for step {step}")) + })?; + let percept = records.percept.as_ref().ok_or_else(|| { + invalid_telemetry(format!("missing percept record for step {step}")) + })?; + push_validated_jsonl_transition( + &mut transitions, + contract, + *step, + action.action, + percept, + )?; + } + } + JsonlTraceConvention::DecisionPerceptThenAction => { + let action_steps = steps + .iter() + .filter_map(|(step, records)| records.action.as_ref().map(|_| *step)) + .collect::>(); + let Some(&first_action_step) = action_steps.first() else { + return Err(invalid_telemetry("trace contains no action records")); + }; + ensure_dense_step_set(&action_steps, "decision-percept JSONL action")?; + let max_action_step = *action_steps + .last() + .ok_or_else(|| invalid_telemetry("trace contains no action records"))?; + let final_percept_step = max_action_step.checked_add(1).ok_or_else(|| { + invalid_telemetry(format!( + "decision-percept JSONL action step {max_action_step} cannot have a successor" + )) + })?; + for step in steps.keys() { + if *step < first_action_step || *step > final_percept_step { + return Err(invalid_telemetry(format!( + "unexpected JSONL step {step} outside dense decision trace domain {first_action_step}..={final_percept_step}" + ))); + } + if !action_steps.contains(step) && *step != final_percept_step { + return Err(invalid_telemetry(format!( + "unexpected percept-only JSONL step {step} inside dense decision trace" + ))); + } + } + for step in &action_steps { + let records = steps + .get(step) + .expect("action_steps contains only keys present in steps"); + let action = records + .action + .as_ref() + .expect("action_steps contains only records with actions"); + if records.percept.is_none() { + return Err(invalid_telemetry(format!( + "missing decision percept record for action step {step}" + ))); + } + let next_step = step.checked_add(1).ok_or_else(|| { + invalid_telemetry(format!( + "decision-percept JSONL action step {step} cannot have a successor" + )) + })?; + let next_records = steps.get(&next_step).ok_or_else(|| { + invalid_telemetry(format!( + "missing post-action percept for action step {step}" + )) + })?; + let Some(percept) = next_records.percept.as_ref() else { + return Err(invalid_telemetry(format!( + "missing post-action percept for action step {step}" + ))); + }; + push_validated_jsonl_transition( + &mut transitions, + contract, + *step, + action.action, + percept, + )?; + } + } + } + if transitions.len() < return_horizon { + return Err(invalid_telemetry(format!( + "trace contains {} complete transitions but return_horizon is {return_horizon}", + transitions.len() + ))); + } + Ok(WarmStartExactJhTeacherTrace { transitions }) +} + +/// Parse a normalized planner JSONL trace into a warm-start teacher trace. +pub fn warmstart_teacher_trace_from_jsonl_reader( + reader: R, + contract: &WarmStartExactJhTeacherContract, + return_horizon: usize, +) -> Result { + let mut steps = BTreeMap::::new(); + for (line_index, line) in reader.lines().enumerate() { + let line_number = line_index + 1; + let line = line.map_err(|err| invalid_telemetry(format!("line {line_number}: {err}")))?; + if line.trim().is_empty() { + return Err(invalid_telemetry(format!( + "line {line_number}: empty JSONL records are not allowed" + ))); + } + let value = serde_json::from_str::(&line) + .map_err(|err| invalid_telemetry(format!("line {line_number}: invalid JSON: {err}")))?; + let object = value.as_object().ok_or_else(|| { + invalid_telemetry(format!("line {line_number}: record must be an object")) + })?; + let kind = value + .get("kind") + .and_then(Value::as_str) + .ok_or_else(|| invalid_telemetry(format!("line {line_number}: missing kind")))?; + let step = usize::try_from(jsonl_required_u64(&value, "t", line_number)?) + .map_err(|_| invalid_telemetry(format!("line {line_number}: t exceeds usize")))?; + match kind { + "action" => { + ensure_jsonl_fields(object, &["kind", "t", "action", "provenance"], line_number)?; + if let Some(provenance_value) = value.get("provenance") { + let provenance = provenance_value.as_str().ok_or_else(|| { + invalid_telemetry(format!( + "line {line_number}: action provenance must be a string" + )) + })?; + PlannerActionProvenance::from_jsonl_str(provenance)?; + } + let action = jsonl_required_u64(&value, "action", line_number)?; + let entry = steps.entry(step).or_default(); + if entry + .action + .replace(JsonlActionRecord { + action, + line_number, + }) + .is_some() + { + return Err(invalid_telemetry(format!( + "duplicate action record for step {step}" + ))); + } + } + "percept" => { + ensure_jsonl_fields( + object, + &["kind", "t", "observations", "reward"], + line_number, + )?; + let observations = jsonl_required_observations(&value, line_number)?; + let reward = jsonl_required_i64(&value, "reward", line_number)?; + let entry = steps.entry(step).or_default(); + if entry + .percept + .replace(JsonlPerceptRecord { + observations, + reward, + line_number, + }) + .is_some() + { + return Err(invalid_telemetry(format!( + "duplicate percept record for step {step}" + ))); + } + } + other => { + return Err(invalid_telemetry(format!( + "line {line_number}: unknown record kind '{other}'" + ))); + } + } + } + jsonl_steps_into_teacher_trace(steps, contract, return_horizon) +} + +/// Parse a normalized planner JSONL trace from bytes. +pub fn warmstart_teacher_trace_from_jsonl_slice( + bytes: &[u8], + contract: &WarmStartExactJhTeacherContract, + return_horizon: usize, +) -> Result { + warmstart_teacher_trace_from_jsonl_reader(BufReader::new(bytes), contract, return_horizon) +} + +/// Parse a normalized planner JSONL trace from a filesystem path. +pub fn warmstart_teacher_trace_from_jsonl_path( + path: impl AsRef, + contract: &WarmStartExactJhTeacherContract, + return_horizon: usize, +) -> Result { + let file = File::open(path.as_ref()).map_err(|err| { + invalid_telemetry(format!( + "failed to open JSONL trace '{}': {err}", + path.as_ref().display() + )) + })?; + warmstart_teacher_trace_from_jsonl_reader(BufReader::new(file), contract, return_horizon) +} + +/// Build the standalone same-task teacher contract for a compiled warm-start planner run. +pub fn standalone_warmstart_teacher_contract_for_compiled_planner_run( + compiled: &CompiledPlannerRunSpec, +) -> Result { + let interface = compiled.interface(); + let (return_horizon, label_phase_period, planner_simulations_per_step) = + match compiled.controller() { + CompiledPlannerController::AiqiWarmstartExactJh { + return_horizon, + label_phase_period, + planner_simulations_per_step, + .. + } => ( + *return_horizon, + *label_phase_period, + *planner_simulations_per_step, + ), + _ => return Err(WarmStartExactJhError::ControllerKindMismatch), + }; + if planner_simulations_per_step == 0 { + return Err(WarmStartExactJhError::PlannerSimulationsZero); + } + if planner_simulations_per_step != 1 { + return Err(WarmStartExactJhError::PlannerSimulationsUnsupported { + configured: planner_simulations_per_step, + }); + } + let task_fingerprint = + warmstart_exact_jh_planner_task_fingerprint(compiled).map_err(|err| { + WarmStartExactJhError::InvalidTeacherDataset { + reason: format!("failed to compute planner task fingerprint: {err}"), + } + })?; + let (observation_adapter_content_crc32, exact_reward_encoding_certificate) = + crate::aixi::warmstart_contract::standalone_teacher_provenance_crc32_pair( + interface.observation_bits, + interface.observation_stream_len.max(1), + interface.reward_bits, + ) + .map_err(|err| WarmStartExactJhError::InvalidTeacherDataset { + reason: format!("failed to compute standalone teacher provenance: {err}"), + })?; + Ok(WarmStartExactJhTeacherContract { + schema_version: WARMSTART_TEACHER_CONTRACT_SCHEMA_VERSION, + task_fingerprint, + action_alphabet_size: interface.agent_actions.get(), + observation_bits: interface.observation_bits, + observation_stream_len: interface.observation_stream_len.max(1), + observation_key_mode: observation_key_mode_name(interface.observation_key_mode).to_string(), + observation_adapter_spec_ref: WARMSTART_STANDALONE_OBSERVATION_ADAPTER_SPEC_REF.to_string(), + observation_adapter_content_crc32, + reward_bits: interface.reward_bits, + return_horizon, + label_phase_period, + scalar_representation: WARMSTART_STANDALONE_SCALAR_REPRESENTATION.to_string(), + exact_reward_encoding_certificate, + }) +} + +/// Return the target exact-return horizon for a compiled warm-start planner run. +pub fn warmstart_target_return_horizon( + compiled: &CompiledPlannerRunSpec, +) -> Result { + match compiled.controller() { + CompiledPlannerController::AiqiWarmstartExactJh { return_horizon, .. } => { + Ok(*return_horizon) + } + _ => Err(WarmStartExactJhError::ControllerKindMismatch), + } +} + +/// Read a warm-start teacher dataset from a filesystem path. +pub fn read_warmstart_teacher_dataset_path( + path: impl AsRef, +) -> Result { + let path_ref = path.as_ref(); + let bytes = + std::fs::read(path_ref).map_err(|err| WarmStartExactJhError::InvalidTeacherDataset { + reason: format!( + "failed to read teacher dataset '{}': {err}", + path_ref.display() + ), + })?; + WarmStartExactJhTeacherDataset::from_json_slice(&bytes) +} + +/// Write a warm-start teacher dataset to a filesystem path. +pub fn write_warmstart_teacher_dataset_path( + path: impl AsRef, + dataset: &WarmStartExactJhTeacherDataset, +) -> Result<(), WarmStartExactJhError> { + let path_ref = path.as_ref(); + let bytes = serde_json::to_vec_pretty(&dataset.to_json_value()).map_err(|err| { + WarmStartExactJhError::InvalidTeacherDataset { + reason: format!("failed to serialize teacher dataset: {err}"), + } + })?; + std::fs::write(path_ref, bytes).map_err(|err| WarmStartExactJhError::InvalidTeacherDataset { + reason: format!( + "failed to write teacher dataset '{}': {err}", + path_ref.display() + ), + }) +} + +/// Configuration parameters for the warm-start exact-J_H controller. +#[derive(Clone)] +#[non_exhaustive] +pub struct WarmStartExactJhConfig { + /// Predictive backend used by the return-label model. + pub rate_backend: RateBackend, + /// Number of bits used to encode observations. + pub observation_bits: usize, + /// Number of observation symbols per environment step. + pub observation_stream_len: usize, + /// Number of bits used to encode rewards. + pub reward_bits: usize, + /// Number of valid actions. + pub agent_actions: ActionAlphabet, + /// Exact finite-horizon return length H. + pub return_horizon: usize, + /// Exact return-label alphabet cardinality. + /// + /// For this nonnegative-reward controller the valid shape is + /// `return_horizon * max_reward + 1`; slack labels are rejected because + /// they would not correspond to any exact finite-horizon return. + pub return_bins: usize, + /// Delayed-label phase period. + pub label_phase_period: usize, + /// Canonical direct-evaluator budget marker. + /// + /// Warm-start exact-\(J_H\) uses deterministic full return-law evaluation, + /// not a simulation loop. The only truthful value is therefore `1`; larger + /// values are rejected instead of being silently ignored. + pub planner_simulations_per_step: usize, + /// Bit-stream semantics used to adapt the return-label predictor. + pub bit_stream_semantics: BitStreamSemantics, + /// Optional deterministic RNG seed. + pub random_seed: Option, +} + +impl Default for WarmStartExactJhConfig { + fn default() -> Self { + Self { + rate_backend: RateBackend::Ctw { depth: 8 }, + observation_bits: 1, + observation_stream_len: 1, + reward_bits: 1, + agent_actions: ActionAlphabet::try_from_usize(2) + .expect("default action alphabet must be non-zero"), + return_horizon: 1, + return_bins: 2, + label_phase_period: 1, + planner_simulations_per_step: 1, + bit_stream_semantics: BitStreamSemantics::BinaryTokens, + random_seed: None, + } + } +} + +impl WarmStartExactJhConfig { + fn canonical_planner_run_spec(&self) -> PlannerRunSpec { + let mut spec = build_default_planner_run_spec( + PlannerInterfaceConfig { + observation_bits: self.observation_bits, + observation_stream_len: self.observation_stream_len, + observation_key_mode: crate::aixi::common::ObservationKeyMode::FullStream, + reward_bits: self.reward_bits, + agent_actions: self.agent_actions, + }, + ControllerSpec::AiqiWarmstartExactJh(WarmStartExactJhControllerSpec { + predictor: self.rate_backend.clone(), + bit_stream_semantics: self.bit_stream_semantics, + return_horizon: self.return_horizon, + return_bins: self.return_bins, + label_phase_period: self.label_phase_period, + teacher_dataset_asset: "programmatic_warmstart_teacher".to_string(), + planner_simulations_per_step: self.planner_simulations_per_step, + }), + self.random_seed, + ); + spec.assets.push(AssetBinding { + id: "programmatic_warmstart_teacher".to_string(), + path: "programmatic_warmstart_teacher.json".to_string(), + }); + spec + } + + fn compile_planner_run_spec(&self) -> Result { + self.canonical_planner_run_spec() + .compile() + .map_err(WarmStartExactJhError::from) + } + + fn validate_runtime_invariants(&self) -> Result<(), WarmStartExactJhError> { + if self.return_horizon == 0 { + return Err(WarmStartExactJhError::ReturnHorizonZero); + } + if self.return_bins == 0 { + return Err(WarmStartExactJhError::ReturnBinsZero); + } + if self.label_phase_period < self.return_horizon { + return Err(WarmStartExactJhError::LabelPhasePeriodTooShort { + label_phase_period: self.label_phase_period, + return_horizon: self.return_horizon, + }); + } + if self.planner_simulations_per_step == 0 { + return Err(WarmStartExactJhError::PlannerSimulationsZero); + } + if self.planner_simulations_per_step != 1 { + return Err(WarmStartExactJhError::PlannerSimulationsUnsupported { + configured: self.planner_simulations_per_step, + }); + } + let return_horizon = NonZeroUsize::new(self.return_horizon) + .ok_or(WarmStartExactJhError::ReturnHorizonZero)?; + let return_bins = + NonZeroUsize::new(self.return_bins).ok_or(WarmStartExactJhError::ReturnBinsZero)?; + let (min_reward, max_reward, _reward_offset) = + reward_bounds_from_exact_return_bins(return_horizon, return_bins, self.reward_bits)?; + validate_exact_return_alphabet( + min_reward, + max_reward, + self.return_horizon, + self.return_bins, + )?; + validate_rate_backend(&self.rate_backend) + .map_err(WarmStartExactJhError::InvalidRateBackend)?; + let compiled = self + .rate_backend + .compile() + .map_err(WarmStartExactJhError::Spec)?; + if !compiled.supports_frozen_conditioning() { + return Err(WarmStartExactJhError::UnsupportedRateBackend { + reason: "warm-start exact-J_H strict mode requires frozen context conditioning; configured rate_backend does not provide strict frozen conditioning", + }); + } + Ok(()) + } + + /// Validate this configuration. + pub fn validate(&self) -> Result<(), WarmStartExactJhError> { + self.validate_runtime_invariants()?; + self.compile_planner_run_spec().map(|_| ()) + } +} + +#[derive(Clone)] +struct WarmStartExactJhRuntimeConfig { + task_fingerprint: TaskFingerprint, + observation_bits: usize, + observation_stream_len: usize, + observation_key_mode: &'static str, + reward_bits: usize, + agent_actions: ActionAlphabet, + min_reward: Reward, + max_reward: Reward, + reward_offset: Reward, + return_horizon: usize, + return_bins: usize, + label_phase_period: usize, + planner_simulations_per_step: usize, + random_seed: u64, + provenance_policy: TeacherProvenancePolicy, +} + +#[derive(Clone, Copy)] +enum TeacherProvenancePolicy { + StandalonePlannerRun, + ExternallyValidatedTunerBridge, +} + +impl WarmStartExactJhRuntimeConfig { + /// Build a canonicalized runtime contract from a compiled planner run. + /// + /// The resulting config captures the exact planner/interface contract that + /// teacher traces are expected to match. This includes the planner task + /// fingerprint plus runtime-visible interface dimensions that must remain + /// aligned with any warm-start dataset. + fn from_compiled(compiled: &CompiledPlannerRunSpec) -> Result { + let planner = compiled.canonical_spec(); + let interface = compiled.interface(); + let runtime = compiled.runtime(); + let (return_horizon, return_bins, label_phase_period, planner_simulations_per_step) = + match compiled.controller() { + CompiledPlannerController::AiqiWarmstartExactJh { + return_horizon, + return_bins, + label_phase_period, + planner_simulations_per_step, + .. + } => ( + *return_horizon, + *return_bins, + *label_phase_period, + *planner_simulations_per_step, + ), + _ => return Err(WarmStartExactJhError::ControllerKindMismatch), + }; + if return_horizon == 0 { + return Err(WarmStartExactJhError::ReturnHorizonZero); + } + if return_bins == 0 { + return Err(WarmStartExactJhError::ReturnBinsZero); + } + if label_phase_period < return_horizon { + return Err(WarmStartExactJhError::LabelPhasePeriodTooShort { + label_phase_period, + return_horizon, + }); + } + if planner_simulations_per_step == 0 { + return Err(WarmStartExactJhError::PlannerSimulationsZero); + } + if planner_simulations_per_step != 1 { + return Err(WarmStartExactJhError::PlannerSimulationsUnsupported { + configured: planner_simulations_per_step, + }); + } + let nonzero_return_horizon = + NonZeroUsize::new(return_horizon).ok_or(WarmStartExactJhError::ReturnHorizonZero)?; + let nonzero_return_bins = + NonZeroUsize::new(return_bins).ok_or(WarmStartExactJhError::ReturnBinsZero)?; + let (min_reward, max_reward, reward_offset) = reward_bounds_from_exact_return_bins( + nonzero_return_horizon, + nonzero_return_bins, + interface.reward_bits, + )?; + validate_exact_return_alphabet(min_reward, max_reward, return_horizon, return_bins)?; + let task_fingerprint = + warmstart_exact_jh_planner_task_fingerprint(compiled).map_err(|err| { + WarmStartExactJhError::InvalidTeacherDataset { + reason: format!("failed to compute planner task fingerprint: {err}"), + } + })?; + let provenance_policy = match &planner.environment { + EnvironmentSpec::Builtin { + builtin: BuiltinEnvironmentSpec::TunerBridge, + } => TeacherProvenancePolicy::ExternallyValidatedTunerBridge, + _ => TeacherProvenancePolicy::StandalonePlannerRun, + }; + Ok(Self { + task_fingerprint, + observation_bits: interface.observation_bits, + observation_stream_len: interface.observation_stream_len.max(1), + observation_key_mode: observation_key_mode_name(interface.observation_key_mode), + reward_bits: interface.reward_bits, + agent_actions: interface.agent_actions, + min_reward, + max_reward, + reward_offset, + return_horizon, + return_bins, + label_phase_period, + planner_simulations_per_step, + random_seed: resolve_random_seed(runtime.random_seed), + provenance_policy, + }) + } + + /// Validate that a parsed teacher contract matches the active planner runtime + /// contract for this agent configuration. + /// + /// This comparison is authoritative for schema/contract compatibility and is + /// intentionally strict on planner-task fields that influence trace encoding. + fn validate_teacher_contract( + &self, + contract: &WarmStartExactJhTeacherContract, + ) -> Result<(), WarmStartExactJhError> { + validate_warmstart_teacher_contract_against_expectation( + contract, + &WarmStartTeacherContractExpectation { + schema_version: WARMSTART_TEACHER_CONTRACT_SCHEMA_VERSION, + task_fingerprint: self.task_fingerprint, + action_alphabet_size: self.agent_actions.get(), + observation_bits: self.observation_bits, + observation_stream_len: self.observation_stream_len, + observation_key_mode: self.observation_key_mode, + reward_bits: self.reward_bits, + return_horizon: self.return_horizon, + label_phase_period: self.label_phase_period, + validate_standalone_provenance: matches!( + self.provenance_policy, + TeacherProvenancePolicy::StandalonePlannerRun + ), + }, + ) + } +} + +pub(crate) fn reward_bounds_from_exact_return_bins( + return_horizon: NonZeroUsize, + return_bins: NonZeroUsize, + reward_bits: usize, +) -> Result<(Reward, Reward, Reward), WarmStartExactJhError> { + let max_reward = max_reward_from_exact_return_bins(return_horizon, return_bins)?; + validate_reward_encoding_bounds(0, max_reward, 0, reward_bits)?; + Ok((0, max_reward, 0)) +} + +pub(crate) fn max_reward_from_exact_return_bins( + return_horizon: NonZeroUsize, + return_bins: NonZeroUsize, +) -> Result { + let return_horizon = return_horizon.get(); + let return_bins = return_bins.get(); + let span = return_bins - 1; + if !span.is_multiple_of(return_horizon) { + return Err(WarmStartExactJhError::ReturnBinsNotExactHorizon { + return_bins, + return_horizon, + }); + } + let max_reward = i64::try_from(span / return_horizon) + .map_err(|_| WarmStartExactJhError::ExactReturnRangeOverflow)?; + Ok(max_reward) +} + +#[derive(Clone, Debug)] +struct StepRecord { + action: Action, + observations: Vec, + reward: Reward, +} + +struct PhaseModel { + predictor: Box, + last_augmented_step: usize, +} + +/// Warm-start exact finite-horizon objective controller. +pub struct WarmStartExactJhAgent { + config: WarmStartExactJhRuntimeConfig, + phases: Vec, + steps: Vec, + return_labels_by_step: Vec>, + total_steps_observed: usize, + action_bits: usize, + return_label_codec: ReturnLabelCodec, + teacher_label_count: usize, + rng: RandomGenerator, +} + +impl WarmStartExactJhAgent { + /// Construct a new warm-start exact-J_H agent. + pub fn new( + config: WarmStartExactJhConfig, + teacher: WarmStartExactJhTeacherDataset, + ) -> Result { + config.validate_runtime_invariants()?; + let compiled = config.compile_planner_run_spec()?; + Self::from_compiled_planner_run(&compiled, teacher) + } + + /// Construct from a compiled planner-run spec and same-task teacher data. + pub fn from_compiled_planner_run( + compiled: &CompiledPlannerRunSpec, + teacher: WarmStartExactJhTeacherDataset, + ) -> Result { + // Validate contract mismatch upfront so invalid metadata fails before costly + // predictor allocation and before trace replay. + let config = WarmStartExactJhRuntimeConfig::from_compiled(compiled)?; + config.validate_teacher_contract(&teacher.contract)?; + let (predictor, bit_stream_semantics) = match compiled.controller() { + CompiledPlannerController::AiqiWarmstartExactJh { + predictor, + bit_stream_semantics, + .. + } => (predictor, *bit_stream_semantics), + _ => return Err(WarmStartExactJhError::ControllerKindMismatch), + }; + if !predictor.supports_frozen_conditioning() { + return Err(WarmStartExactJhError::UnsupportedRateBackend { + reason: "warm-start exact-J_H strict mode requires frozen context conditioning; configured rate_backend does not provide strict frozen conditioning", + }); + } + let action_bits = compiled.action_bits(); + let return_label_codec = ReturnLabelCodec::value_monotone(config.return_bins); + let return_bits = return_label_codec.bits(); + let mut phases = Vec::with_capacity(config.label_phase_period); + for _ in 0..config.label_phase_period { + phases.push(PhaseModel { + predictor: build_aiqi_predictor(predictor, return_bits, bit_stream_semantics) + .map_err(WarmStartExactJhError::Predictor)?, + last_augmented_step: 0, + }); + } + let rng = RandomGenerator::from_seed(config.random_seed); + let mut agent = Self { + action_bits, + return_label_codec, + phases, + steps: Vec::new(), + return_labels_by_step: Vec::new(), + total_steps_observed: 0, + teacher_label_count: 0, + rng, + config, + }; + agent.warm_start_from_teacher(&teacher)?; + Ok(agent) + } + + /// Number of transitions incorporated from live interaction. + pub fn steps_observed(&self) -> usize { + self.total_steps_observed + } + + /// Number of warm-start labels incorporated from the teacher dataset. + pub fn teacher_label_count(&self) -> usize { + self.teacher_label_count + } + + /// Extract the current live same-task trajectory as a teacher trace. + /// + /// The trace is admissible under the same runtime validator used for + /// teacher datasets because it was produced through `observe_transition`. + /// + /// Cost: despite the `&self` receiver this is a full materialization, not a + /// cheap accessor. It allocates a fresh transition vector and clones every + /// stored observation stream, so a call is `O(steps * observation_stream_len)` + /// in both time and allocated memory. It is intended for occasional + /// refresh/export points; callers in a hot loop should cache the result + /// rather than re-deriving it per step. + pub fn same_task_live_trace(&self) -> Option { + if self.steps.len() < self.config.return_horizon { + return None; + } + Some(WarmStartExactJhTeacherTrace { + transitions: self + .steps + .iter() + .map(|step| WarmStartExactJhTransition { + action: step.action, + observations: step.observations.clone(), + reward: step.reward, + }) + .collect(), + }) + } + + /// Configured action alphabet cardinality. + pub fn num_actions(&self) -> ActionAlphabet { + self.config.agent_actions + } + + /// Canonical direct-evaluator budget marker. + /// + /// This is always `1`; warm-start exact-\(J_H\) has no simulation loop. + pub fn planner_simulations_per_step(&self) -> usize { + self.config.planner_simulations_per_step + } + + /// Resolved deterministic seed. + pub fn resolved_random_seed(&self) -> u64 { + self.config.random_seed + } + + pub(crate) fn reseed_random(&mut self, seed: u64) { + self.config.random_seed = seed; + self.rng = RandomGenerator::from_seed(seed); + } + + /// Select the next greedy action from the current exact-return model. + /// + /// # Panics + /// + /// Panics if retained live history violates the validated warm-start + /// runtime contract. Use [`Self::try_get_planned_action`] to receive that + /// condition as [`WarmStartExactJhError`]. + pub fn get_planned_action(&mut self) -> Action { + self.try_get_planned_action() + .expect("warm-start planning state must satisfy validated history invariants") + } + + /// Fallibly select the next greedy action from the current exact-return model. + pub fn try_get_planned_action(&mut self) -> Result { + let q_values = self.estimate_q_values()?; + Ok(argmax_with_fixed_tie_break(&q_values) as u64) + } + + /// Estimate exact finite-horizon action values at the current decision state. + pub fn estimate_action_values(&mut self) -> Result, WarmStartExactJhError> { + self.estimate_q_values() + } + + /// Select the next action with optional epsilon exploration. + /// + /// Warm-start exact-\(J_H\) has no baseline exploration parameter; this + /// method's argument is the entire exploration probability. + /// + /// # Panics + /// + /// Panics if greedy planning is reached and retained live history violates + /// the validated warm-start runtime contract. Use + /// [`Self::try_get_planned_action_with_extra_exploration_flag`] to receive + /// that condition as [`WarmStartExactJhError`]. + pub fn get_planned_action_with_extra_exploration(&mut self, extra_exploration: f64) -> Action { + self.get_planned_action_with_extra_exploration_flag(extra_exploration) + .0 + } + + /// Select the next action with optional epsilon exploration and return whether exploration fired. + /// + /// Warm-start exact-\(J_H\) has no baseline exploration parameter; this + /// method's argument is the entire exploration probability. + /// + /// # Panics + /// + /// Panics if greedy planning is reached and retained live history violates + /// the validated warm-start runtime contract. Use + /// [`Self::try_get_planned_action_with_extra_exploration_flag`] to receive + /// that condition as [`WarmStartExactJhError`]. + pub fn get_planned_action_with_extra_exploration_flag( + &mut self, + extra_exploration: f64, + ) -> (Action, bool) { + self.try_get_planned_action_with_extra_exploration_flag(extra_exploration) + .expect("warm-start planning state must satisfy validated history invariants") + } + + /// Fallibly select the next action with optional epsilon exploration and return whether exploration fired. + /// + /// Warm-start exact-\(J_H\) has no baseline exploration parameter; this + /// method's argument is the entire exploration probability. + pub fn try_get_planned_action_with_extra_exploration_flag( + &mut self, + extra_exploration: f64, + ) -> Result<(Action, bool), WarmStartExactJhError> { + let extra = extra_exploration.clamp(0.0, 1.0); + if extra > 0.0 && self.rng.gen_bool(extra) { + Ok(( + self.rng.gen_range(self.config.agent_actions.get()) as u64, + true, + )) + } else { + Ok((self.try_get_planned_action()?, false)) + } + } + + /// Record one live environment transition. + pub fn observe_transition( + &mut self, + action: Action, + observations: &[PerceptVal], + reward: Reward, + ) -> Result<(), WarmStartExactJhError> { + self.validate_transition(action, observations, reward)?; + self.steps.push(StepRecord { + action, + observations: observations.to_vec(), + reward, + }); + self.total_steps_observed = self.total_steps_observed.saturating_add(1); + self.return_labels_by_step.push(None); + self.maybe_learn_new_return() + } + + /// Warm-start from teacher traces (trace payload validation only). + /// + /// Contract-level validation is performed before this method is called in the + /// constructor hot path; this keeps construction cheap on malformed contracts. + fn warm_start_from_teacher( + &mut self, + teacher: &WarmStartExactJhTeacherDataset, + ) -> Result<(), WarmStartExactJhError> { + let mut label_count = 0usize; + for trace in &teacher.traces { + self.validate_teacher_trace(trace)?; + label_count = label_count.saturating_add(self.commit_teacher_trace(trace)?); + for phase in &mut self.phases { + phase + .predictor + .reset_conditioning_history() + .map_err(|reason| WarmStartExactJhError::PredictorConditioningReset { + reason, + })?; + } + } + if label_count == 0 { + return Err(WarmStartExactJhError::InvalidTeacherDataset { + reason: "teacher dataset did not contain any complete H-step labels".to_string(), + }); + } + self.teacher_label_count = label_count; + Ok(()) + } + + fn validate_teacher_trace( + &self, + trace: &WarmStartExactJhTeacherTrace, + ) -> Result<(), WarmStartExactJhError> { + for step in &trace.transitions { + self.validate_transition(step.action, &step.observations, step.reward)?; + } + Ok(()) + } + + fn validate_transition( + &self, + action: Action, + observations: &[PerceptVal], + reward: Reward, + ) -> Result<(), WarmStartExactJhError> { + validate_runtime_transition(&self.config, action, observations, reward) + } + + fn commit_teacher_trace( + &mut self, + trace: &WarmStartExactJhTeacherTrace, + ) -> Result { + let labels = exact_return_labels_for_trace(&self.config, &trace.transitions)?; + let mut committed = 0usize; + for phase in 0..self.config.label_phase_period { + let model = &mut self.phases[phase]; + for (idx0, step) in trace.transitions.iter().enumerate() { + let step_index = idx0 + 1; + push_action_tokens_commit_history( + model.predictor.as_mut(), + step.action, + self.action_bits, + ); + if step_index % self.config.label_phase_period == phase + && let Some(label) = labels[idx0] + { + self.return_label_codec + .push_label_commit(model.predictor.as_mut(), label); + committed = committed.saturating_add(1); + } + push_percept_tokens_commit_history( + &self.config, + model.predictor.as_mut(), + &step.observations, + step.reward, + )?; + } + } + Ok(committed) + } + + fn maybe_learn_new_return(&mut self) -> Result<(), WarmStartExactJhError> { + let t = self.total_steps_observed; + let h = self.config.return_horizon; + if t < h { + return Ok(()); + } + let start_step = t + 1 - h; + let label = self.compute_return_label(start_step)?; + self.return_labels_by_step[start_step - 1] = Some(label); + let phase = start_step % self.config.label_phase_period; + self.advance_phase_model_to_step(phase, start_step) + } + + fn estimate_q_values(&mut self) -> Result, WarmStartExactJhError> { + let min_return = (self.config.min_reward as i128) + .checked_mul(self.config.return_horizon as i128) + .ok_or(WarmStartExactJhError::ExactReturnRangeOverflow)? + as f64; + let codec = self.return_label_codec; + let step = self.total_steps_observed + 1; + let phase = step % self.config.label_phase_period; + let live_config = &self.config; + let steps = &self.steps; + let return_labels_by_step = &self.return_labels_by_step; + let action_bits = self.action_bits; + let token_ctx = WarmStartAugmentedTokenContext { + config: live_config, + steps, + return_labels_by_step, + action_bits, + return_label_codec: self.return_label_codec, + phase, + }; + let mut q_values = Vec::with_capacity(self.config.agent_actions.get()); + let mut pushed_history = 0usize; + { + let model = &mut self.phases[phase]; + let start = model.last_augmented_step + 1; + let end = step.saturating_sub(1); + if start <= end { + for idx in start..=end { + match push_step_tokens_history(&token_ctx, model.predictor.as_mut(), idx) { + Ok(pushed) => { + pushed_history += pushed; + } + Err(err) => { + pop_history_bits(model.predictor.as_mut(), pushed_history); + return Err(err); + } + } + } + } + for action in 0..self.config.agent_actions.get() { + let pushed_action = + push_encoded_bits_history(model.predictor.as_mut(), action as u64, action_bits); + let expected_label = predict_expected_label( + model.predictor.as_mut(), + codec, + ReturnPrefixUpdate::Training, + ReturnLawEvaluator::SharedPrefix, + ); + pop_history_bits(model.predictor.as_mut(), pushed_action); + q_values.push(min_return + expected_label); + } + pop_history_bits(model.predictor.as_mut(), pushed_history); + } + Ok(q_values) + } + + fn advance_phase_model_to_step( + &mut self, + phase: usize, + target_step: usize, + ) -> Result<(), WarmStartExactJhError> { + let token_ctx = WarmStartAugmentedTokenContext { + config: &self.config, + steps: &self.steps, + return_labels_by_step: &self.return_labels_by_step, + action_bits: self.action_bits, + return_label_codec: self.return_label_codec, + phase, + }; + let model = &mut self.phases[phase]; + if target_step <= model.last_augmented_step { + return Ok(()); + } + let start = model.last_augmented_step + 1; + for idx in start..=target_step { + push_augmented_step_tokens_commit(&token_ctx, model.predictor.as_mut(), idx)?; + } + model.last_augmented_step = target_step; + Ok(()) + } + + fn compute_return_label(&self, start_step: usize) -> Result { + let mut total = 0i128; + for offset in 0..self.config.return_horizon { + let idx = start_step + offset; + let step = + self.steps + .get(idx - 1) + .ok_or(WarmStartExactJhError::HistoryIndexOutOfRange { + global_step: idx, + total_steps_observed: self.total_steps_observed, + })?; + total += step.reward as i128; + } + label_for_exact_return(&self.config, total) + } +} + +/// Error type for warm-start exact-J_H agent construction and execution. +#[derive(Debug)] +#[non_exhaustive] +pub enum WarmStartExactJhError { + /// The compiled planner controller was not a warm-start exact-J_H controller. + ControllerKindMismatch, + /// The return horizon was zero. + ReturnHorizonZero, + /// The return-label alphabet was empty. + ReturnBinsZero, + /// The label phase period was smaller than the return horizon. + LabelPhasePeriodTooShort { + /// Configured label phase period. + label_phase_period: usize, + /// Configured return horizon. + return_horizon: usize, + }, + /// The direct-evaluator budget marker was zero. + PlannerSimulationsZero, + /// The direct-evaluator budget marker was not the canonical value. + PlannerSimulationsUnsupported { + /// Configured unsupported value. + configured: usize, + }, + /// The exact return range cannot be represented by `return_bins`. + ReturnBinsTooSmall { + /// Required exact labels. + required: u128, + /// Configured labels. + configured: usize, + }, + /// `return_bins` would leave unreachable exact-return labels. + ReturnBinsNotExactHorizon { + /// Configured label count. + return_bins: usize, + /// Configured return horizon. + return_horizon: usize, + }, + /// The exact return range overflowed the supported integer domain. + ExactReturnRangeOverflow, + /// The configured reward range is not representable. + RewardEncoding(RewardEncodingError), + /// Invalid rate backend. + InvalidRateBackend(crate::error::InfotheoryError), + /// Unsupported rate backend semantics. + UnsupportedRateBackend { + /// Human-readable reason. + reason: &'static str, + }, + /// Spec compilation failed. + Spec(SpecError), + /// Predictor construction failed. + Predictor(PredictorBuildError), + /// Predictor conditioning history reset failed. + PredictorConditioningReset { + /// Human-readable reason. + reason: String, + }, + /// Teacher dataset was malformed or semantically inadmissible. + InvalidTeacherDataset { + /// Human-readable reason. + reason: String, + }, + /// JSONL telemetry was malformed. + InvalidTelemetry { + /// Human-readable reason. + reason: String, + }, + /// Action outside the configured alphabet. + ActionOutOfRange { + /// Invalid action. + action: Action, + /// Configured alphabet. + agent_actions: ActionAlphabet, + }, + /// Observation stream length mismatch. + ObservationStreamLengthMismatch { + /// Expected stream length. + expected: usize, + /// Actual stream length. + actual: usize, + }, + /// Observation value exceeded its bit width. + ObservationValueOutOfRange { + /// Invalid observation. + observation: PerceptVal, + /// Configured observation bits. + observation_bits: usize, + /// Maximum representable value. + maximum: PerceptVal, + }, + /// Reward outside the configured exact reward range. + RewardOutOfRange { + /// Invalid reward. + reward: Reward, + /// Minimum reward. + min_reward: Reward, + /// Maximum reward. + max_reward: Reward, + }, + /// Live history index was unavailable. + HistoryIndexOutOfRange { + /// Requested 1-based step index. + global_step: usize, + /// Total observed steps. + total_steps_observed: usize, + }, + /// A delayed label was required but absent. + MissingReturnLabel { + /// Step index. + step: usize, + /// Phase index. + phase: usize, + }, +} + +impl fmt::Display for WarmStartExactJhError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::ControllerKindMismatch => { + f.write_str("compiled controller kind is not aiqi_warmstart_exact_jh") + } + Self::ReturnHorizonZero => f.write_str("return_horizon must be >= 1"), + Self::ReturnBinsZero => f.write_str("return_bins must be >= 1"), + Self::LabelPhasePeriodTooShort { + label_phase_period, + return_horizon, + } => write!( + f, + "label_phase_period ({label_phase_period}) must be >= return_horizon ({return_horizon})" + ), + Self::PlannerSimulationsZero => f.write_str( + "planner_simulations_per_step must be exactly 1 for warm-start exact-J_H direct evaluation", + ), + Self::PlannerSimulationsUnsupported { configured } => write!( + f, + "planner_simulations_per_step must be exactly 1 for warm-start exact-J_H direct evaluation, got {configured}" + ), + Self::ReturnBinsTooSmall { + required, + configured, + } => write!( + f, + "return_bins too small for exact J_H labels: required {required}, configured {configured}" + ), + Self::ReturnBinsNotExactHorizon { + return_bins, + return_horizon, + } => write!( + f, + "return_bins must be exactly H * max_reward + 1 for warm-start exact-J_H; got return_bins={return_bins}, return_horizon={return_horizon}" + ), + Self::ExactReturnRangeOverflow => { + f.write_str("exact finite-horizon return range overflowed supported integer domain") + } + Self::RewardEncoding(err) => write!(f, "{err}"), + Self::InvalidRateBackend(err) => write!(f, "invalid rate_backend: {err}"), + Self::UnsupportedRateBackend { reason } => f.write_str(reason), + Self::Spec(err) => write!(f, "{err}"), + Self::Predictor(err) => write!(f, "failed to construct predictor: {err}"), + Self::PredictorConditioningReset { reason } => { + write!( + f, + "failed to reset predictor conditioning history: {reason}" + ) + } + Self::InvalidTeacherDataset { reason } => { + write!(f, "invalid teacher dataset: {reason}") + } + Self::InvalidTelemetry { reason } => write!(f, "invalid telemetry: {reason}"), + Self::ActionOutOfRange { + action, + agent_actions, + } => write!( + f, + "action {action} is outside configured action alphabet {agent_actions}" + ), + Self::ObservationStreamLengthMismatch { expected, actual } => write!( + f, + "observation stream length mismatch: expected {expected}, got {actual}" + ), + Self::ObservationValueOutOfRange { + observation, + observation_bits, + maximum, + } => write!( + f, + "observation value {observation} does not fit observation_bits={observation_bits} (max={maximum})" + ), + Self::RewardOutOfRange { + reward, + min_reward, + max_reward, + } => write!( + f, + "reward {reward} outside configured range [{min_reward}, {max_reward}]" + ), + Self::HistoryIndexOutOfRange { + global_step, + total_steps_observed, + } => write!( + f, + "global step {global_step} out of observed history range [1, {total_steps_observed}]" + ), + Self::MissingReturnLabel { step, phase } => { + write!( + f, + "missing exact return label for step {step} in phase {phase}" + ) + } + } + } +} + +impl Error for WarmStartExactJhError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::RewardEncoding(err) => Some(err), + Self::InvalidRateBackend(err) => Some(err), + Self::Spec(err) => Some(err), + Self::Predictor(err) => Some(err), + Self::PredictorConditioningReset { .. } => None, + _ => None, + } + } +} + +impl From for WarmStartExactJhError { + fn from(value: RewardEncodingError) -> Self { + Self::RewardEncoding(value) + } +} + +impl From for WarmStartExactJhError { + fn from(value: SpecError) -> Self { + Self::Spec(value) + } +} + +fn parse_teacher_contract( + object: &serde_json::Map, + schema_version: u64, +) -> Result { + let contract = object + .get("contract") + .and_then(Value::as_object) + .ok_or_else(|| WarmStartExactJhError::InvalidTeacherDataset { + reason: "teacher dataset requires a 'contract' object".to_string(), + })?; + ensure_teacher_fields( + contract, + &[ + "task_fingerprint", + "action_alphabet_size", + "observation_bits", + "observation_stream_len", + "observation_key_mode", + "observation_adapter_spec_ref", + "observation_adapter_content_crc32", + "reward_bits", + "return_horizon", + "label_phase_period", + "scalar_representation", + "exact_reward_encoding_certificate", + ], + "contract", + )?; + Ok(WarmStartExactJhTeacherContract { + schema_version, + task_fingerprint: required_teacher_task_fingerprint(contract, "task_fingerprint")?, + action_alphabet_size: required_teacher_usize(contract, "action_alphabet_size")?, + observation_bits: required_teacher_usize(contract, "observation_bits")?, + observation_stream_len: required_teacher_usize(contract, "observation_stream_len")?, + observation_key_mode: required_teacher_string(contract, "observation_key_mode")?, + observation_adapter_spec_ref: required_teacher_string( + contract, + "observation_adapter_spec_ref", + )?, + observation_adapter_content_crc32: required_teacher_string( + contract, + "observation_adapter_content_crc32", + )?, + reward_bits: required_teacher_usize(contract, "reward_bits")?, + return_horizon: required_teacher_usize(contract, "return_horizon")?, + label_phase_period: required_teacher_usize(contract, "label_phase_period")?, + scalar_representation: required_teacher_string(contract, "scalar_representation")?, + exact_reward_encoding_certificate: required_teacher_string( + contract, + "exact_reward_encoding_certificate", + )?, + }) +} + +fn required_teacher_task_fingerprint( + object: &serde_json::Map, + field: &str, +) -> Result { + let value = object.get(field).and_then(Value::as_str).ok_or_else(|| { + WarmStartExactJhError::InvalidTeacherDataset { + reason: format!("teacher contract field '{field}' must be a string"), + } + })?; + TaskFingerprint::parse_hex(value).ok_or_else(|| WarmStartExactJhError::InvalidTeacherDataset { + reason: format!( + "teacher contract field '{field}' must be a 64-digit lowercase hexadecimal SHA-256 digest, got '{value}'" + ), + }) +} + +fn required_teacher_string( + object: &serde_json::Map, + field: &str, +) -> Result { + object + .get(field) + .and_then(Value::as_str) + .map(str::to_string) + .ok_or_else(|| WarmStartExactJhError::InvalidTeacherDataset { + reason: format!("teacher contract field '{field}' must be a string"), + }) +} + +fn required_teacher_usize( + object: &serde_json::Map, + field: &str, +) -> Result { + let value = object.get(field).and_then(Value::as_u64).ok_or_else(|| { + WarmStartExactJhError::InvalidTeacherDataset { + reason: format!("teacher contract field '{field}' must be an unsigned integer"), + } + })?; + usize::try_from(value).map_err(|_| WarmStartExactJhError::InvalidTeacherDataset { + reason: format!("teacher contract field '{field}' does not fit usize"), + }) +} + +fn parse_teacher_trace( + value: &Value, + trace_index: usize, +) -> Result { + let object = value + .as_object() + .ok_or_else(|| WarmStartExactJhError::InvalidTeacherDataset { + reason: format!("traces[{trace_index}] must be an object with transitions"), + })?; + ensure_teacher_fields(object, &["transitions"], &format!("traces[{trace_index}]"))?; + let transitions_value = object + .get("transitions") + .and_then(Value::as_array) + .ok_or_else(|| WarmStartExactJhError::InvalidTeacherDataset { + reason: format!("traces[{trace_index}].transitions must be an array"), + })?; + let mut transitions = Vec::with_capacity(transitions_value.len()); + for (step_index, transition) in transitions_value.iter().enumerate() { + transitions.push(parse_teacher_transition( + transition, + trace_index, + step_index, + )?); + } + Ok(WarmStartExactJhTeacherTrace { transitions }) +} + +fn parse_teacher_transition( + value: &Value, + trace_index: usize, + step_index: usize, +) -> Result { + let object = value + .as_object() + .ok_or_else(|| WarmStartExactJhError::InvalidTeacherDataset { + reason: format!("traces[{trace_index}].transitions[{step_index}] must be an object"), + })?; + ensure_teacher_fields( + object, + &["action", "observations", "reward"], + &format!("traces[{trace_index}].transitions[{step_index}]"), + )?; + let action = object + .get("action") + .and_then(Value::as_u64) + .ok_or_else(|| WarmStartExactJhError::InvalidTeacherDataset { + reason: format!( + "traces[{trace_index}].transitions[{step_index}].action must be an integer" + ), + })?; + let observations_value = + object + .get("observations") + .ok_or_else(|| WarmStartExactJhError::InvalidTeacherDataset { + reason: format!( + "traces[{trace_index}].transitions[{step_index}] requires observations" + ), + })?; + let observations = observations_value + .as_array() + .ok_or_else(|| WarmStartExactJhError::InvalidTeacherDataset { + reason: format!( + "traces[{trace_index}].transitions[{step_index}].observations must be an array" + ), + })? + .iter() + .enumerate() + .map(|(obs_index, obs)| { + obs.as_u64().ok_or_else(|| WarmStartExactJhError::InvalidTeacherDataset { + reason: format!( + "traces[{trace_index}].transitions[{step_index}].observations[{obs_index}] must be an integer" + ), + }) + }) + .collect::, _>>()?; + let reward = object + .get("reward") + .and_then(Value::as_i64) + .ok_or_else(|| WarmStartExactJhError::InvalidTeacherDataset { + reason: format!( + "traces[{trace_index}].transitions[{step_index}].reward must be an integer" + ), + })?; + Ok(WarmStartExactJhTransition { + action, + observations, + reward, + }) +} + +fn ensure_teacher_fields( + object: &serde_json::Map, + allowed: &[&str], + label: &str, +) -> Result<(), WarmStartExactJhError> { + for key in object.keys() { + if !allowed.contains(&key.as_str()) { + return Err(WarmStartExactJhError::InvalidTeacherDataset { + reason: format!("{label} contains unknown teacher field '{key}'"), + }); + } + } + Ok(()) +} + +fn validate_exact_return_alphabet( + min_reward: Reward, + max_reward: Reward, + return_horizon: usize, + return_bins: usize, +) -> Result<(), WarmStartExactJhError> { + let min_return = (min_reward as i128) + .checked_mul(return_horizon as i128) + .ok_or(WarmStartExactJhError::ExactReturnRangeOverflow)?; + let max_return = (max_reward as i128) + .checked_mul(return_horizon as i128) + .ok_or(WarmStartExactJhError::ExactReturnRangeOverflow)?; + let span = max_return + .checked_sub(min_return) + .and_then(|value| value.checked_add(1)) + .ok_or(WarmStartExactJhError::ExactReturnRangeOverflow)?; + let required = + u128::try_from(span).map_err(|_| WarmStartExactJhError::ExactReturnRangeOverflow)?; + if required > return_bins as u128 { + return Err(WarmStartExactJhError::ReturnBinsTooSmall { + required, + configured: return_bins, + }); + } + Ok(()) +} + +fn exact_return_labels_for_trace( + config: &WarmStartExactJhRuntimeConfig, + steps: &[WarmStartExactJhTransition], +) -> Result>, WarmStartExactJhError> { + let mut labels = vec![None; steps.len()]; + if config.return_horizon == 0 || steps.len() < config.return_horizon { + return Ok(labels); + } + + let horizon = config.return_horizon; + let mut window_sum = 0_i128; + for (index, step) in steps.iter().enumerate() { + window_sum += step.reward as i128; + if index >= horizon { + window_sum -= steps[index - horizon].reward as i128; + } + if index + 1 >= horizon { + let start0 = index + 1 - horizon; + labels[start0] = Some(label_for_exact_return(config, window_sum)?); + } + } + Ok(labels) +} + +fn label_for_exact_return( + config: &WarmStartExactJhRuntimeConfig, + exact_return: i128, +) -> Result { + let min_return = (config.min_reward as i128) + .checked_mul(config.return_horizon as i128) + .ok_or(WarmStartExactJhError::ExactReturnRangeOverflow)?; + let label = exact_return + .checked_sub(min_return) + .ok_or(WarmStartExactJhError::ExactReturnRangeOverflow)?; + if label < 0 || label >= config.return_bins as i128 { + return Err(WarmStartExactJhError::ReturnBinsTooSmall { + required: (label + 1).max(0) as u128, + configured: config.return_bins, + }); + } + u64::try_from(label).map_err(|_| WarmStartExactJhError::ExactReturnRangeOverflow) +} + +fn validate_runtime_transition( + config: &WarmStartExactJhRuntimeConfig, + action: Action, + observations: &[PerceptVal], + reward: Reward, +) -> Result<(), WarmStartExactJhError> { + if action as usize >= config.agent_actions.get() { + return Err(WarmStartExactJhError::ActionOutOfRange { + action, + agent_actions: config.agent_actions, + }); + } + if observations.len() != config.observation_stream_len { + return Err(WarmStartExactJhError::ObservationStreamLengthMismatch { + expected: config.observation_stream_len, + actual: observations.len(), + }); + } + let obs_max = max_value_for_bits(config.observation_bits); + for &observation in observations { + if observation > obs_max { + return Err(WarmStartExactJhError::ObservationValueOutOfRange { + observation, + observation_bits: config.observation_bits, + maximum: obs_max, + }); + } + } + if reward < config.min_reward || reward > config.max_reward { + return Err(WarmStartExactJhError::RewardOutOfRange { + reward, + min_reward: config.min_reward, + max_reward: config.max_reward, + }); + } + Ok(()) +} + +struct WarmStartAugmentedTokenContext<'a> { + config: &'a WarmStartExactJhRuntimeConfig, + steps: &'a [StepRecord], + return_labels_by_step: &'a [Option], + action_bits: usize, + return_label_codec: ReturnLabelCodec, + phase: usize, +} + +fn push_augmented_step_tokens_commit( + ctx: &WarmStartAugmentedTokenContext<'_>, + predictor: &mut dyn Predictor, + idx: usize, +) -> Result { + let step = &ctx.steps[idx - 1]; + let return_label = if idx % ctx.config.label_phase_period == ctx.phase { + Some(ctx.return_labels_by_step[idx - 1].ok_or( + WarmStartExactJhError::MissingReturnLabel { + step: idx, + phase: ctx.phase, + }, + )?) + } else { + None + }; + let reward_value = encoded_reward_value( + step.reward, + ctx.config.reward_bits, + ctx.config.reward_offset, + )?; + + let mut pushed = 0usize; + pushed += push_action_tokens_commit_history(predictor, step.action, ctx.action_bits); + if let Some(label) = return_label { + pushed += ctx.return_label_codec.push_label_commit(predictor, label); + } + pushed += push_percept_tokens_commit_history_encoded( + ctx.config, + predictor, + &step.observations, + reward_value, + ); + Ok(pushed) +} + +fn push_step_tokens_history( + ctx: &WarmStartAugmentedTokenContext<'_>, + predictor: &mut dyn Predictor, + idx: usize, +) -> Result { + let step = &ctx.steps[idx - 1]; + let reward_value = encoded_reward_value( + step.reward, + ctx.config.reward_bits, + ctx.config.reward_offset, + )?; + + let mut pushed = 0usize; + pushed += push_encoded_bits_history(predictor, step.action, ctx.action_bits); + if idx % ctx.config.label_phase_period == ctx.phase + && let Some(label) = ctx.return_labels_by_step[idx - 1] + { + pushed += ctx.return_label_codec.push_label_history(predictor, label); + } + pushed += push_percept_tokens_history_encoded( + ctx.config, + predictor, + &step.observations, + reward_value, + ); + Ok(pushed) +} + +fn push_percept_tokens_commit_history( + config: &WarmStartExactJhRuntimeConfig, + predictor: &mut dyn Predictor, + observations: &[PerceptVal], + reward: Reward, +) -> Result { + let reward_value = encoded_reward_value(reward, config.reward_bits, config.reward_offset)?; + Ok(push_percept_tokens_commit_history_encoded( + config, + predictor, + observations, + reward_value, + )) +} + +fn push_percept_tokens_commit_history_encoded( + config: &WarmStartExactJhRuntimeConfig, + predictor: &mut dyn Predictor, + observations: &[PerceptVal], + reward_value: u64, +) -> usize { + let mut pushed = 0usize; + for &observation in observations { + pushed += push_encoded_bits_commit_history(predictor, observation, config.observation_bits); + } + pushed += push_encoded_bits_commit_history(predictor, reward_value, config.reward_bits); + pushed +} + +fn push_percept_tokens_history_encoded( + config: &WarmStartExactJhRuntimeConfig, + predictor: &mut dyn Predictor, + observations: &[PerceptVal], + reward_value: u64, +) -> usize { + let mut pushed = 0usize; + for &observation in observations { + pushed += push_encoded_bits_history(predictor, observation, config.observation_bits); + } + pushed += push_encoded_bits_history(predictor, reward_value, config.reward_bits); + pushed +} + +fn push_action_tokens_commit_history( + predictor: &mut dyn Predictor, + action: Action, + action_bits: usize, +) -> usize { + push_encoded_bits_commit_history(predictor, action, action_bits) +} + +fn push_encoded_bits_history(predictor: &mut dyn Predictor, value: u64, bits: usize) -> usize { + let mut v = value; + for _ in 0..bits { + predictor.update_history((v & 1) == 1); + v >>= 1; + } + bits +} + +fn push_encoded_bits_commit_history( + predictor: &mut dyn Predictor, + value: u64, + bits: usize, +) -> usize { + let mut v = value; + for _ in 0..bits { + predictor.commit_update_history((v & 1) == 1); + v >>= 1; + } + bits +} + +fn encoded_reward_value( + reward: Reward, + bits: usize, + offset: Reward, +) -> Result { + validate_reward_encoding_bounds(reward, reward, offset, bits) + .map_err(WarmStartExactJhError::from)?; + let shifted = (reward as i128) + (offset as i128); + debug_assert!( + shifted >= 0, + "validate_reward_encoding_bounds implies shifted minimum >= 0" + ); + Ok(shifted as u64) +} + +fn pop_history_bits(predictor: &mut dyn Predictor, bits: usize) { + for _ in 0..bits { + predictor.pop_history(); + } +} + +fn max_value_for_bits(bits: usize) -> u64 { + if bits >= 64 { + u64::MAX + } else if bits == 0 { + 0 + } else { + (1u64 << bits) - 1 + } +} + +fn argmax_with_fixed_tie_break(values: &[f64]) -> usize { + let mut best_value = f64::NEG_INFINITY; + let mut best_index = 0usize; + for (index, &value) in values.iter().enumerate() { + if value > best_value { + best_value = value; + best_index = index; + } + } + best_index +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::aixi::warmstart_contract::standalone_teacher_provenance_crc32_pair; + use std::sync::{Arc, Mutex}; + + const TEST_TASK_FINGERPRINT_HEX: &str = + "0102030401020304010203040102030401020304010203040102030401020304"; + #[cfg(feature = "backend-ctw")] + const ZERO_TASK_FINGERPRINT_HEX: &str = + "0000000000000000000000000000000000000000000000000000000000000000"; + #[cfg(feature = "backend-ctw")] + const MISMATCH_TASK_FINGERPRINT_HEX: &str = + "deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef"; + + fn action_alphabet(n: usize) -> ActionAlphabet { + ActionAlphabet::try_from_usize(n).expect("test action alphabet must be non-zero") + } + + fn config() -> WarmStartExactJhConfig { + WarmStartExactJhConfig { + rate_backend: RateBackend::Ctw { depth: 4 }, + observation_bits: 2, + observation_stream_len: 1, + reward_bits: 2, + agent_actions: action_alphabet(2), + return_horizon: 1, + return_bins: 4, + label_phase_period: 1, + planner_simulations_per_step: 1, + bit_stream_semantics: BitStreamSemantics::BinaryTokens, + random_seed: Some(9), + } + } + + #[test] + fn reward_bounds_from_exact_return_bins_matches_default_test_config() { + let cfg = config(); + assert_eq!( + reward_bounds_from_exact_return_bins( + NonZeroUsize::new(cfg.return_horizon).expect("non-zero return horizon"), + NonZeroUsize::new(cfg.return_bins).expect("non-zero return bins"), + cfg.reward_bits + ) + .expect("bounds"), + (0, 3, 0) + ); + } + + #[test] + fn exact_return_labels_cover_each_dense_window() { + let runtime = WarmStartExactJhRuntimeConfig { + task_fingerprint: TaskFingerprint::parse_hex(TEST_TASK_FINGERPRINT_HEX) + .expect("test fingerprint"), + observation_bits: 2, + observation_stream_len: 1, + observation_key_mode: "full_stream", + reward_bits: 2, + agent_actions: action_alphabet(2), + min_reward: 0, + max_reward: 3, + reward_offset: 0, + return_horizon: 3, + return_bins: 10, + label_phase_period: 3, + planner_simulations_per_step: 1, + random_seed: 11, + provenance_policy: TeacherProvenancePolicy::StandalonePlannerRun, + }; + let steps = [1, 2, 0, 3, 1] + .into_iter() + .map(|reward| WarmStartExactJhTransition { + action: 0, + observations: vec![0], + reward, + }) + .collect::>(); + + let labels = exact_return_labels_for_trace(&runtime, &steps).expect("dense return labels"); + + assert_eq!(labels, vec![Some(3), Some(5), Some(4), None, None]); + } + + #[cfg(feature = "backend-ctw")] + #[test] + fn validate_warmstart_teacher_against_compiled_accepts_matching_contract() { + let cfg = config(); + let compiled = cfg.compile_planner_run_spec().expect("compile planner run"); + let teacher = teacher_for_config(&cfg); + validate_warmstart_teacher_against_compiled_planner_run(&compiled, &teacher.contract) + .expect("matching teacher must validate"); + validate_warmstart_teacher_dataset_for_compiled_planner_run(&compiled, &teacher) + .expect("matching teacher dataset must validate"); + } + + #[cfg(feature = "backend-ctw")] + #[test] + fn full_teacher_validator_rejects_short_trace_before_export() { + let mut cfg = config(); + cfg.return_horizon = 2; + cfg.return_bins = 5; + cfg.label_phase_period = 2; + let compiled = cfg.compile_planner_run_spec().expect("compile planner run"); + let mut teacher = teacher_for_config(&cfg); + teacher.traces[0].transitions = vec![WarmStartExactJhTransition { + action: 0, + observations: vec![1], + reward: 0, + }]; + + let err = validate_warmstart_teacher_dataset_for_compiled_planner_run(&compiled, &teacher) + .expect_err("short trace must fail before write/load"); + assert!(err.to_string().contains("return_horizon is 2"), "{err}"); + } + + #[cfg(feature = "backend-ctw")] + #[test] + fn full_teacher_validator_rejects_bit_width_valid_but_runtime_invalid_reward() { + let mut cfg = config(); + cfg.return_horizon = 2; + cfg.return_bins = 5; + cfg.label_phase_period = 2; + let compiled = cfg.compile_planner_run_spec().expect("compile planner run"); + let mut teacher = teacher_for_config(&cfg); + teacher.traces[0].transitions = vec![ + WarmStartExactJhTransition { + action: 0, + observations: vec![1], + reward: 3, + }, + WarmStartExactJhTransition { + action: 1, + observations: vec![2], + reward: 0, + }, + ]; + + let err = validate_warmstart_teacher_dataset_for_compiled_planner_run(&compiled, &teacher) + .expect_err("reward valid for reward_bits but outside exact runtime range must fail"); + assert!(err.to_string().contains("runtime contract"), "{err}"); + assert!( + err.to_string().contains("outside configured range"), + "{err}" + ); + } + + #[cfg(feature = "backend-ctw")] + #[test] + fn validate_warmstart_teacher_against_compiled_rejects_reward_certificate_crc_mismatch() { + let cfg = config(); + let compiled = cfg.compile_planner_run_spec().expect("compile planner run"); + let mut teacher = teacher_for_config(&cfg); + teacher.contract.exact_reward_encoding_certificate = "00000000".to_string(); + let err = + validate_warmstart_teacher_against_compiled_planner_run(&compiled, &teacher.contract) + .expect_err("corrupted certificate hash must fail"); + assert!(matches!( + err, + WarmStartExactJhError::InvalidTeacherDataset { .. } + )); + } + + #[cfg(feature = "backend-ctw")] + #[test] + fn validate_warmstart_teacher_planner_task_fingerprint_rejects_mismatch_with_stable_markers() { + let cfg = config(); + let compiled = cfg.compile_planner_run_spec().expect("compile planner run"); + let mut teacher = teacher_for_config(&cfg); + teacher.contract.task_fingerprint = TaskFingerprint::parse_hex(ZERO_TASK_FINGERPRINT_HEX) + .expect("valid mismatch fingerprint"); + let err = validate_warmstart_teacher_planner_task_fingerprint(&compiled, &teacher.contract) + .expect_err("wrong fingerprint must fail"); + let msg = err.to_string(); + assert!(msg.contains("task_fingerprint"), "{msg}"); + assert!(msg.contains("current planner_run '"), "{msg}"); + } + + /// Single source of truth for the test teacher-contract field wiring. + /// + /// This builder performs no backend compilation: the `task_fingerprint` and + /// `observation_key_mode` are supplied by the caller. The CTW-dependent path + /// derives them from a real compiled planner run; the backend-independent + /// path supplies synthetic-but-faithful values so the contract-consuming + /// tests stay runnable under the feature-light `aixi` slice. + fn teacher_contract_for( + cfg: &WarmStartExactJhConfig, + task_fingerprint: TaskFingerprint, + observation_key_mode: &str, + ) -> WarmStartExactJhTeacherContract { + let observation_stream_len = cfg.observation_stream_len.max(1); + let (adapter_crc, reward_cert) = standalone_teacher_provenance_crc32_pair( + cfg.observation_bits, + observation_stream_len, + cfg.reward_bits, + ) + .expect("standalone teacher provenance crc pair"); + WarmStartExactJhTeacherContract { + schema_version: WARMSTART_TEACHER_CONTRACT_SCHEMA_VERSION, + task_fingerprint, + action_alphabet_size: cfg.agent_actions.get(), + observation_bits: cfg.observation_bits, + observation_stream_len, + observation_key_mode: observation_key_mode.to_string(), + observation_adapter_spec_ref: WARMSTART_STANDALONE_OBSERVATION_ADAPTER_SPEC_REF + .to_string(), + observation_adapter_content_crc32: adapter_crc, + reward_bits: cfg.reward_bits, + return_horizon: cfg.return_horizon, + label_phase_period: cfg.label_phase_period, + scalar_representation: WARMSTART_STANDALONE_SCALAR_REPRESENTATION.to_string(), + exact_reward_encoding_certificate: reward_cert, + } + } + + /// Backend-independent teacher contract for tests that only validate + /// contract-shaped data (JSONL conversion, trace recording, dataset + /// canonicalization) and never compile a planner run. The fingerprint and + /// key mode are fixed: `full_stream` matches the canonical planner + /// interface's hardcoded `ObservationKeyMode::FullStream`. + fn teacher_contract() -> WarmStartExactJhTeacherContract { + teacher_contract_for( + &config(), + TaskFingerprint::parse_hex(TEST_TASK_FINGERPRINT_HEX).expect("test task fingerprint"), + "full_stream", + ) + } + + #[cfg(feature = "backend-ctw")] + fn teacher_for_config(cfg: &WarmStartExactJhConfig) -> WarmStartExactJhTeacherDataset { + let compiled = cfg + .compile_planner_run_spec() + .expect("test planner run must compile"); + let task_fingerprint = warmstart_exact_jh_planner_task_fingerprint(&compiled) + .expect("test planner fingerprint"); + let observation_key_mode = + observation_key_mode_name(compiled.interface().observation_key_mode); + WarmStartExactJhTeacherDataset { + contract: teacher_contract_for(cfg, task_fingerprint, observation_key_mode), + traces: vec![WarmStartExactJhTeacherTrace { + transitions: vec![ + WarmStartExactJhTransition { + action: 0, + observations: vec![1], + reward: 0, + }, + WarmStartExactJhTransition { + action: 1, + observations: vec![2], + reward: 3, + }, + WarmStartExactJhTransition { + action: 1, + observations: vec![2], + reward: 3, + }, + ], + }], + } + } + + #[cfg(feature = "backend-ctw")] + fn teacher() -> WarmStartExactJhTeacherDataset { + teacher_for_config(&config()) + } + + #[test] + fn jsonl_trace_converter_round_trips_action_percept_pairs() { + let contract = teacher_contract(); + let jsonl = [ + warmstart_jsonl_action_record(0, 0, PlannerActionProvenance::Greedy).to_string(), + warmstart_jsonl_percept_record(0, &[1], 0).to_string(), + warmstart_jsonl_action_record(1, 1, PlannerActionProvenance::Exploratory).to_string(), + warmstart_jsonl_percept_record(1, &[2], 3).to_string(), + ] + .join("\n"); + let trace = warmstart_teacher_trace_from_jsonl_slice( + jsonl.as_bytes(), + &contract, + contract.return_horizon, + ) + .expect("jsonl trace should parse"); + assert_eq!( + trace.transitions, + vec![ + WarmStartExactJhTransition { + action: 0, + observations: vec![1], + reward: 0, + }, + WarmStartExactJhTransition { + action: 1, + observations: vec![2], + reward: 3, + }, + ] + ); + } + + #[test] + fn jsonl_trace_converter_converts_mcaixi_decision_percept_order() { + let contract = teacher_contract(); + let jsonl = [ + warmstart_jsonl_percept_record(0, &[0], 0).to_string(), + warmstart_jsonl_action_record(0, 0, PlannerActionProvenance::Greedy).to_string(), + warmstart_jsonl_percept_record(1, &[1], 3).to_string(), + warmstart_jsonl_action_record(1, 1, PlannerActionProvenance::Greedy).to_string(), + warmstart_jsonl_percept_record(2, &[2], 0).to_string(), + ] + .join("\n"); + let trace = warmstart_teacher_trace_from_jsonl_slice( + jsonl.as_bytes(), + &contract, + contract.return_horizon, + ) + .expect("MC-AIXI-order jsonl trace should parse"); + assert_eq!( + trace.transitions, + vec![ + WarmStartExactJhTransition { + action: 0, + observations: vec![1], + reward: 3, + }, + WarmStartExactJhTransition { + action: 1, + observations: vec![2], + reward: 0, + }, + ] + ); + } + + #[test] + fn jsonl_trace_converter_rejects_sparse_action_then_percept_steps() { + let contract = teacher_contract(); + let jsonl = [ + warmstart_jsonl_action_record(0, 0, PlannerActionProvenance::Greedy).to_string(), + warmstart_jsonl_percept_record(0, &[1], 0).to_string(), + warmstart_jsonl_action_record(2, 1, PlannerActionProvenance::Greedy).to_string(), + warmstart_jsonl_percept_record(2, &[2], 3).to_string(), + ] + .join("\n"); + + let err = warmstart_teacher_trace_from_jsonl_slice( + jsonl.as_bytes(), + &contract, + contract.return_horizon, + ) + .expect_err("sparse action/percept JSONL trace must fail"); + + assert!(err.to_string().contains("not contiguous"), "{err}"); + assert!(err.to_string().contains("step 1"), "{err}"); + } + + #[test] + fn jsonl_trace_converter_rejects_sparse_decision_percept_steps() { + let contract = teacher_contract(); + let jsonl = [ + warmstart_jsonl_percept_record(0, &[0], 0).to_string(), + warmstart_jsonl_action_record(0, 0, PlannerActionProvenance::Greedy).to_string(), + warmstart_jsonl_percept_record(1, &[1], 3).to_string(), + warmstart_jsonl_percept_record(2, &[2], 0).to_string(), + warmstart_jsonl_action_record(2, 1, PlannerActionProvenance::Greedy).to_string(), + warmstart_jsonl_percept_record(3, &[2], 0).to_string(), + ] + .join("\n"); + + let err = warmstart_teacher_trace_from_jsonl_slice( + jsonl.as_bytes(), + &contract, + contract.return_horizon, + ) + .expect_err("sparse decision-percept JSONL trace must fail"); + + assert!(err.to_string().contains("not contiguous"), "{err}"); + assert!(err.to_string().contains("step 1"), "{err}"); + } + + #[test] + fn jsonl_trace_converter_rejects_malformed_and_inconsistent_records() { + let contract = teacher_contract(); + let return_horizon = contract.return_horizon; + for (jsonl, expected) in [ + ("{", "invalid JSON"), + ( + r#"{"kind":"action","t":0,"action":0,"provenance":"unknown"}"#, + "unknown action provenance", + ), + ( + r#"{"kind":"action","t":0,"action":0,"provenance":7}"#, + "action provenance must be a string", + ), + ( + r#"{"kind":"action","t":0,"action":0}"#, + "cannot infer JSONL action/percept convention", + ), + ( + r#"{"kind":"percept","t":0,"observations":[1],"reward":0}"#, + "cannot infer JSONL action/percept convention", + ), + ( + concat!( + r#"{"kind":"action","t":0,"action":0}"#, + "\n", + r#"{"kind":"percept","t":0,"observations":[4],"reward":0}"# + ), + "observation value", + ), + ( + r#"{"kind":"action","t":0,"action":0,"extra":true}"#, + "unknown field 'extra'", + ), + ("\n", "empty JSONL records are not allowed"), + ( + concat!( + r#"{"kind":"action","t":0,"action":0}"#, + "\n", + r#"{"kind":"percept","t":0,"observations":[1],"reward":0}"#, + "\n", + r#"{"kind":"percept","t":1,"observations":[1],"reward":0}"#, + "\n", + r#"{"kind":"action","t":1,"action":0}"# + ), + "mixed JSONL action/percept conventions", + ), + ] { + let err = warmstart_teacher_trace_from_jsonl_slice( + jsonl.as_bytes(), + &contract, + return_horizon, + ) + .expect_err("invalid JSONL trace should fail"); + assert!( + err.to_string().contains(expected), + "expected '{expected}' in {err}" + ); + } + } + + #[test] + fn jsonl_trace_converter_accepts_absent_provenance() { + let contract = teacher_contract(); + let absent = concat!( + r#"{"kind":"action","t":0,"action":1}"#, + "\n", + r#"{"kind":"percept","t":0,"observations":[2],"reward":1}"#, + "\n", + r#"{"kind":"action","t":1,"action":0}"#, + "\n", + r#"{"kind":"percept","t":1,"observations":[1],"reward":0}"# + ); + let trace = warmstart_teacher_trace_from_jsonl_slice(absent.as_bytes(), &contract, 1) + .expect("legacy trace without provenance should parse"); + assert_eq!(trace.transitions.len(), 2); + assert_eq!(trace.transitions[0].action, 1); + assert_eq!(trace.transitions[1].action, 0); + } + + #[test] + fn jsonl_trace_converter_rejects_duplicate_action_and_percept_records() { + let contract = teacher_contract(); + let duplicate_action = concat!( + r#"{"kind":"action","t":0,"action":0}"#, + "\n", + r#"{"kind":"action","t":0,"action":1}"#, + "\n", + r#"{"kind":"percept","t":0,"observations":[1],"reward":0}"# + ); + let err = + warmstart_teacher_trace_from_jsonl_slice(duplicate_action.as_bytes(), &contract, 1) + .expect_err("duplicate action must fail"); + assert!(err.to_string().contains("duplicate action record"), "{err}"); + + let duplicate_percept = concat!( + r#"{"kind":"action","t":0,"action":0}"#, + "\n", + r#"{"kind":"percept","t":0,"observations":[1],"reward":0}"#, + "\n", + r#"{"kind":"percept","t":0,"observations":[2],"reward":1}"# + ); + let err = + warmstart_teacher_trace_from_jsonl_slice(duplicate_percept.as_bytes(), &contract, 1) + .expect_err("duplicate percept must fail"); + assert!( + err.to_string().contains("duplicate percept record"), + "{err}" + ); + } + + #[test] + fn trace_recorder_requires_a_complete_return_horizon_window() { + let contract = teacher_contract(); + let mut recorder = WarmStartExactJhTraceRecorder::new(); + recorder + .record_action(0, 1) + .expect("record action at step 0"); + recorder + .record_percept(0, &[2], 3) + .expect("record percept at step 0"); + let err = recorder + .into_teacher_trace(&contract, 2) + .expect_err("single transition must fail for return_horizon 2"); + assert!(err.to_string().contains("return_horizon is 2"), "{err}"); + + let mut recorder = WarmStartExactJhTraceRecorder::new(); + recorder + .record_action(0, 1) + .expect("record action at step 0"); + recorder + .record_percept(0, &[2], 3) + .expect("record percept at step 0"); + let trace = recorder + .into_teacher_trace(&contract, 1) + .expect("one transition covers horizon one"); + assert_eq!(trace.transitions.len(), 1); + assert_eq!(trace.transitions[0].action, 1); + assert_eq!(trace.transitions[0].observations, vec![2]); + assert_eq!(trace.transitions[0].reward, 3); + } + + #[test] + fn trace_recorder_rejects_sparse_step_sets() { + let contract = teacher_contract(); + let mut recorder = WarmStartExactJhTraceRecorder::new(); + recorder + .record_action(0, 0) + .expect("record action at step 0"); + recorder + .record_percept(0, &[1], 0) + .expect("record percept at step 0"); + recorder + .record_action(2, 1) + .expect("record action at step 2"); + recorder + .record_percept(2, &[2], 3) + .expect("record percept at step 2"); + + let err = recorder + .into_teacher_trace(&contract, 2) + .expect_err("sparse recorder trace must fail"); + + assert!(err.to_string().contains("not contiguous"), "{err}"); + assert!(err.to_string().contains("step 1"), "{err}"); + } + + #[derive(Clone, Debug, Default, Eq, PartialEq)] + struct PhaseUpdateCounts { + commit_label_bits: usize, + commit_history_bits: usize, + history_bits: usize, + } + + #[derive(Clone)] + struct PhaseUpdateCountingPredictor { + counts: Arc>, + } + + impl Predictor for PhaseUpdateCountingPredictor { + fn update(&mut self, _sym: bool) {} + + fn commit_update(&mut self, _sym: bool) { + self.counts + .lock() + .expect("counts mutex poisoned") + .commit_label_bits += 1; + } + + fn update_history(&mut self, _sym: bool) { + self.counts + .lock() + .expect("counts mutex poisoned") + .history_bits += 1; + } + + fn commit_update_history(&mut self, _sym: bool) { + self.counts + .lock() + .expect("counts mutex poisoned") + .commit_history_bits += 1; + } + + fn revert(&mut self) {} + + fn pop_history(&mut self) {} + + fn predict_prob(&mut self, sym: bool) -> f64 { + if sym { 0.75 } else { 0.25 } + } + + fn model_name(&self) -> String { + "PhaseUpdateCountingPredictor".to_string() + } + + fn boxed_clone(&self) -> Box { + Box::new(self.clone()) + } + } + + #[test] + fn warmstart_offline_phase_stream_updates_one_phase_per_closed_horizon_window() { + let counts = (0..3) + .map(|_| Arc::new(Mutex::new(PhaseUpdateCounts::default()))) + .collect::>(); + let cfg = WarmStartExactJhRuntimeConfig { + task_fingerprint: TaskFingerprint::parse_hex(TEST_TASK_FINGERPRINT_HEX) + .expect("test fingerprint"), + observation_bits: 2, + observation_stream_len: 1, + observation_key_mode: "full_stream", + reward_bits: 2, + agent_actions: action_alphabet(2), + min_reward: 0, + max_reward: 3, + reward_offset: 0, + return_horizon: 2, + return_bins: 7, + label_phase_period: 3, + planner_simulations_per_step: 1, + random_seed: 11, + provenance_policy: TeacherProvenancePolicy::StandalonePlannerRun, + }; + let teacher = WarmStartExactJhTeacherDataset { + contract: WarmStartExactJhTeacherContract { + schema_version: 1, + task_fingerprint: TaskFingerprint::parse_hex(TEST_TASK_FINGERPRINT_HEX) + .expect("test fingerprint"), + action_alphabet_size: 2, + observation_bits: 2, + observation_stream_len: 1, + observation_key_mode: "full_stream".to_string(), + observation_adapter_spec_ref: "test-observation-adapter".to_string(), + observation_adapter_content_crc32: "test-observation-adapter-crc32".to_string(), + reward_bits: 2, + return_horizon: 2, + label_phase_period: 3, + scalar_representation: "test-scalar".to_string(), + exact_reward_encoding_certificate: "test-cert".to_string(), + }, + traces: vec![WarmStartExactJhTeacherTrace { + transitions: vec![ + WarmStartExactJhTransition { + action: 0, + observations: vec![1], + reward: 1, + }, + WarmStartExactJhTransition { + action: 1, + observations: vec![2], + reward: 2, + }, + WarmStartExactJhTransition { + action: 0, + observations: vec![3], + reward: 0, + }, + ], + }], + }; + let mut agent = WarmStartExactJhAgent { + config: cfg, + phases: counts + .iter() + .map(|counts| PhaseModel { + predictor: Box::new(PhaseUpdateCountingPredictor { + counts: counts.clone(), + }), + last_augmented_step: 0, + }) + .collect(), + steps: Vec::new(), + return_labels_by_step: Vec::new(), + total_steps_observed: 0, + action_bits: 1, + return_label_codec: ReturnLabelCodec::value_monotone(7), + teacher_label_count: 0, + rng: RandomGenerator::from_seed(11), + }; + + agent + .warm_start_from_teacher(&teacher) + .expect("offline teacher trace should warm-start"); + + let snapshots = counts + .iter() + .map(|counts| counts.lock().expect("counts mutex poisoned").clone()) + .collect::>(); + assert_eq!(agent.teacher_label_count(), 2); + assert_eq!(agent.steps_observed(), 0); + assert!(agent.same_task_live_trace().is_none()); + assert_eq!( + snapshots, + vec![ + PhaseUpdateCounts { + commit_label_bits: 0, + commit_history_bits: 15, + history_bits: 0, + }, + PhaseUpdateCounts { + commit_label_bits: 3, + commit_history_bits: 15, + history_bits: 0, + }, + PhaseUpdateCounts { + commit_label_bits: 3, + commit_history_bits: 15, + history_bits: 0, + }, + ] + ); + } + + #[test] + fn warmstart_action_values_propagate_history_encoding_errors_without_mutation() { + let counts = Arc::new(Mutex::new(PhaseUpdateCounts::default())); + let cfg = WarmStartExactJhRuntimeConfig { + task_fingerprint: TaskFingerprint::parse_hex(TEST_TASK_FINGERPRINT_HEX) + .expect("test fingerprint"), + observation_bits: 2, + observation_stream_len: 1, + observation_key_mode: "full_stream", + reward_bits: 1, + agent_actions: action_alphabet(2), + min_reward: 0, + max_reward: 1, + reward_offset: 0, + return_horizon: 1, + return_bins: 2, + label_phase_period: 1, + planner_simulations_per_step: 1, + random_seed: 11, + provenance_policy: TeacherProvenancePolicy::StandalonePlannerRun, + }; + let mut agent = WarmStartExactJhAgent { + config: cfg, + phases: vec![PhaseModel { + predictor: Box::new(PhaseUpdateCountingPredictor { + counts: counts.clone(), + }), + last_augmented_step: 0, + }], + steps: vec![StepRecord { + action: 0, + observations: vec![0], + reward: 2, + }], + return_labels_by_step: vec![Some(0)], + total_steps_observed: 1, + action_bits: 1, + return_label_codec: ReturnLabelCodec::value_monotone(2), + teacher_label_count: 0, + rng: RandomGenerator::from_seed(11), + }; + + let err = agent + .estimate_action_values() + .expect_err("retained invalid reward must be reported"); + + assert!(matches!(err, WarmStartExactJhError::RewardEncoding(_))); + assert_eq!( + *counts.lock().expect("counts mutex poisoned"), + PhaseUpdateCounts::default() + ); + } + + #[cfg(feature = "backend-ctw")] + #[test] + fn standalone_warmstart_teacher_contract_matches_compiled_validation() { + let cfg = config(); + let compiled = cfg.compile_planner_run_spec().expect("compile planner run"); + let contract = standalone_warmstart_teacher_contract_for_compiled_planner_run(&compiled) + .expect("standalone contract"); + validate_warmstart_teacher_against_compiled_planner_run(&compiled, &contract) + .expect("standalone contract must validate against compiled planner run"); + } + + #[test] + fn merge_warmstart_teacher_traces_preserves_deterministic_order_and_dedups() { + let middle = WarmStartExactJhTeacherTrace { + transitions: vec![WarmStartExactJhTransition { + action: 0, + observations: vec![2], + reward: 0, + }], + }; + let mut traces = vec![middle.clone()]; + let high = WarmStartExactJhTeacherTrace { + transitions: vec![WarmStartExactJhTransition { + action: 1, + observations: vec![2], + reward: 3, + }], + }; + let low = WarmStartExactJhTeacherTrace { + transitions: vec![WarmStartExactJhTransition { + action: 0, + observations: vec![1], + reward: 0, + }], + }; + let (inserted, _) = + merge_warmstart_teacher_traces_deterministic(&mut traces, vec![high.clone(), low]); + assert_eq!(inserted, 2); + assert!(!merge_warmstart_teacher_trace_deterministic( + &mut traces, + high + )); + assert!(!merge_warmstart_teacher_trace_deterministic( + &mut traces, + middle + )); + assert_eq!(traces.len(), 3); + assert_eq!(traces[0].transitions[0].action, 0); + assert_eq!(traces[0].transitions[0].observations, vec![1]); + assert_eq!(traces[1].transitions[0].observations, vec![2]); + assert_eq!(traces[2].transitions[0].action, 1); + } + + #[test] + fn teacher_dataset_new_canonicalizes_trace_order_and_dedups() { + let high = WarmStartExactJhTeacherTrace { + transitions: vec![WarmStartExactJhTransition { + action: 1, + observations: vec![2], + reward: 3, + }], + }; + let low = WarmStartExactJhTeacherTrace { + transitions: vec![WarmStartExactJhTransition { + action: 0, + observations: vec![1], + reward: 0, + }], + }; + let dataset = WarmStartExactJhTeacherDataset::new( + teacher_contract(), + vec![high.clone(), low.clone(), high.clone()], + ); + assert_eq!(dataset.traces, vec![low, high]); + } + + #[test] + fn json_teacher_dataset_requires_same_task_traces() { + let value = serde_json::json!({ + "schema_version": 1, + "contract": { + "task_fingerprint": TEST_TASK_FINGERPRINT_HEX, + "action_alphabet_size": 2, + "observation_bits": 2, + "observation_stream_len": 1, + "observation_key_mode": "full_stream", + "observation_adapter_spec_ref": "test-observation-adapter", + "observation_adapter_content_crc32": "test-observation-adapter-crc32", + "reward_bits": 2, + "return_horizon": 1, + "label_phase_period": 1, + "scalar_representation": "test-scalar", + "exact_reward_encoding_certificate": "test-cert" + }, + "traces": [{ + "transitions": [{"action": 1, "observations": [2], "reward": 3}] + }] + }); + let parsed = WarmStartExactJhTeacherDataset::from_json_value(&value) + .expect("teacher trace should parse"); + assert_eq!(parsed.traces.len(), 1); + assert_eq!(parsed.traces[0].transitions[0].action, 1); + } + + #[test] + fn json_teacher_dataset_rejects_malformed_task_fingerprint() { + let value = serde_json::json!({ + "schema_version": 1, + "contract": { + "task_fingerprint": "not-a-fingerprint", + "action_alphabet_size": 2, + "observation_bits": 2, + "observation_stream_len": 1, + "observation_key_mode": "full_stream", + "observation_adapter_spec_ref": "test-observation-adapter", + "observation_adapter_content_crc32": "test-observation-adapter-crc32", + "reward_bits": 2, + "return_horizon": 1, + "label_phase_period": 1, + "scalar_representation": "test-scalar", + "exact_reward_encoding_certificate": "test-cert" + }, + "traces": [{ + "transitions": [{"action": 1, "observations": [2], "reward": 3}] + }] + }); + let err = WarmStartExactJhTeacherDataset::from_json_value(&value) + .expect_err("malformed task fingerprint must fail at parse time"); + assert!( + err.to_string() + .contains("must be a 64-digit lowercase hexadecimal SHA-256 digest"), + "{err}" + ); + } + + #[test] + fn json_teacher_dataset_rejects_legacy_trace_and_observation_aliases() { + let mut value = serde_json::json!({ + "schema_version": 1, + "contract": { + "task_fingerprint": TEST_TASK_FINGERPRINT_HEX, + "action_alphabet_size": 2, + "observation_bits": 2, + "observation_stream_len": 1, + "observation_key_mode": "full_stream", + "observation_adapter_spec_ref": "test-observation-adapter", + "observation_adapter_content_crc32": "test-observation-adapter-crc32", + "reward_bits": 2, + "return_horizon": 1, + "label_phase_period": 1, + "scalar_representation": "test-scalar", + "exact_reward_encoding_certificate": "test-cert" + }, + "traces": [[{"action": 1, "observations": [2], "reward": 3}]] + }); + let err = WarmStartExactJhTeacherDataset::from_json_value(&value) + .expect_err("bare trace arrays must be rejected"); + assert!( + err.to_string() + .contains("must be an object with transitions") + ); + + value["traces"] = serde_json::json!([{ + "transitions": [{"action": 1, "obs": [2], "reward": 3}] + }]); + let err = WarmStartExactJhTeacherDataset::from_json_value(&value) + .expect_err("obs alias must be rejected"); + assert!(err.to_string().contains("unknown teacher field 'obs'")); + + let mut top_extra = value.clone(); + top_extra["traces"] = serde_json::json!([{ + "transitions": [{"action": 1, "observations": [2], "reward": 3}] + }]); + top_extra["extra"] = serde_json::json!(true); + let err = WarmStartExactJhTeacherDataset::from_json_value(&top_extra) + .expect_err("top-level unknown fields must be rejected"); + assert!(err.to_string().contains("unknown teacher field 'extra'")); + + let mut contract_extra = top_extra; + contract_extra + .as_object_mut() + .expect("object") + .remove("extra"); + contract_extra["contract"]["extra"] = serde_json::json!(true); + let err = WarmStartExactJhTeacherDataset::from_json_value(&contract_extra) + .expect_err("contract unknown fields must be rejected"); + assert!(err.to_string().contains("unknown teacher field 'extra'")); + } + + #[test] + fn teacher_dataset_slice_parser_and_label_count_cover_horizon_windows() { + let err = WarmStartExactJhTeacherDataset::from_json_slice(b"{") + .expect_err("invalid json must be rejected"); + assert!(err.to_string().contains("invalid teacher JSON"), "{err}"); + + let value = serde_json::json!({ + "schema_version": 1, + "contract": { + "task_fingerprint": TEST_TASK_FINGERPRINT_HEX, + "action_alphabet_size": 2, + "observation_bits": 2, + "observation_stream_len": 1, + "observation_key_mode": "full_stream", + "observation_adapter_spec_ref": "test-observation-adapter", + "observation_adapter_content_crc32": "test-observation-adapter-crc32", + "reward_bits": 2, + "return_horizon": 2, + "label_phase_period": 2, + "scalar_representation": "test-scalar", + "exact_reward_encoding_certificate": "test-cert" + }, + "traces": [ + {"transitions": [ + {"action": 0, "observations": [1], "reward": 0}, + {"action": 1, "observations": [2], "reward": 3}, + {"action": 1, "observations": [2], "reward": 3} + ]}, + {"transitions": [ + {"action": 0, "observations": [0], "reward": 1} + ]} + ] + }); + let bytes = serde_json::to_vec(&value).expect("teacher json"); + let parsed = WarmStartExactJhTeacherDataset::from_json_slice(&bytes) + .expect("teacher dataset should parse from slice"); + + assert_eq!(parsed.label_count_for_horizon(0), 0); + assert_eq!(parsed.label_count_for_horizon(1), 4); + assert_eq!(parsed.label_count_for_horizon(2), 2); + assert_eq!(parsed.label_count_for_horizon(4), 0); + } + + #[test] + fn config_validation_reports_local_contract_errors_before_backend_use() { + let mut cfg = config(); + cfg.return_horizon = 0; + assert!(matches!( + cfg.validate(), + Err(WarmStartExactJhError::ReturnHorizonZero) + )); + + let mut cfg = config(); + cfg.return_bins = 0; + assert!(matches!( + cfg.validate(), + Err(WarmStartExactJhError::ReturnBinsZero) + )); + + let mut cfg = config(); + cfg.return_horizon = 2; + cfg.label_phase_period = 1; + assert!(matches!( + cfg.validate(), + Err(WarmStartExactJhError::LabelPhasePeriodTooShort { .. }) + )); + + let mut cfg = config(); + cfg.planner_simulations_per_step = 0; + assert!(matches!( + cfg.validate(), + Err(WarmStartExactJhError::PlannerSimulationsZero) + )); + + let mut cfg = config(); + cfg.planner_simulations_per_step = 2; + assert!(matches!( + cfg.validate(), + Err(WarmStartExactJhError::PlannerSimulationsUnsupported { configured: 2 }) + )); + + let mut cfg = config(); + cfg.return_horizon = 4; + cfg.return_bins = 8; + cfg.label_phase_period = 4; + assert!(matches!( + cfg.validate(), + Err(WarmStartExactJhError::ReturnBinsNotExactHorizon { + return_bins: 8, + return_horizon: 4 + }) + )); + + let mut cfg = config(); + cfg.reward_bits = 1; + assert!(matches!( + cfg.validate(), + Err(WarmStartExactJhError::RewardEncoding(_)) + )); + } + + #[cfg(feature = "backend-ctw")] + #[test] + fn warmstart_agent_rejects_teacher_transitions_outside_interface_contract() { + let mut invalid = teacher(); + invalid.traces[0].transitions[0].action = 2; + let err = match WarmStartExactJhAgent::new(config(), invalid) { + Ok(_) => panic!("teacher action outside alphabet must fail"), + Err(err) => err, + }; + assert!(matches!( + err, + WarmStartExactJhError::ActionOutOfRange { .. } + )); + + let mut invalid = teacher(); + invalid.traces[0].transitions[0].observations = vec![1, 2]; + let err = match WarmStartExactJhAgent::new(config(), invalid) { + Ok(_) => panic!("teacher observation stream length must fail"), + Err(err) => err, + }; + assert!(matches!( + err, + WarmStartExactJhError::ObservationStreamLengthMismatch { .. } + )); + + let mut invalid = teacher(); + invalid.traces[0].transitions[0].observations = vec![4]; + let err = match WarmStartExactJhAgent::new(config(), invalid) { + Ok(_) => panic!("teacher observation value must fail"), + Err(err) => err, + }; + assert!(matches!( + err, + WarmStartExactJhError::ObservationValueOutOfRange { .. } + )); + + let mut invalid = teacher(); + invalid.traces[0].transitions[0].reward = 4; + let err = match WarmStartExactJhAgent::new(config(), invalid) { + Ok(_) => panic!("teacher reward outside range must fail"), + Err(err) => err, + }; + assert!(matches!( + err, + WarmStartExactJhError::RewardOutOfRange { .. } + )); + } + + #[cfg(feature = "backend-ctw")] + #[test] + fn warmstart_agent_rejects_teacher_contract_task_fingerprint_mismatch() { + let mut invalid = teacher(); + invalid.contract.task_fingerprint = + TaskFingerprint::parse_hex(MISMATCH_TASK_FINGERPRINT_HEX) + .expect("valid mismatch fingerprint"); + let err = match WarmStartExactJhAgent::new(config(), invalid) { + Ok(_) => panic!("teacher task fingerprint mismatch must fail"), + Err(err) => err, + }; + assert!(matches!( + err, + WarmStartExactJhError::InvalidTeacherDataset { .. } + )); + assert!(err.to_string().contains("task_fingerprint"), "{err}"); + } + + #[cfg(feature = "backend-ctw")] + #[test] + fn warmstart_agent_rejects_teacher_contract_interface_mismatch() { + let mut invalid = teacher(); + invalid.contract.action_alphabet_size = 3; + let err = match WarmStartExactJhAgent::new(config(), invalid) { + Ok(_) => panic!("teacher contract interface mismatch must fail"), + Err(err) => err, + }; + assert!(matches!( + err, + WarmStartExactJhError::InvalidTeacherDataset { .. } + )); + assert!(err.to_string().contains("action_alphabet_size"), "{err}"); + } + + #[cfg(feature = "backend-ctw")] + #[test] + fn warmstart_agent_rejects_teacher_contract_return_horizon_mismatch() { + let mut invalid = teacher(); + invalid.contract.return_horizon = 2; + let err = match WarmStartExactJhAgent::new(config(), invalid) { + Ok(_) => panic!("teacher contract return_horizon mismatch must fail"), + Err(err) => err, + }; + assert!(matches!( + err, + WarmStartExactJhError::InvalidTeacherDataset { .. } + )); + assert!(err.to_string().contains("return_horizon"), "{err}"); + } + + #[cfg(feature = "backend-ctw")] + #[test] + fn warmstart_agent_rejects_teacher_contract_label_phase_period_mismatch() { + let mut invalid = teacher(); + invalid.contract.label_phase_period = 2; + let err = match WarmStartExactJhAgent::new(config(), invalid) { + Ok(_) => panic!("teacher contract label_phase_period mismatch must fail"), + Err(err) => err, + }; + assert!(matches!( + err, + WarmStartExactJhError::InvalidTeacherDataset { .. } + )); + assert!(err.to_string().contains("label_phase_period"), "{err}"); + } + + #[cfg(feature = "backend-ctw")] + #[test] + fn warmstart_agent_rejects_teacher_contract_observation_key_mode_mismatch() { + let mut invalid = teacher(); + invalid.contract.observation_key_mode = "definitely-not-a-mode".to_string(); + let err = match WarmStartExactJhAgent::new(config(), invalid) { + Ok(_) => panic!("teacher contract observation_key_mode mismatch must fail"), + Err(err) => err, + }; + assert!(matches!( + err, + WarmStartExactJhError::InvalidTeacherDataset { .. } + )); + assert!(err.to_string().contains("observation_key_mode"), "{err}"); + } + + #[cfg(feature = "backend-ctw")] + #[test] + fn warmstart_agent_rejects_teacher_contract_scalar_provenance_mismatch() { + let mut invalid = teacher(); + invalid.contract.scalar_representation = "different-scalar".to_string(); + let err = match WarmStartExactJhAgent::new(config(), invalid) { + Ok(_) => panic!("teacher contract scalar provenance mismatch must fail"), + Err(err) => err, + }; + assert!(matches!( + err, + WarmStartExactJhError::InvalidTeacherDataset { .. } + )); + assert!(err.to_string().contains("scalar_representation"), "{err}"); + } + + #[cfg(feature = "backend-ctw")] + #[test] + fn warmstart_agent_delays_live_trace_until_complete_return_horizon() { + let mut cfg = config(); + cfg.return_horizon = 2; + cfg.return_bins = 7; + cfg.label_phase_period = 2; + cfg.random_seed = Some(16); + let mut agent = WarmStartExactJhAgent::new(cfg.clone(), teacher_for_config(&cfg)) + .expect("warmstart agent should initialize"); + + assert_eq!(agent.teacher_label_count(), 2); + assert_eq!(agent.num_actions(), action_alphabet(2)); + assert_eq!(agent.planner_simulations_per_step(), 1); + assert_eq!(agent.resolved_random_seed(), 16); + assert!(agent.same_task_live_trace().is_none()); + + agent + .observe_transition(0, &[1], 1) + .expect("first live transition"); + assert!(agent.same_task_live_trace().is_none()); + + agent + .observe_transition(1, &[2], 2) + .expect("second live transition"); + let live = agent + .same_task_live_trace() + .expect("complete live trace should be available"); + assert_eq!(live.transitions.len(), 2); + assert_eq!(live.transitions[0].action, 0); + assert_eq!(live.transitions[1].reward, 2); + + let greedy = agent.get_planned_action(); + let first_exploratory = agent.get_planned_action_with_extra_exploration(1.0); + let second_exploratory = agent.get_planned_action_with_extra_exploration(1.0); + assert_ne!( + first_exploratory, second_exploratory, + "test seed must make forced exploration distinguishable from a fixed action" + ); + assert!(first_exploratory != greedy || second_exploratory != greedy); + } + + #[cfg(feature = "backend-ctw")] + #[test] + fn warmstart_bytepacked_ctw_teacher_replay_and_live_step() { + use crate::api::BitOrder; + + let cfg = WarmStartExactJhConfig { + rate_backend: RateBackend::Ctw { depth: 8 }, + bit_stream_semantics: BitStreamSemantics::BytePacked { + order: BitOrder::MsbFirst, + }, + observation_bits: 8, + observation_stream_len: 1, + reward_bits: 8, + agent_actions: action_alphabet(256), + return_horizon: 1, + return_bins: 256, + label_phase_period: 1, + planner_simulations_per_step: 1, + random_seed: Some(42), + }; + let teacher = WarmStartExactJhTeacherDataset { + contract: teacher_for_config(&cfg).contract, + traces: vec![WarmStartExactJhTeacherTrace { + transitions: vec![WarmStartExactJhTransition { + action: 0, + observations: vec![5], + reward: 17, + }], + }], + }; + let mut agent_a = WarmStartExactJhAgent::new(cfg.clone(), teacher.clone()) + .expect("byte-packed warmstart agent"); + let mut agent_b = + WarmStartExactJhAgent::new(cfg, teacher).expect("byte-packed warmstart replay agent"); + assert_eq!(agent_a.teacher_label_count(), 1); + let planned_a = agent_a.get_planned_action(); + let planned_b = agent_b.get_planned_action(); + assert_eq!( + planned_a, planned_b, + "byte-packed warmstart planning must be deterministic under identical seed" + ); + assert!(planned_a < 256); + agent_a + .observe_transition(planned_a, &[3], 10) + .expect("byte-packed live transition"); + assert_eq!(agent_a.steps_observed(), 1); + } + + #[cfg(feature = "backend-ctw")] + #[test] + fn warmstart_binarytokens_fac_ctw_teacher_replay_and_live_step() { + let cfg = WarmStartExactJhConfig { + rate_backend: RateBackend::FacCtw { + base_depth: 8, + num_percept_bits: 8, + encoding_bits: 8, + msb_first: Some(true), + }, + bit_stream_semantics: BitStreamSemantics::BinaryTokens, + observation_bits: 2, + observation_stream_len: 1, + reward_bits: 2, + agent_actions: action_alphabet(2), + return_horizon: 1, + return_bins: 4, + label_phase_period: 1, + planner_simulations_per_step: 1, + random_seed: Some(7), + }; + let teacher = teacher_for_config(&cfg); + let mut agent_a = WarmStartExactJhAgent::new(cfg.clone(), teacher.clone()) + .expect("BinaryTokens FAC-CTW warmstart agent"); + let mut agent_b = WarmStartExactJhAgent::new(cfg, teacher) + .expect("BinaryTokens FAC-CTW warmstart replay agent"); + assert_eq!(agent_a.teacher_label_count(), 3); + let planned_a = agent_a.get_planned_action(); + let planned_b = agent_b.get_planned_action(); + assert_eq!( + planned_a, planned_b, + "BinaryTokens FAC-CTW warmstart planning must be deterministic under identical seed" + ); + agent_a + .observe_transition(planned_a, &[1], 1) + .expect("BinaryTokens FAC-CTW live transition"); + assert_eq!(agent_a.steps_observed(), 1); + } + + #[cfg(feature = "backend-ctw")] + #[test] + fn warmstart_agent_learns_teacher_labels_and_observes_live_steps() { + let mut agent = WarmStartExactJhAgent::new(config(), teacher()) + .expect("warmstart agent should initialize"); + assert_eq!(agent.teacher_label_count(), 3); + let action = agent.get_planned_action(); + assert!(action < 2); + agent + .observe_transition(action, &[1], 1) + .expect("first transition"); + assert_eq!(agent.steps_observed(), 1); + } + + #[test] + fn validate_rejects_reward_bits_too_narrow_for_derived_instantaneous_bounds() { + let mut cfg = config(); + cfg.return_horizon = 1; + cfg.label_phase_period = 1; + cfg.return_bins = 100; + cfg.reward_bits = 1; + let err = cfg + .validate() + .expect_err("derived max instantaneous reward must fit reward_bits"); + assert!(matches!(err, WarmStartExactJhError::RewardEncoding(_))); + } + + #[derive(Clone, Default)] + struct ResetSpyCounts { + reset_calls: usize, + } + + #[derive(Clone)] + struct ResetSpyPredictor { + counts: Arc>, + } + + impl Predictor for ResetSpyPredictor { + fn update(&mut self, _sym: bool) {} + + fn update_history(&mut self, _sym: bool) {} + + fn revert(&mut self) {} + + fn pop_history(&mut self) {} + + fn predict_prob(&mut self, sym: bool) -> f64 { + if sym { 0.75 } else { 0.25 } + } + + fn model_name(&self) -> String { + "ResetSpyPredictor".to_string() + } + + fn boxed_clone(&self) -> Box { + Box::new(self.clone()) + } + + fn reset_conditioning_history(&mut self) -> Result<(), String> { + self.counts + .lock() + .expect("counts mutex poisoned") + .reset_calls += 1; + Ok(()) + } + } + + #[test] + fn warmstart_resets_predictor_conditioning_between_teacher_traces() { + let counts = Arc::new(Mutex::new(ResetSpyCounts::default())); + let spy = ResetSpyPredictor { + counts: counts.clone(), + }; + + let cfg = WarmStartExactJhRuntimeConfig { + task_fingerprint: TaskFingerprint::parse_hex(TEST_TASK_FINGERPRINT_HEX) + .expect("test fingerprint"), + observation_bits: 2, + observation_stream_len: 1, + observation_key_mode: "full_stream", + reward_bits: 2, + agent_actions: action_alphabet(2), + min_reward: 0, + max_reward: 3, + reward_offset: 0, + return_horizon: 1, + return_bins: 4, + label_phase_period: 1, + planner_simulations_per_step: 1, + random_seed: 7, + provenance_policy: TeacherProvenancePolicy::StandalonePlannerRun, + }; + + let teacher = WarmStartExactJhTeacherDataset { + contract: WarmStartExactJhTeacherContract { + schema_version: 1, + task_fingerprint: TaskFingerprint::parse_hex(TEST_TASK_FINGERPRINT_HEX) + .expect("test fingerprint"), + action_alphabet_size: 2, + observation_bits: 2, + observation_stream_len: 1, + observation_key_mode: "full_stream".to_string(), + observation_adapter_spec_ref: "test-observation-adapter".to_string(), + observation_adapter_content_crc32: "test-observation-adapter-crc32".to_string(), + reward_bits: 2, + return_horizon: 1, + label_phase_period: 1, + scalar_representation: "test-scalar".to_string(), + exact_reward_encoding_certificate: "test-cert".to_string(), + }, + traces: vec![ + WarmStartExactJhTeacherTrace { + transitions: vec![WarmStartExactJhTransition { + action: 0, + observations: vec![1], + reward: 1, + }], + }, + WarmStartExactJhTeacherTrace { + transitions: vec![WarmStartExactJhTransition { + action: 1, + observations: vec![2], + reward: 2, + }], + }, + ], + }; + + let mut agent = WarmStartExactJhAgent { + config: cfg, + phases: vec![PhaseModel { + predictor: Box::new(spy), + last_augmented_step: 0, + }], + steps: Vec::new(), + return_labels_by_step: Vec::new(), + total_steps_observed: 0, + action_bits: 1, + return_label_codec: ReturnLabelCodec::value_monotone(4), + teacher_label_count: 0, + rng: RandomGenerator::from_seed(7), + }; + + agent + .warm_start_from_teacher(&teacher) + .expect("warm-start should succeed"); + + let snapshot = counts.lock().expect("counts mutex poisoned").clone(); + assert_eq!(snapshot.reset_calls, teacher.traces.len()); + assert_eq!(agent.teacher_label_count(), 2); + } +} diff --git a/crates/infotheory/src/aixi/warmstart_contract.rs b/crates/infotheory/src/aixi/warmstart_contract.rs new file mode 100644 index 00000000..7928f4c5 --- /dev/null +++ b/crates/infotheory/src/aixi/warmstart_contract.rs @@ -0,0 +1,520 @@ +//! Shared helpers for warm-start exact-J_H planner contract metadata. +//! +//! These helpers are intentionally canonicalized in one place so planner-run +//! task identity and observation-key encoding names do not drift between runtime +//! and CLI validation paths. +//! +//! Normative semantics for standalone teacher artifacts, task fingerprints, and +//! provenance declarations are specified in `docs/warmstart-exact-jh.tex`. + +use crate::aixi::common::{ObservationKeyMode, nonnegative_reward_encoding_bounds}; +use crate::spec::{AssetRef, CanonicalJson, CompiledPlannerRunSpec, canonical_json_bytes}; +use crc32fast::Hasher; +use serde_json::{Value, json}; +use sha2::{Digest, Sha256}; +use std::fmt; +use std::fs; + +pub const WARMSTART_TEACHER_CONTRACT_SCHEMA_VERSION: u64 = 1; + +/// SHA-256 task identity binding warm-start artifacts to one planner-run task. +/// +/// Wire format is exactly 64 lowercase hexadecimal digits. +/// [`Self::parse_hex`] is strict: only that form is accepted, so malformed +/// fingerprints are unrepresentable. +/// +/// Invariant: for every value produced by [`Self::from_payload_bytes`], +/// `parse_hex(&display(v)) == Some(v)`. +#[derive(Clone, Copy, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct TaskFingerprint([u8; 32]); + +impl TaskFingerprint { + /// SHA-256 digest of the canonical task-fingerprint JSON payload bytes. + pub fn from_payload_bytes(bytes: &[u8]) -> Self { + let digest = Sha256::digest(bytes); + let mut output = [0_u8; 32]; + output.copy_from_slice(&digest); + Self(output) + } + + /// Parse the canonical 64-digit lowercase hex wire form. + pub fn parse_hex(value: &str) -> Option { + if value.len() != 64 { + return None; + } + let mut bytes = [0_u8; 32]; + for (index, pair) in value.as_bytes().chunks_exact(2).enumerate() { + let high = decode_lower_hex_nibble(pair[0])?; + let low = decode_lower_hex_nibble(pair[1])?; + bytes[index] = (high << 4) | low; + } + Some(Self(bytes)) + } +} + +impl fmt::Display for TaskFingerprint { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + for byte in self.0 { + write!(f, "{byte:02x}")?; + } + Ok(()) + } +} + +impl fmt::Debug for TaskFingerprint { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "TaskFingerprint({self})") + } +} + +/// Versioned declaration string stored in `observation_adapter_spec_ref` for standalone runs. +pub const WARMSTART_STANDALONE_OBSERVATION_ADAPTER_SPEC_REF: &str = + "direct-planner-percept-lsb-first-v1"; + +/// Declared canonical scalar type for instantaneous rewards on the standalone path. +pub const WARMSTART_STANDALONE_SCALAR_REPRESENTATION: &str = + "nonnegative-integer-i64-instantaneous-reward-v1"; + +/// Structured observation adapter \(\eta_O\) for direct environment → planner percept. +/// +/// Dimensions `(observation_bits, observation_stream_len, reward_bits)` are part of the +/// committed object so teachers cannot be mixed across incompatible interface shapes. +pub fn standalone_observation_adapter_spec_value( + observation_bits: usize, + observation_stream_len: usize, + reward_bits: usize, +) -> Value { + json!({ + "kind": WARMSTART_STANDALONE_OBSERVATION_ADAPTER_SPEC_REF, + "schema_version": 1, + "observation_stream_len": observation_stream_len, + "observation_bits_per_cell": observation_bits, + "observation_cell_domain": "u64_values_bounded_by_observation_bits", + "reward_bits": reward_bits, + "reward_channel_offset": 0, + "bit_order_within_field": "lsb_first", + "reference_encoding": "crate::aixi::common::encode/encode_reward/decode_reward", + }) +} + +/// CRC32 (hex, 8 lowercase hex digits) of [`standalone_observation_adapter_spec_value`]. +pub fn standalone_observation_adapter_content_crc32( + observation_bits: usize, + observation_stream_len: usize, + reward_bits: usize, +) -> Result { + let bytes = canonical_json_bytes(&standalone_observation_adapter_spec_value( + observation_bits, + observation_stream_len, + reward_bits, + ))?; + Ok(crc32_hex(&bytes)) +} + +/// Certificate payload for identity-style \(\Omega_H\) on nonnegative integers in the reward channel. +pub fn standalone_exact_reward_encoding_certificate_value(reward_bits: usize) -> Value { + let (_min, max_channel, _offset) = nonnegative_reward_encoding_bounds(reward_bits); + json!({ + "kind": "standalone-identity-reward-encoder-v1", + "schema_version": 1, + "reward_bits": reward_bits, + "min_instantaneous_reward": 0, + "max_instantaneous_reward_channel": max_channel, + "omega": "identity_on_representable_nonnegative_integers", + "injectivity": "identity_is_injective_on_closed_interval_0_max_channel", + }) +} + +/// CRC32 (hex) of [`standalone_exact_reward_encoding_certificate_value`]. +pub fn standalone_exact_reward_encoding_certificate_hash( + reward_bits: usize, +) -> Result { + let bytes = canonical_json_bytes(&standalone_exact_reward_encoding_certificate_value( + reward_bits, + ))?; + Ok(crc32_hex(&bytes)) +} + +/// Pair `(observation_adapter_content_crc32, exact_reward_encoding_certificate)` for standalone teachers. +/// +/// Callers building [`WarmStartExactJhTeacherContract`](crate::aixi::warmstart::WarmStartExactJhTeacherContract) +/// JSON should use this helper so adapter and reward hashes stay aligned with validation. +pub fn standalone_teacher_provenance_crc32_pair( + observation_bits: usize, + observation_stream_len: usize, + reward_bits: usize, +) -> Result<(String, String), serde_json::Error> { + Ok(( + standalone_observation_adapter_content_crc32( + observation_bits, + observation_stream_len, + reward_bits, + )?, + standalone_exact_reward_encoding_certificate_hash(reward_bits)?, + )) +} + +/// Compute the stable planner-task fingerprint for an exact-J_H planner run. +/// +/// The payload includes schema-affecting fields that define dataset applicability. +/// The warm-start teacher asset reference itself is deliberately removed before +/// hashing to avoid making a same-task teacher depend on its own file path or +/// asset selector. +pub fn warmstart_exact_jh_planner_task_fingerprint( + compiled: &CompiledPlannerRunSpec, +) -> Result { + let task_asset_content_sha256 = planner_task_asset_content_commitments(compiled)?; + let planner_run_task_sha256 = warmstart_planner_task_sha256(compiled)?; + let payload = json!({ + "planner_run_task_sha256": planner_run_task_sha256, + "controller_kind": compiled.controller().kind_str(), + "controller_backend": compiled.controller().backend_label(), + "teacher_contract_schema_version": WARMSTART_TEACHER_CONTRACT_SCHEMA_VERSION, + "task_asset_content_sha256": task_asset_content_sha256, + }); + canonical_json_bytes(&payload) + .map(|bytes| TaskFingerprint::from_payload_bytes(&bytes)) + .map_err(|err| format!("failed to encode warm-start task fingerprint payload: {err}")) +} + +fn warmstart_teacher_asset_id(compiled: &CompiledPlannerRunSpec) -> Option<&str> { + match compiled.controller() { + crate::spec::CompiledPlannerController::AiqiWarmstartExactJh { + teacher_dataset_asset, + .. + } => Some(teacher_dataset_asset.as_str()), + _ => None, + } +} + +fn warmstart_planner_task_sha256(compiled: &CompiledPlannerRunSpec) -> Result { + let teacher_asset = warmstart_teacher_asset_id(compiled); + let mut value = compiled + .canonical_spec() + .to_canonical_json_value() + .map_err(|err| format!("failed to encode planner task JSON: {err}"))?; + if let Value::Object(root) = &mut value { + if let Some(Value::Array(assets)) = root.get_mut("assets") { + assets.retain(|asset| { + asset + .get("id") + .and_then(Value::as_str) + .is_none_or(|id| Some(id) != teacher_asset) + }); + } + if let Some(Value::Object(controller)) = root.get_mut("controller") { + controller.remove("teacher_dataset_asset"); + } + } + canonical_json_bytes(&value) + .map(|bytes| sha256_hex(&bytes)) + .map_err(|err| format!("failed to encode warm-start planner task payload: {err}")) +} + +fn planner_task_asset_content_commitments( + compiled: &CompiledPlannerRunSpec, +) -> Result, String> { + let teacher_asset = warmstart_teacher_asset_id(compiled); + let mut commitments = Vec::::new(); + for binding in compiled.resolved_assets() { + if teacher_asset == Some(binding.id.as_str()) { + continue; + } + let AssetRef::Filesystem(path) = &binding.asset; + let bytes = fs::read(path).map_err(|err| { + format!( + "failed to read task asset '{}' for warm-start task fingerprint: {err}", + path.display() + ) + })?; + commitments.push(json!({ + "id": binding.id.as_str(), + "content_sha256": sha256_hex(&bytes), + })); + } + commitments.sort_by(|left, right| { + left["id"] + .as_str() + .unwrap_or_default() + .cmp(right["id"].as_str().unwrap_or_default()) + }); + Ok(commitments) +} + +/// Convert planner observation keying mode to the canonical contract string. +pub fn observation_key_mode_name(mode: ObservationKeyMode) -> &'static str { + match mode { + ObservationKeyMode::First => "first", + ObservationKeyMode::Last => "last", + ObservationKeyMode::StreamHash => "stream_hash", + ObservationKeyMode::FullStream => "full_stream", + } +} + +fn crc32_hex(bytes: &[u8]) -> String { + let mut hasher = Hasher::new(); + hasher.update(bytes); + format!("{:08x}", hasher.finalize()) +} + +fn sha256_hex(bytes: &[u8]) -> String { + let digest = Sha256::digest(bytes); + let mut output = String::with_capacity(64); + for byte in digest { + use std::fmt::Write as _; + write!(&mut output, "{byte:02x}").expect("writing to String cannot fail"); + } + output +} + +fn decode_lower_hex_nibble(byte: u8) -> Option { + match byte { + b'0'..=b'9' => Some(byte - b'0'), + b'a'..=b'f' => Some(byte - b'a' + 10), + _ => None, + } +} + +#[cfg(all(test, feature = "backend-ctw"))] +mod tests { + use super::{ + TaskFingerprint, standalone_teacher_provenance_crc32_pair, + warmstart_exact_jh_planner_task_fingerprint, + }; + use crate::aixi::common::{ActionAlphabet, ObservationKeyMode}; + use crate::api::{BitStreamSemantics, RateBackend}; + use crate::spec::{ + AssetBinding, BuiltinEnvironmentSpec, CanonicalJson, ControllerSpec, EnvironmentSpec, + PlannerInterfaceSpec, PlannerRunSpec, PlannerRuntimeSpec, SpecDocument, SpecEnvironment, + WarmStartExactJhControllerSpec, + }; + use std::path::{Path, PathBuf}; + use std::time::{SystemTime, UNIX_EPOCH}; + + fn temp_dir(label: &str) -> PathBuf { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("clock") + .as_nanos(); + std::env::temp_dir().join(format!( + "infotheory-warmstart-contract-{label}-{}-{nanos}", + std::process::id() + )) + } + + fn action_alphabet(n: usize) -> ActionAlphabet { + ActionAlphabet::try_from_usize(n).expect("test action alphabet must be non-zero") + } + + fn write_asset(dir: &Path, name: &str, bytes: &[u8]) { + std::fs::write(dir.join(name), bytes).expect("write asset"); + } + + fn sample_exact_jh_spec() -> PlannerRunSpec { + PlannerRunSpec { + assets: vec![ + AssetBinding { + id: "teacher".to_string(), + path: "teacher.json".to_string(), + }, + AssetBinding { + id: "task_input".to_string(), + path: "task_input.bin".to_string(), + }, + ], + environment: EnvironmentSpec::Builtin { + builtin: BuiltinEnvironmentSpec::CoinFlip, + }, + interface: PlannerInterfaceSpec { + observation_bits: 2, + observation_stream_len: 1, + observation_key_mode: ObservationKeyMode::FullStream, + reward_bits: 2, + agent_actions: action_alphabet(2), + }, + controller: ControllerSpec::AiqiWarmstartExactJh(WarmStartExactJhControllerSpec { + predictor: RateBackend::Ctw { depth: 4 }, + bit_stream_semantics: BitStreamSemantics::BinaryTokens, + return_horizon: 1, + return_bins: 4, + label_phase_period: 1, + teacher_dataset_asset: "teacher".to_string(), + planner_simulations_per_step: 1, + }), + runtime: PlannerRuntimeSpec { + random_seed: Some(7), + learn_cycles: Some(1), + eval_cycles: Some(1), + terminate_lifetime: 2, + log_every: 1, + perf: false, + vm_perf_only: false, + explore_epsilon: 0.0, + explore_gamma: 1.0, + }, + } + } + + fn compile_in_dir(spec: PlannerRunSpec, dir: &Path) -> crate::spec::CompiledPlannerRunSpec { + let value = SpecDocument::PlannerRun(spec) + .to_canonical_json_value() + .expect("canonical json"); + let document = + SpecDocument::parse_json_value(&value, dir).expect("parse warmstart planner run"); + let SpecDocument::PlannerRun(parsed) = document else { + panic!("expected planner_run document"); + }; + parsed + .compile_in(&SpecEnvironment::new(dir)) + .expect("compile warmstart planner run") + } + + fn committed_non_teacher_asset_ids( + compiled: &crate::spec::CompiledPlannerRunSpec, + ) -> Vec { + let teacher_asset = match compiled.controller() { + crate::spec::CompiledPlannerController::AiqiWarmstartExactJh { + teacher_dataset_asset, + .. + } => Some(teacher_dataset_asset.as_str()), + _ => None, + }; + compiled + .resolved_assets() + .iter() + .filter(|binding| Some(binding.id.as_str()) != teacher_asset) + .map(|binding| binding.id.clone()) + .collect() + } + + #[test] + fn exact_jh_warmstart_fingerprint_golden_values_are_stable() { + let (adapter_crc, reward_certificate) = + standalone_teacher_provenance_crc32_pair(1, 1, 1).expect("standalone provenance"); + assert_eq!(adapter_crc, "30267f35"); + assert_eq!(reward_certificate, "e09613fc"); + + let dir = temp_dir("exact-jh-fingerprint-golden"); + std::fs::create_dir_all(&dir).expect("create temp dir"); + write_asset( + &dir, + "teacher.json", + br#"{"schema_version":1,"contract":{},"traces":[]}"#, + ); + write_asset(&dir, "task_input.bin", b"task-input-v1"); + + let compiled = compile_in_dir(sample_exact_jh_spec(), &dir); + let fingerprint = warmstart_exact_jh_planner_task_fingerprint(&compiled) + .expect("task fingerprint") + .to_string(); + assert_eq!( + fingerprint, + "34c801345a415878e31ccfc860ccefe77f22150721ab2989135fcefd3b7f2427" + ); + + let _ = std::fs::remove_dir_all(dir); + } + + #[test] + fn exact_jh_task_fingerprint_excludes_teacher_asset_path_and_content() { + let dir = temp_dir("exact-jh-teacher-exclusion"); + std::fs::create_dir_all(&dir).expect("create temp dir"); + write_asset( + &dir, + "teacher.json", + br#"{"schema_version":1,"contract":{},"traces":[]}"#, + ); + write_asset(&dir, "task_input.bin", b"task-input-v1"); + + let compiled = compile_in_dir(sample_exact_jh_spec(), &dir); + let baseline = + warmstart_exact_jh_planner_task_fingerprint(&compiled).expect("baseline fingerprint"); + let committed = committed_non_teacher_asset_ids(&compiled); + assert!( + !committed.iter().any(|id| id == "teacher"), + "teacher asset must be excluded from fingerprint commitments: {committed:?}" + ); + assert!( + committed.iter().any(|id| id == "task_input"), + "non-teacher assets must remain committed: {committed:?}" + ); + + write_asset( + &dir, + "teacher.json", + br#"{"schema_version":1,"contract":{"task_fingerprint":"mutated"},"traces":[]}"#, + ); + let after_teacher_content = compile_in_dir(sample_exact_jh_spec(), &dir); + let after_content = warmstart_exact_jh_planner_task_fingerprint(&after_teacher_content) + .expect("fingerprint after teacher content mutation"); + assert_eq!( + baseline, after_content, + "mutating teacher asset bytes must not change task fingerprint" + ); + + let mut moved_teacher_spec = sample_exact_jh_spec(); + moved_teacher_spec.assets[0].path = "teacher_moved.json".to_string(); + write_asset( + &dir, + "teacher_moved.json", + br#"{"schema_version":1,"contract":{},"traces":[]}"#, + ); + let after_teacher_path = compile_in_dir(moved_teacher_spec, &dir); + let after_path = warmstart_exact_jh_planner_task_fingerprint(&after_teacher_path) + .expect("fingerprint after teacher path mutation"); + assert_eq!( + baseline, after_path, + "moving teacher asset path must not change task fingerprint" + ); + + write_asset(&dir, "task_input.bin", b"task-input-v2"); + let after_input_mutation = compile_in_dir(sample_exact_jh_spec(), &dir); + let after_input = warmstart_exact_jh_planner_task_fingerprint(&after_input_mutation) + .expect("fingerprint after non-teacher asset mutation"); + assert_ne!( + baseline, after_input, + "mutating committed non-teacher asset bytes must change task fingerprint" + ); + + let _ = std::fs::remove_dir_all(dir); + } + + #[test] + fn task_fingerprint_display_parse_round_trip() { + let fingerprint = TaskFingerprint::from_payload_bytes(b"warm-start-task-payload"); + let wire = fingerprint.to_string(); + assert_eq!(wire.len(), 64); + assert!( + wire.bytes() + .all(|byte| matches!(byte, b'0'..=b'9' | b'a'..=b'f')) + ); + assert_eq!(TaskFingerprint::parse_hex(&wire), Some(fingerprint)); + } + + #[test] + fn task_fingerprint_parse_hex_rejects_non_canonical_forms() { + assert!(TaskFingerprint::parse_hex("").is_none()); + let too_short = "0".repeat(63); + let too_long = "0".repeat(65); + assert!(TaskFingerprint::parse_hex(&too_short).is_none()); + assert!(TaskFingerprint::parse_hex(&too_long).is_none()); + assert!( + TaskFingerprint::parse_hex( + "ABCDEF0123456789abcdef0123456789abcdef0123456789abcdef0123456789" + ) + .is_none() + ); + assert!( + TaskFingerprint::parse_hex( + "gggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggg" + ) + .is_none() + ); + assert!( + TaskFingerprint::parse_hex( + "000000000000000000000000000000000000000000000000000000000000000g" + ) + .is_none() + ); + } +} diff --git a/crates/infotheory/src/api/compression.rs b/crates/infotheory/src/api/compression.rs new file mode 100644 index 00000000..abb8ab4b --- /dev/null +++ b/crates/infotheory/src/api/compression.rs @@ -0,0 +1,353 @@ +//! Compression-focused public API surface. + +use rayon::prelude::*; + +use crate::error::{InfotheoryError, InfotheoryResult}; +use crate::spec::CompiledCompressionBackend; + +use crate::runtime::CompressionRuntime; +use crate::with_default_ctx; + +/// Per-call control over operation-level parallelism. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub enum OperationParallelism { + /// Execute operation-level work serially. + Serial, + /// Use adaptive/default parallel operation behavior. + #[default] + Auto, + /// Execute operation-level work on a bounded Rayon pool with `threads`. + Threads(usize), +} + +/// NCD compute options (operation-level parallelism only). +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct NcdComputeOptions { + /// Operation-level parallelism policy used while computing NCD values. (external parellization, doesn't affect compression algorithm itself) + pub parallelism: OperationParallelism, +} + +/// Compute compressed size (bytes) for a logical concatenation of `parts` using `backend`. +pub fn try_compress_size_chain_backend( + parts: &[&[u8]], + backend: &CompiledCompressionBackend, +) -> InfotheoryResult { + let mut runtime = crate::runtime::build_compression_runtime(backend) + .map_err(InfotheoryError::invalid_backend_config)?; + runtime.compress_size_chain(parts) +} + +/// Compute compressed size (bytes) for `data` using `backend`. +pub fn try_compress_size_backend( + data: &[u8], + backend: &CompiledCompressionBackend, +) -> InfotheoryResult { + let mut runtime = crate::runtime::build_compression_runtime(backend) + .map_err(InfotheoryError::invalid_backend_config)?; + runtime.compress_size(data) +} + +/// Compress `data` with `backend` and return encoded bytes. +pub fn try_compress_bytes_backend( + data: &[u8], + backend: &CompiledCompressionBackend, +) -> InfotheoryResult> { + let mut runtime = crate::runtime::build_compression_runtime(backend) + .map_err(InfotheoryError::invalid_backend_config)?; + runtime.compress_bytes(data) +} + +/// Decompress `input` with `backend` and return decoded bytes. +pub fn try_decompress_bytes_backend( + input: &[u8], + backend: &CompiledCompressionBackend, +) -> InfotheoryResult> { + let mut runtime = crate::runtime::build_compression_runtime(backend) + .map_err(InfotheoryError::invalid_backend_config)?; + runtime.decompress_bytes(input) +} + +/// Normalized compression-distance formula variant. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum NcdVariant { + /// Vitanyi-style NCD: `(C(xy) - min(C(x), C(y))) / max(C(x), C(y))`. + Vitanyi, + /// Symmetric Vitanyi-style NCD using `min(C(xy), C(yx))`. + SymVitanyi, + /// Constructive NCD: `(C(xy) - min(C(x), C(y))) / C(xy)`. + Cons, + /// Symmetric constructive NCD using `min(C(xy), C(yx))` as denominator. + SymCons, +} + +#[inline(always)] +fn ncd_from_sizes(cx: u64, cy: u64, cxy: u64, cyx: Option, variant: NcdVariant) -> f64 { + let min_c = cx.min(cy) as f64; + let max_c = cx.max(cy) as f64; + + match variant { + NcdVariant::Vitanyi => { + if max_c == 0.0 { + 0.0 + } else { + (cxy as f64 - min_c) / max_c + } + } + NcdVariant::SymVitanyi => { + let m = cxy.min(cyx.expect("cyx required for SymVitanyi")) as f64; + if max_c == 0.0 { + 0.0 + } else { + (m - min_c) / max_c + } + } + NcdVariant::Cons => { + let denom = cxy as f64; + if denom == 0.0 { + 0.0 + } else { + (cxy as f64 - min_c) / denom + } + } + NcdVariant::SymCons => { + let m = cxy.min(cyx.expect("cyx required for SymCons")) as f64; + if m == 0.0 { 0.0 } else { (m - min_c) / m } + } + } +} + +#[inline(always)] +/// Compute NCD for byte slices using the thread-local default context. +pub fn try_ncd_bytes_default(x: &[u8], y: &[u8], variant: NcdVariant) -> InfotheoryResult { + with_default_ctx(|ctx| ctx.try_ncd_bytes(x, y, variant)) +} + +/// Compute NCD for byte slices with an explicit compression backend. +pub fn try_ncd_bytes_backend( + x: &[u8], + y: &[u8], + backend: &CompiledCompressionBackend, + variant: NcdVariant, +) -> InfotheoryResult { + try_ncd_bytes_backend_with_options(x, y, backend, variant, NcdComputeOptions::default()) +} + +/// Compute NCD for byte slices with an explicit compression backend and +/// explicit operation-level parallelism controls. +pub fn try_ncd_bytes_backend_with_options( + x: &[u8], + y: &[u8], + backend: &CompiledCompressionBackend, + variant: NcdVariant, + options: NcdComputeOptions, +) -> InfotheoryResult { + let compute = || -> InfotheoryResult { + let (cx, cy) = rayon::join( + || try_compress_size_backend(x, backend), + || try_compress_size_backend(y, backend), + ); + let cx = cx?; + let cy = cy?; + + let cxy = try_compress_size_chain_backend(&[x, y], backend)?; + + let cyx = match variant { + NcdVariant::SymVitanyi | NcdVariant::SymCons => { + Some(try_compress_size_chain_backend(&[y, x], backend)?) + } + _ => None, + }; + + Ok(ncd_from_sizes(cx, cy, cxy, cyx, variant)) + }; + + match options.parallelism { + OperationParallelism::Serial => { + let cx = try_compress_size_backend(x, backend)?; + let cy = try_compress_size_backend(y, backend)?; + let cxy = try_compress_size_chain_backend(&[x, y], backend)?; + let cyx = match variant { + NcdVariant::SymVitanyi | NcdVariant::SymCons => { + Some(try_compress_size_chain_backend(&[y, x], backend)?) + } + _ => None, + }; + Ok(ncd_from_sizes(cx, cy, cxy, cyx, variant)) + } + OperationParallelism::Auto => compute(), + OperationParallelism::Threads(threads) => { + if threads <= 1 { + return try_ncd_bytes_backend_with_options( + x, + y, + backend, + variant, + NcdComputeOptions { + parallelism: OperationParallelism::Serial, + }, + ); + } + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(threads) + .build() + .map_err(|err| { + InfotheoryError::runtime(format!("failed to build rayon pool: {err}")) + })?; + pool.install(compute) + } + } +} + +/// Compute an `n x n` pairwise NCD matrix (row-major) using the thread-local default context. +/// +/// `out[i * n + j]` corresponds to `NCD(datas[i], datas[j])`. +pub fn try_ncd_matrix_bytes_default( + datas: &[Vec], + variant: NcdVariant, +) -> InfotheoryResult> { + with_default_ctx(|ctx| try_ncd_matrix_bytes_backend(datas, &ctx.compression_backend, variant)) +} + +/// Compute an `n x n` pairwise NCD matrix (row-major) with an explicit compression backend. +/// +/// `out[i * n + j]` corresponds to `NCD(datas[i], datas[j])`. +pub fn try_ncd_matrix_bytes_backend( + datas: &[Vec], + backend: &CompiledCompressionBackend, + variant: NcdVariant, +) -> InfotheoryResult> { + try_ncd_matrix_bytes_backend_with_options(datas, backend, variant, NcdComputeOptions::default()) +} + +/// Compute an `n x n` pairwise NCD matrix with explicit operation-level +/// parallelism controls. +pub fn try_ncd_matrix_bytes_backend_with_options( + datas: &[Vec], + backend: &CompiledCompressionBackend, + variant: NcdVariant, + options: NcdComputeOptions, +) -> InfotheoryResult> { + let compute = || try_ncd_matrix_bytes_backend_impl(datas, backend, variant); + match options.parallelism { + OperationParallelism::Serial => { + try_ncd_matrix_bytes_backend_serial(datas, backend, variant) + } + OperationParallelism::Auto => compute(), + OperationParallelism::Threads(threads) => { + if threads <= 1 { + return try_ncd_matrix_bytes_backend_serial(datas, backend, variant); + } + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(threads) + .build() + .map_err(|err| { + InfotheoryError::runtime(format!("failed to build rayon pool: {err}")) + })?; + pool.install(compute) + } + } +} + +fn try_ncd_matrix_bytes_backend_impl( + datas: &[Vec], + backend: &CompiledCompressionBackend, + variant: NcdVariant, +) -> InfotheoryResult> { + let n = datas.len(); + let cx = datas + .par_iter() + .map(|d| try_compress_size_backend(d, backend)) + .collect::>() + .into_iter() + .collect::>>()?; + + let mut out = vec![0.0f64; n * n]; + + match variant { + NcdVariant::SymVitanyi | NcdVariant::SymCons => { + let pairs = (0..n) + .flat_map(|i| (i + 1..n).map(move |j| (i, j))) + .collect::>(); + let pair_results = pairs + .into_par_iter() + .map(|(i, j)| -> InfotheoryResult<(usize, usize, f64)> { + let x = &datas[i]; + let y = &datas[j]; + let cxy = try_compress_size_chain_backend(&[x, y], backend)?; + let cyx = try_compress_size_chain_backend(&[y, x], backend)?; + + let d = ncd_from_sizes(cx[i], cx[j], cxy, Some(cyx), variant); + Ok((i, j, d)) + }) + .collect::>(); + for entry in pair_results { + let (i, j, d) = entry?; + out[i * n + j] = d; + out[j * n + i] = d; + } + } + NcdVariant::Vitanyi | NcdVariant::Cons => { + let rows = (0..n) + .into_par_iter() + .map(|i| -> InfotheoryResult> { + let x = &datas[i]; + let mut row = Vec::with_capacity(n); + for j in 0..n { + let d = if i == j { + 0.0 + } else { + let y = &datas[j]; + let cxy = try_compress_size_chain_backend(&[x, y], backend)?; + ncd_from_sizes(cx[i], cx[j], cxy, None, variant) + }; + row.push((i, j, d)); + } + Ok(row) + }) + .collect::>(); + for row in rows { + for (i, j, d) in row? { + out[i * n + j] = d; + } + } + } + } + + Ok(out) +} + +fn try_ncd_matrix_bytes_backend_serial( + datas: &[Vec], + backend: &CompiledCompressionBackend, + variant: NcdVariant, +) -> InfotheoryResult> { + let n = datas.len(); + let mut cx = Vec::with_capacity(n); + for d in datas { + cx.push(try_compress_size_backend(d, backend)?); + } + let mut out = vec![0.0f64; n * n]; + for i in 0..n { + for j in 0..n { + if i == j { + out[i * n + j] = 0.0; + continue; + } + let cxy = try_compress_size_chain_backend( + &[datas[i].as_slice(), datas[j].as_slice()], + backend, + )?; + let cyx = match variant { + NcdVariant::SymVitanyi | NcdVariant::SymCons => { + Some(try_compress_size_chain_backend( + &[datas[j].as_slice(), datas[i].as_slice()], + backend, + )?) + } + _ => None, + }; + out[i * n + j] = ncd_from_sizes(cx[i], cx[j], cxy, cyx, variant); + } + } + Ok(out) +} diff --git a/crates/infotheory/src/api/context.rs b/crates/infotheory/src/api/context.rs new file mode 100644 index 00000000..2d50a721 --- /dev/null +++ b/crates/infotheory/src/api/context.rs @@ -0,0 +1,1600 @@ +//! Stateful context and session API surface. + +use super::compression::{NcdVariant, try_ncd_bytes_backend}; +use super::generation::{GenerationRng, pick_generated_byte, try_generate_rate_backend_chain}; +use super::metrics::{ + empirical_entropy_bytes, try_biased_entropy_rate_backend, try_cross_entropy_rate_backend, + try_entropy_rate_backend, try_joint_entropy_rate_backend, try_mutual_information_rate_backend, + try_ned_rate_backend, try_nte_rate_backend, +}; +use super::types::{CompressionBackend, GenerationConfig, GenerationUpdateMode, RateBackend}; +use crate::aligned_prefix; +use crate::error::{InfotheoryError, InfotheoryResult}; +use crate::mixture::{OnlineBytePredictor, RateBackendPredictorCheckpoint}; +use crate::prediction::{ + BinaryPrediction, BitOrder, BitStreamSemantics, BytePrefixMass, + binary_prediction_from_log_probs, +}; +use crate::spec::{CanonicalBytes, CompiledCompressionBackend, CompiledRateBackend}; + +/// Returns the current default information theory context for this thread. +pub fn get_default_ctx() -> InfotheoryResult { + crate::get_default_ctx() +} + +/// Sets the current default information theory context for this thread. +pub fn set_default_ctx(ctx: InfotheoryCtx) { + crate::set_default_ctx(ctx); +} + +/// Reusable execution context holding default rate and compression backends. +#[derive(Clone)] +pub struct InfotheoryCtx { + /// Default rate backend for entropy/rate metrics. + pub rate_backend: CompiledRateBackend, + /// Default compression backend for NCD/compression primitives. + pub compression_backend: CompiledCompressionBackend, +} + +/// Stateful rate-backend session for fitting, conditioning, and continuation. +pub struct RateBackendSession { + predictor: crate::mixture::RateBackendPredictor, +} + +/// Stateful bit-level session over a rate backend. +/// +/// Byte-packed sessions keep the underlying backend byte-native and expose it +/// through a lazy prefix-mass view. They therefore require whole-byte stream +/// boundaries: `total_bits`, when provided, must be a multiple of `8`, and +/// `finish` must not leave a dangling partial byte. Within one buffered byte, +/// callers must also stay within either adaptive updates or conditioning-only +/// updates; switching modes mid-byte is rejected because the backend only +/// commits whole-byte symbols. Binary-token sessions model each bit either +/// through the backend's native binary-token application or, for byte-native +/// backends, by adapting the predictor to the literal byte symbols `0` and `1` +/// and renormalizing those two choices. +/// +/// The complete checkpoint/restore contract is available in both Rust and +/// Python (`infotheory_rs.RateBackendBitSession` with +/// `RateBackendBitSessionCheckpoint`). +#[derive(Clone)] +pub struct RateBackendBitSession { + backend_code: CanonicalBytes, + predictor: crate::mixture::RateBackendPredictor, + semantics: BitStreamSemantics, + min_prob: f64, + prefix: Option, + discardable_scopes: usize, +} + +/// Opaque checkpoint for restoring a [`RateBackendBitSession`]. +/// +/// Checkpoints capture the underlying rate predictor plus any in-flight +/// byte-prefix state, so they are valid even between byte-packed bits before a +/// full byte has been committed to the backend. +/// +/// Snapshot-backed predictors restore their predictive state exactly. Compact +/// journaled predictors may replay reversible markers during restore, so their +/// floating-point probabilities are restored up to normal round-off while the +/// discrete model state and stream position are restored to the checkpoint. +/// +/// (Python exposes this as `infotheory_rs.RateBackendBitSessionCheckpoint`.) +#[derive(Clone)] +pub struct RateBackendBitSessionCheckpoint { + backend_code: CanonicalBytes, + predictor: crate::mixture::RateBackendPredictorCheckpoint, + semantics: BitStreamSemantics, + prefix: Option, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum BufferedByteUpdateMode { + Adaptive, + Frozen, +} + +impl BufferedByteUpdateMode { + fn verb(self) -> &'static str { + match self { + Self::Adaptive => "adaptive", + Self::Frozen => "conditioning-only", + } + } +} + +#[derive(Clone)] +struct BufferedBytePrefix { + kind: BufferedBytePrefixKind, + update_mode: Option, +} + +#[derive(Clone)] +// `BytePrefixMass` is the resident prefix tree for the byte-packed path and is +// intentionally inline. Native-MSB start checkpoints are boxed below so the +// small native state does not inline a full predictor checkpoint. +#[allow(clippy::large_enum_variant)] +enum BufferedBytePrefixKind { + Mass(BytePrefixMass), + NativeMsb { + // Boxed so partial-byte abort/replay stores a checkpoint out-of-line; + // the checkpoint enum itself is pointer-sized (Full holds + // Box). Allocation is paid only when a native-MSB + // prefix needs rollback across lifecycle reset or frozen byte replay. + start_checkpoint: Option>, + symbol: u8, + bits: usize, + }, +} + +impl BufferedBytePrefix { + fn new_mass(mass: BytePrefixMass) -> Self { + Self { + kind: BufferedBytePrefixKind::Mass(mass), + update_mode: None, + } + } + + fn new_native_msb(start_checkpoint: Option) -> Self { + Self { + kind: BufferedBytePrefixKind::NativeMsb { + start_checkpoint: start_checkpoint.map(Box::new), + symbol: 0, + bits: 0, + }, + update_mode: None, + } + } + + fn has_partial_bits(&self) -> bool { + match &self.kind { + BufferedBytePrefixKind::Mass(mass) => mass.has_partial_bits(), + BufferedBytePrefixKind::NativeMsb { bits, .. } => *bits > 0, + } + } + + fn record_mode(&mut self, requested: BufferedByteUpdateMode) -> InfotheoryResult<()> { + if let Some(active) = self.update_mode + && self.has_partial_bits() + && active != requested + { + return Err(InfotheoryError::runtime(format!( + "byte-packed bit sessions cannot mix {} and {} updates within the same buffered byte; finish the byte with one mode, use `try_observe_bit`/`try_condition_bit` to handle this error explicitly, or switch to BitStreamSemantics::BinaryTokens for mid-byte mode changes", + active.verb(), + requested.verb(), + ))); + } + self.update_mode = Some(requested); + Ok(()) + } +} + +fn byte_packed_total_symbols(total_bits: Option) -> Result, String> { + let Some(total_bits) = total_bits else { + return Ok(None); + }; + if total_bits % 8 != 0 { + return Err(format!( + "byte-packed bit streams require a whole number of bytes; got {total_bits} bits. Use BitStreamSemantics::BinaryTokens for arbitrary-length bit streams" + )); + } + Ok(Some(total_bits / 8)) +} + +fn total_symbols_for_bit_semantics( + total_bits: Option, + semantics: BitStreamSemantics, +) -> Result, String> { + match semantics { + BitStreamSemantics::BytePacked { .. } => byte_packed_total_symbols(total_bits), + BitStreamSemantics::BinaryTokens => Ok(total_bits), + } +} + +impl RateBackendBitSession { + fn observed_native_prefix_bit(symbol: u8, bit_idx: usize) -> bool { + (symbol & (1u8 << (7 - bit_idx))) != 0 + } + + /// Restore to `start`, abort empty native MSB prefix, then unconditionally discard `start`. + /// + /// Discard only adjusts journal depth; predictor state is already at `start` after restore. + fn restore_start_checkpoint_abort_and_discard( + &mut self, + start: RateBackendPredictorCheckpoint, + ) -> Result { + self.predictor.restore_checkpoint(&start); + let abort_res = self.predictor.abort_empty_native_msb_byte_prefix(); + self.predictor.discard_checkpoint(start); + abort_res + } + + fn release_inflight_prefix_checkpoint_for_restore(&mut self) { + let Some(prefix) = self.prefix.take() else { + return; + }; + if let BufferedBytePrefixKind::NativeMsb { + start_checkpoint: Some(checkpoint), + .. + } = prefix.kind + { + self.predictor.discard_checkpoint(*checkpoint); + } + } + + fn discard_inflight_prefix_checkpoint(&mut self) { + let Some(prefix) = self.prefix.take() else { + return; + }; + match prefix.kind { + BufferedBytePrefixKind::Mass(_) => {} + BufferedBytePrefixKind::NativeMsb { + start_checkpoint: Some(checkpoint), + .. + } => { + self.restore_start_checkpoint_abort_and_discard(*checkpoint) + .expect("native MSB prefix start checkpoint must be empty"); + } + BufferedBytePrefixKind::NativeMsb { + start_checkpoint: None, + bits, + .. + } => { + if bits == 0 { + self.predictor + .abort_empty_native_msb_byte_prefix() + .expect("empty native MSB prefix abort must succeed"); + } else { + panic!( + "discarding an adaptive native MSB byte-prefix with observed bits; \ + missing prefix start checkpoint" + ); + } + } + } + } + + /// Make clearing checkpoint journals preserve the byte-prefix invariant: + /// `NativeMsb` session state exists only while the predictor has an active + /// native MSB prefix. Partial native prefixes are downgraded to the generic + /// mass prefix before clearing so their rollback checkpoint can be released. + fn normalize_prefix_for_checkpoint_clear(&mut self) -> InfotheoryResult { + let Some(prefix) = self.prefix.take() else { + return Ok(true); + }; + let update_mode = prefix.update_mode; + match prefix.kind { + BufferedBytePrefixKind::Mass(mass) => { + self.prefix = Some(BufferedBytePrefix { + kind: BufferedBytePrefixKind::Mass(mass), + update_mode, + }); + Ok(true) + } + BufferedBytePrefixKind::NativeMsb { + start_checkpoint: None, + bits: 0, + .. + } => { + self.predictor + .abort_empty_native_msb_byte_prefix() + .map_err(InfotheoryError::runtime)?; + Ok(true) + } + BufferedBytePrefixKind::NativeMsb { + start_checkpoint: Some(checkpoint), + symbol, + bits, + } => { + let checkpoint = *checkpoint; + self.restore_start_checkpoint_abort_and_discard(checkpoint) + .map_err(InfotheoryError::runtime)?; + + if bits == 0 { + return Ok(true); + } + + let mut logps = [0.0f64; 256]; + self.predictor.fill_log_probs(&mut logps); + + let mut mass = BytePrefixMass::from_log_probs(&logps, BitOrder::MsbFirst); + for bit_idx in 0..bits { + mass.observe(Self::observed_native_prefix_bit(symbol, bit_idx)); + } + self.prefix = Some(BufferedBytePrefix { + kind: BufferedBytePrefixKind::Mass(mass), + update_mode, + }); + Ok(true) + } + BufferedBytePrefixKind::NativeMsb { + start_checkpoint: None, + symbol, + bits, + } => { + self.prefix = Some(BufferedBytePrefix { + kind: BufferedBytePrefixKind::NativeMsb { + start_checkpoint: None, + symbol, + bits, + }, + update_mode, + }); + Ok(false) + } + } + } + + /// Create a bit session from an explicit compiled backend. + /// + /// For [`BitStreamSemantics::BytePacked`], `total_bits` must be `None` or a + /// multiple of `8`. + pub fn from_backend( + backend: CompiledRateBackend, + total_bits: Option, + semantics: BitStreamSemantics, + ) -> InfotheoryResult { + Self::from_backend_with_min_prob( + backend, + total_bits, + semantics, + crate::mixture::DEFAULT_MIN_PROB, + ) + } + + pub(crate) fn from_backend_with_min_prob( + backend: CompiledRateBackend, + total_bits: Option, + semantics: BitStreamSemantics, + min_prob: f64, + ) -> InfotheoryResult { + let total_symbols = total_symbols_for_bit_semantics(total_bits, semantics) + .map_err(InfotheoryError::runtime)?; + let mut predictor = match semantics { + BitStreamSemantics::BinaryTokens => { + crate::runtime::build_rate_backend_binary_token_predictor(&backend, min_prob) + } + BitStreamSemantics::BytePacked { .. } => { + if !backend.supports_byte_prefix_mass() { + return Err(InfotheoryError::invalid_backend_config(format!( + "backend '{}' does not support BitStreamSemantics::BytePacked", + backend.canonical_name() + ))); + } + if !backend.supports_efficient_byte_packed_bit_sessions() { + return Err(InfotheoryError::invalid_backend_config(format!( + "backend '{}' can expose byte probabilities but does not support efficient BitStreamSemantics::BytePacked sessions; use BitStreamSemantics::BinaryTokens or a backend with native or cached byte-prefix support", + backend.canonical_name() + ))); + } + crate::runtime::build_rate_backend_predictor(&backend, min_prob) + } + } + .map_err(InfotheoryError::invalid_backend_config)?; + predictor + .begin_stream(total_symbols) + .map_err(InfotheoryError::runtime)?; + let backend_code = backend.canonical_bytes().clone(); + Ok(Self { + backend_code, + predictor, + semantics, + min_prob, + prefix: None, + discardable_scopes: 0, + }) + } + + /// Create a bit session from a wrapper backend spec. + /// + /// For [`BitStreamSemantics::BytePacked`], `total_bits` must be `None` or a + /// multiple of `8`. + pub fn from_spec( + backend: RateBackend, + total_bits: Option, + semantics: BitStreamSemantics, + ) -> InfotheoryResult { + let compiled = backend + .compile() + .map_err(|err| InfotheoryError::invalid_backend_config(err.to_string()))?; + Self::from_backend(compiled, total_bits, semantics) + } + + /// Predict the next bit without updating state. + pub fn predict_bit(&mut self) -> BinaryPrediction { + match self.semantics { + BitStreamSemantics::BinaryTokens => binary_prediction_from_log_probs( + self.predictor.log_prob(0), + self.predictor.log_prob(1), + self.min_prob, + ), + BitStreamSemantics::BytePacked { order } => { + if self.prefix.is_none() + && order == BitOrder::MsbFirst + && self.predictor.has_native_msb_byte_prefix() + { + self.try_begin_native_msb_prefix(None).unwrap_or_else(|err| { + panic!( + "RateBackendPredictor failed to begin native MSB byte-prefix: {err} \ + (contract violation; BytePrefixMass fallback is not safe on Err)" + ) + }); + } + self.ensure_mass_prefix(order); + let native_bits = match &self.prefix.as_ref().expect("prefix initialized").kind { + BufferedBytePrefixKind::Mass(mass) => return mass.prediction(), + BufferedBytePrefixKind::NativeMsb { bits, .. } => *bits, + }; + let p1 = self + .predictor + .native_msb_prefix_prob_one(native_bits) + .expect( + "native_msb_prefix_prob_one failed or returned error for active NativeMsb \ + prefix (RateBackendBitSession invariant; predictors must uphold finite \ + contract or report via Result consistently)", + ); + BinaryPrediction::from_prob_one(p1, self.min_prob) + } + } + } + + /// Convenience prediction for `P(bit = 1)`. + pub fn predict_one(&mut self) -> f64 { + self.predict_bit().p1 + } + + /// Capture a reversible checkpoint for later restoration. + /// + /// Backends that store full snapshots restore bit-identical floating-point + /// predictions. Backends that use compact reversible journals may differ by + /// a few ULP after restore because floating-point accumulators are replayed + /// instead of cloned byte-for-byte. + pub fn checkpoint(&mut self) -> RateBackendBitSessionCheckpoint { + debug_assert_eq!( + self.discardable_scopes, 0, + "RateBackendBitSession checkpoints are invalid inside discardable simulation scopes" + ); + RateBackendBitSessionCheckpoint { + backend_code: self.backend_code.clone(), + predictor: self.predictor.checkpoint(), + semantics: self.semantics, + prefix: self.prefix.clone(), + } + } + + #[cfg(any(feature = "aixi", test))] + pub(crate) fn begin_discardable_scope(&mut self) { + self.discardable_scopes = self.discardable_scopes.saturating_add(1); + } + + #[cfg(feature = "aixi")] + pub(crate) fn clear_discardable_scopes(&mut self) { + self.discardable_scopes = 0; + } + + /// Restore the session to a previously captured checkpoint. + /// + /// Snapshot-backed predictors restore bit-identical predictions. Compact + /// journaled predictors restore the same discrete predictor state and stream + /// position, with predictions equal up to floating-point round-off. + /// + /// A checkpoint is tied to the backend and bit-stream semantics it was + /// created from. Restoring a checkpoint into a different bit session is a + /// programmer error and returns an explicit runtime error. + pub fn restore_checkpoint( + &mut self, + checkpoint: &RateBackendBitSessionCheckpoint, + ) -> InfotheoryResult<()> { + if self.backend_code != checkpoint.backend_code || self.semantics != checkpoint.semantics { + return Err(InfotheoryError::runtime( + "RateBackendBitSession checkpoint belongs to a different backend or bit semantics", + )); + } + self.release_inflight_prefix_checkpoint_for_restore(); + self.predictor.restore_checkpoint(&checkpoint.predictor); + let mut restored_prefix = checkpoint.prefix.clone(); + if let Some(BufferedBytePrefix { + kind: + BufferedBytePrefixKind::NativeMsb { + start_checkpoint: Some(start_checkpoint), + symbol, + bits, + }, + .. + }) = checkpoint.prefix.as_ref() + { + let restored_state = self.predictor.checkpoint(); + self.predictor.restore_checkpoint(start_checkpoint.as_ref()); + let abort_res = self.predictor.abort_empty_native_msb_byte_prefix(); + if let Err(err) = abort_res { + self.predictor.restore_checkpoint(&restored_state); + self.predictor.discard_checkpoint(restored_state); + return Err(InfotheoryError::runtime(err)); + } + let fresh_start = self.predictor.checkpoint(); + match self.predictor.begin_native_msb_byte_prefix() { + Ok(true) => {} + Ok(false) => { + self.predictor.discard_checkpoint(fresh_start); + self.predictor.restore_checkpoint(&restored_state); + self.predictor.discard_checkpoint(restored_state); + return Err(InfotheoryError::runtime( + "stored checkpoint requires native MSB-first byte-prefix support during restore", + )); + } + Err(err) => { + self.predictor.discard_checkpoint(fresh_start); + self.predictor.restore_checkpoint(&restored_state); + self.predictor.discard_checkpoint(restored_state); + return Err(InfotheoryError::runtime(err)); + } + } + for bit_idx in 0..*bits { + let bit = Self::observed_native_prefix_bit(*symbol, bit_idx); + if let Err(err) = self.predictor.observe_native_msb_prefix_bit(bit_idx, bit) { + self.predictor.restore_checkpoint(&restored_state); + self.predictor.discard_checkpoint(fresh_start); + self.predictor.discard_checkpoint(restored_state); + return Err(InfotheoryError::runtime(err)); + } + } + self.predictor.discard_checkpoint(restored_state); + if let Some(prefix) = restored_prefix.as_mut() + && let BufferedBytePrefixKind::NativeMsb { + start_checkpoint, .. + } = &mut prefix.kind + { + *start_checkpoint = Some(Box::new(fresh_start)); + } + self.prefix = restored_prefix; + return Ok(()); + } + self.prefix = restored_prefix; + Ok(()) + } + + /// Clear compact checkpoint journals when no stored checkpoints remain. + /// + /// This is an optimization hint for backends with journaled checkpoints. + /// Calling it while a checkpoint may still be restored violates the + /// checkpoint contract. Byte-packed sessions preserve the invariant between + /// buffered prefix state and predictor-native prefix state before clearing. + pub fn clear_checkpoints_if_supported(&mut self) { + match self.normalize_prefix_for_checkpoint_clear() { + Ok(true) => self.predictor.clear_checkpoints_if_supported(), + Ok(false) => {} + Err(err) => { + panic!("failed to normalize native byte-prefix before checkpoint clear: {err}") + } + } + } + + /// Predict and then observe one adaptive/fitting bit. + pub fn try_step_bit(&mut self, bit: bool) -> InfotheoryResult { + let prediction = self.predict_bit(); + self.try_observe_bit(bit)?; + Ok(prediction) + } + + /// Predict and then observe one adaptive/fitting bit. + pub fn step_bit(&mut self, bit: bool) -> BinaryPrediction { + self.try_step_bit(bit) + .unwrap_or_else(|err| panic!("step_bit rejected an invalid bit-session update: {err}")) + } + + /// Observe one adaptive/fitting bit. + pub fn try_observe_bit(&mut self, bit: bool) -> InfotheoryResult<()> { + match self.semantics { + BitStreamSemantics::BinaryTokens => { + self.predictor.update(u8::from(bit)); + Ok(()) + } + BitStreamSemantics::BytePacked { order } => { + self.update_byte_packed_bit(bit, order, BufferedByteUpdateMode::Adaptive) + } + } + } + + /// Observe one adaptive/fitting bit. + pub fn observe_bit(&mut self, bit: bool) { + self.try_observe_bit(bit).unwrap_or_else(|err| { + panic!("observe_bit rejected an invalid bit-session update: {err}") + }); + } + + /// Advance conditioning state with one bit without fitting/adapting. + pub fn try_condition_bit(&mut self, bit: bool) -> InfotheoryResult<()> { + match self.semantics { + BitStreamSemantics::BinaryTokens => { + self.predictor.update_frozen(u8::from(bit)); + Ok(()) + } + BitStreamSemantics::BytePacked { order } => { + self.update_byte_packed_bit(bit, order, BufferedByteUpdateMode::Frozen) + } + } + } + + /// Advance conditioning state with one bit without fitting/adapting. + pub fn condition_bit(&mut self, bit: bool) { + self.try_condition_bit(bit).unwrap_or_else(|err| { + panic!("condition_bit rejected an invalid bit-session update: {err}") + }); + } + + /// Reset dynamic conditioning state while preserving fitted parameters/statistics. + pub fn reset_frozen(&mut self, total_bits: Option) -> InfotheoryResult<()> { + let total_symbols = total_symbols_for_bit_semantics(total_bits, self.semantics) + .map_err(InfotheoryError::runtime)?; + self.discard_inflight_prefix_checkpoint(); + self.predictor + .reset_frozen(total_symbols) + .map_err(InfotheoryError::runtime) + } + + /// Finalize the underlying stream if the backend needs it. + pub fn finish(&mut self) -> InfotheoryResult<()> { + if matches!(self.semantics, BitStreamSemantics::BytePacked { .. }) + && self + .prefix + .as_ref() + .is_some_and(BufferedBytePrefix::has_partial_bits) + { + return Err(InfotheoryError::runtime( + "byte-packed bit streams must finish on a whole-byte boundary; use BitStreamSemantics::BinaryTokens for arbitrary-length bit streams", + )); + } + self.discard_inflight_prefix_checkpoint(); + self.predictor + .finish_stream() + .map_err(InfotheoryError::runtime) + } + + fn ensure_prefix_for_update( + &mut self, + order: BitOrder, + update_mode: BufferedByteUpdateMode, + ) -> InfotheoryResult<()> { + if self.prefix.is_some() { + return Ok(()); + } + if order == BitOrder::MsbFirst && self.predictor.has_native_msb_byte_prefix() { + let checkpoint = if update_mode == BufferedByteUpdateMode::Frozen { + Some(self.predictor.checkpoint()) + } else { + None + }; + match self.try_begin_native_msb_prefix(checkpoint) { + Ok(true) => return Ok(()), + Ok(false) => {} + Err(err) => return Err(InfotheoryError::runtime(err)), + } + } + self.ensure_mass_prefix(order); + Ok(()) + } + + /// Try to enter native MSB byte-prefix mode for byte-packed sessions. + /// + /// - `Ok(true)`: native prefix active (`self.prefix` set) + /// - `Ok(false)`: caller should use [`Self::ensure_mass_prefix`] (predictor unchanged) + /// - `Err`: predictor state-machine failure; caller must not silently fall back + fn try_begin_native_msb_prefix( + &mut self, + start_checkpoint: Option, + ) -> Result { + match self.predictor.begin_native_msb_byte_prefix() { + Ok(true) => { + self.prefix = Some(BufferedBytePrefix::new_native_msb(start_checkpoint)); + Ok(true) + } + Ok(false) => { + if let Some(checkpoint) = start_checkpoint { + self.predictor.discard_checkpoint(checkpoint); + } + Ok(false) + } + Err(err) => { + if let Some(checkpoint) = start_checkpoint { + self.predictor.discard_checkpoint(checkpoint); + } + Err(err) + } + } + } + + fn ensure_mass_prefix(&mut self, order: BitOrder) { + if self.prefix.is_some() { + return; + } + let mut logps = [0.0f64; 256]; + self.predictor.fill_log_probs(&mut logps); + self.prefix = Some(BufferedBytePrefix::new_mass( + BytePrefixMass::from_log_probs(&logps, order), + )); + } + + fn update_byte_packed_bit( + &mut self, + bit: bool, + order: BitOrder, + update_mode: BufferedByteUpdateMode, + ) -> InfotheoryResult<()> { + self.ensure_prefix_for_update(order, update_mode)?; + let needs_adaptive_prefix_checkpoint = + update_mode != BufferedByteUpdateMode::Adaptive || self.discardable_scopes == 0; + let prefix = self.prefix.as_mut().expect("prefix initialized"); + prefix.record_mode(update_mode)?; + match &mut prefix.kind { + BufferedBytePrefixKind::Mass(mass) => { + mass.observe(bit); + if mass.is_complete() { + let symbol = mass.symbol(); + match update_mode { + BufferedByteUpdateMode::Adaptive => self.predictor.update(symbol), + BufferedByteUpdateMode::Frozen => self.predictor.update_frozen(symbol), + } + self.prefix = None; + } + } + BufferedBytePrefixKind::NativeMsb { + start_checkpoint, + symbol, + bits, + } => { + if start_checkpoint.is_none() && *bits == 0 && needs_adaptive_prefix_checkpoint { + *start_checkpoint = Some(Box::new(self.predictor.checkpoint())); + } + self.predictor + .observe_native_msb_prefix_bit(*bits, bit) + .map_err(InfotheoryError::runtime)?; + if bit { + *symbol |= 1u8 << (7 - *bits); + } + *bits += 1; + if *bits == 8 { + let completed_symbol = *symbol; + if update_mode == BufferedByteUpdateMode::Frozen { + let checkpoint_box = start_checkpoint.take().ok_or_else(|| { + InfotheoryError::runtime( + "native byte-prefix frozen update is missing its start checkpoint", + ) + })?; + let checkpoint = *checkpoint_box; + self.restore_start_checkpoint_abort_and_discard(checkpoint) + .map_err(InfotheoryError::runtime)?; + self.predictor.update_frozen(completed_symbol); + } else { + self.predictor + .finish_native_msb_byte_prefix(completed_symbol) + .map_err(InfotheoryError::runtime)?; + if let Some(checkpoint) = start_checkpoint.take() { + self.predictor.discard_checkpoint(*checkpoint); + } + } + self.prefix = None; + } + } + } + Ok(()) + } +} + +impl crate::prediction::OnlineBitPredictor for RateBackendBitSession { + fn begin_bit_stream( + &mut self, + total_bits: Option, + semantics: BitStreamSemantics, + ) -> Result<(), String> { + if semantics != self.semantics { + return Err( + "bit stream semantics are fixed for a RateBackendBitSession; create a new session" + .to_string(), + ); + } + let total_symbols = total_symbols_for_bit_semantics(total_bits, self.semantics)?; + self.discard_inflight_prefix_checkpoint(); + self.predictor.begin_fresh_stream(total_symbols) + } + + fn finish_bit_stream(&mut self) -> Result<(), String> { + self.finish().map_err(|err| err.to_string()) + } + + fn bit_prediction(&mut self) -> BinaryPrediction { + self.predict_bit() + } + + fn update_bit(&mut self, bit: bool) { + self.observe_bit(bit); + } + + fn update_bit_frozen(&mut self, bit: bool) { + self.condition_bit(bit); + } +} + +impl RateBackendSession { + /// Create a session from an explicit backend. + /// + /// Algorithmic configuration (such as ROSA's `max_order`) lives inside the + /// backend's variant; the session does not take it as an argument. + pub fn from_backend( + backend: CompiledRateBackend, + total_symbols: Option, + ) -> InfotheoryResult { + let mut predictor = crate::runtime::build_rate_backend_predictor_default(&backend) + .map_err(InfotheoryError::invalid_backend_config)?; + predictor + .begin_stream(total_symbols) + .map_err(InfotheoryError::runtime)?; + Ok(Self { predictor }) + } + + /// Create a session from a wrapper backend spec. + pub fn from_spec(backend: RateBackend, total_symbols: Option) -> InfotheoryResult { + let compiled = backend + .compile() + .map_err(|err| InfotheoryError::invalid_backend_config(err.to_string()))?; + Self::from_backend(compiled, total_symbols) + } + + /// Observe bytes while adapting/fitting the model. + pub fn observe(&mut self, data: &[u8]) { + for &byte in data { + self.predictor.update(byte); + } + } + + /// Advance conditioning state without changing fitted parameters/statistics. + pub fn condition(&mut self, data: &[u8]) { + for &byte in data { + self.predictor.update_frozen(byte); + } + } + + /// Reset dynamic conditioning state while preserving fitted parameters/statistics. + pub fn reset_frozen(&mut self, total_symbols: Option) -> InfotheoryResult<()> { + self.predictor + .reset_frozen(total_symbols) + .map_err(InfotheoryError::runtime) + } + + /// Start a new stream while preserving each backend's semantic contract. + /// + /// Backends that support frozen-reset semantics will restart via + /// `reset_frozen`. Backends that do not (for example ZPAQ) restart through + /// ordinary stream lifecycle hooks instead. + pub fn begin_stream(&mut self, total_symbols: Option) -> InfotheoryResult<()> { + self.predictor + .begin_fresh_stream(total_symbols) + .map_err(InfotheoryError::runtime) + } + + /// Fill the 256-way next-byte log-probabilities. + pub fn fill_log_probs(&mut self, out: &mut [f64; 256]) { + self.predictor.fill_log_probs(out); + } + + /// Generate continuation bytes from the current state. + pub fn generate_bytes(&mut self, bytes: usize, config: GenerationConfig) -> Vec { + if bytes == 0 { + return Vec::new(); + } + + let mut out = Vec::with_capacity(bytes); + let mut logps = [0.0f64; 256]; + let mut rng = GenerationRng::new(config.seed); + + for _ in 0..bytes { + match &mut self.predictor { + #[cfg(feature = "backend-rosa")] + crate::mixture::RateBackendPredictor::Rosa { .. } => { + for (sym, slot) in logps.iter_mut().enumerate() { + *slot = self.predictor.log_prob(sym as u8); + } + } + _ => self.predictor.fill_log_probs(&mut logps), + } + let byte = pick_generated_byte(&logps, config, &mut rng); + match config.update_mode { + GenerationUpdateMode::Adaptive => self.predictor.update(byte), + GenerationUpdateMode::Frozen => self.predictor.update_frozen(byte), + } + out.push(byte); + } + + out + } + + /// Finalize the underlying stream if the backend needs it. + pub fn finish(&mut self) -> InfotheoryResult<()> { + self.predictor + .finish_stream() + .map_err(InfotheoryError::runtime) + } +} + +impl InfotheoryCtx { + /// Create the current build's implicit default context. + pub fn try_default() -> InfotheoryResult { + Self::from_specs( + RateBackend::try_default()?, + CompressionBackend::try_default()?, + ) + } + + /// Create a context from explicit rate and compression backends. + pub fn new( + rate_backend: CompiledRateBackend, + compression_backend: CompiledCompressionBackend, + ) -> Self { + Self { + rate_backend, + compression_backend, + } + } + + /// Create a context from wrapper backend specs. + pub fn from_specs( + rate_backend: RateBackend, + compression_backend: CompressionBackend, + ) -> InfotheoryResult { + Ok(Self { + rate_backend: rate_backend + .compile() + .map_err(|err| InfotheoryError::invalid_backend_config(err.to_string()))?, + compression_backend: compression_backend + .compile() + .map_err(|err| InfotheoryError::invalid_backend_config(err.to_string()))?, + }) + } + + /// Create a context with adaptive ROSA+ rate backend and ZPAQ compression backend. + pub fn try_with_zpaq(method: impl Into) -> InfotheoryResult { + Self::from_specs( + RateBackend::RosaPlus { max_order: -1 }, + CompressionBackend::zpaq(method), + ) + } + + /// Compressed length of one byte slice under this context's compressor. + pub fn try_compress_size(&self, data: &[u8]) -> InfotheoryResult { + crate::api::compression::try_compress_size_backend(data, &self.compression_backend) + } + + /// Compressed length of chained slices under one stream. + pub fn try_compress_size_chain(&self, parts: &[&[u8]]) -> InfotheoryResult { + crate::api::compression::try_compress_size_chain_backend(parts, &self.compression_backend) + } + + /// Create a stateful session for the active rate backend. + pub fn rate_backend_session( + &self, + total_symbols: Option, + ) -> InfotheoryResult { + RateBackendSession::from_backend(self.rate_backend.clone(), total_symbols) + } + + /// Create a stateful bit-level session for the active rate backend. + pub fn rate_backend_bit_session( + &self, + total_bits: Option, + semantics: BitStreamSemantics, + ) -> InfotheoryResult { + RateBackendBitSession::from_backend(self.rate_backend.clone(), total_bits, semantics) + } + + /// Fallible entropy-rate estimate for `data` under this context's rate backend. + pub fn try_entropy_rate_bytes(&self, data: &[u8]) -> InfotheoryResult { + try_entropy_rate_backend(data, &self.rate_backend) + } + + /// Fallible biased entropy-rate estimate (plugin variant) for `data`. + pub fn try_biased_entropy_rate_bytes(&self, data: &[u8]) -> InfotheoryResult { + try_biased_entropy_rate_backend(data, &self.rate_backend) + } + + /// Fallible cross entropy of `test_data` under model trained on `train_data`. + pub fn try_cross_entropy_rate_bytes( + &self, + test_data: &[u8], + train_data: &[u8], + ) -> InfotheoryResult { + try_cross_entropy_rate_backend(test_data, train_data, &self.rate_backend) + } + + /// Cross entropy under the active rate backend. + pub fn try_cross_entropy_bytes( + &self, + test_data: &[u8], + train_data: &[u8], + ) -> InfotheoryResult { + self.try_cross_entropy_rate_bytes(test_data, train_data) + } + + /// Fallible joint entropy-rate estimate `H(X,Y)` under aligned-prefix semantics. + pub fn try_joint_entropy_rate_bytes(&self, x: &[u8], y: &[u8]) -> InfotheoryResult { + let (x, y) = aligned_prefix(x, y); + if x.is_empty() { + return Ok(0.0); + } + try_joint_entropy_rate_backend(x, y, &self.rate_backend) + } + + /// Fallible conditional entropy-rate estimate `H(X|Y)`. + pub fn try_conditional_entropy_rate_bytes(&self, x: &[u8], y: &[u8]) -> InfotheoryResult { + let (x, y) = aligned_prefix(x, y); + if x.is_empty() { + return Ok(0.0); + } + let h_xy = self.try_joint_entropy_rate_bytes(x, y)?; + let h_y = self.try_entropy_rate_bytes(y)?; + Ok((h_xy - h_y).max(0.0)) + } + + /// Fallible `H(data | prefix_parts)` by conditioning the active rate backend + /// on an explicit prefix chain. + pub fn try_cross_entropy_conditional_chain( + &self, + prefix_parts: &[&[u8]], + data: &[u8], + ) -> InfotheoryResult { + crate::runtime::try_cross_entropy_conditional_chain_backend( + prefix_parts, + data, + &self.rate_backend, + ) + } + + /// Generate a continuation from `prompt` with [`GenerationConfig::default()`]. + pub fn try_generate_bytes(&self, prompt: &[u8], bytes: usize) -> InfotheoryResult> { + self.try_generate_bytes_with_config(prompt, bytes, GenerationConfig::default()) + } + + /// Fallible continuation generation from `prompt` using an explicit config. + pub fn try_generate_bytes_with_config( + &self, + prompt: &[u8], + bytes: usize, + config: GenerationConfig, + ) -> InfotheoryResult> { + try_generate_rate_backend_chain(&[prompt], bytes, &self.rate_backend, config) + } + + /// Generate a continuation after conditioning on an explicit chain of prefix parts. + pub fn try_generate_bytes_conditional_chain( + &self, + prefix_parts: &[&[u8]], + bytes: usize, + ) -> InfotheoryResult> { + self.try_generate_bytes_conditional_chain_with_config( + prefix_parts, + bytes, + GenerationConfig::default(), + ) + } + + /// Fallible continuation generation after conditioning on an explicit chain of prefix parts. + pub fn try_generate_bytes_conditional_chain_with_config( + &self, + prefix_parts: &[&[u8]], + bytes: usize, + config: GenerationConfig, + ) -> InfotheoryResult> { + try_generate_rate_backend_chain(prefix_parts, bytes, &self.rate_backend, config) + } + + /// NCD between byte slices using this context's compression backend. + pub fn try_ncd_bytes(&self, x: &[u8], y: &[u8], variant: NcdVariant) -> InfotheoryResult { + try_ncd_bytes_backend(x, y, &self.compression_backend, variant) + } + + /// Rate-backend mutual information estimate. + pub fn try_mutual_information_rate_bytes(&self, x: &[u8], y: &[u8]) -> InfotheoryResult { + try_mutual_information_rate_backend(x, y, &self.rate_backend) + } + + /// Mutual information under the active rate backend. + pub fn try_mutual_information_bytes(&self, x: &[u8], y: &[u8]) -> InfotheoryResult { + self.try_mutual_information_rate_bytes(x, y) + } + + /// Conditional entropy `H(X|Y)` under the active rate backend. + pub fn try_conditional_entropy_bytes(&self, x: &[u8], y: &[u8]) -> InfotheoryResult { + let (x, y) = aligned_prefix(x, y); + let h_xy = self.try_joint_entropy_rate_bytes(x, y)?; + let h_y = self.try_entropy_rate_bytes(y)?; + Ok((h_xy - h_y).max(0.0)) + } + + /// Normalized entropy distance (NED) under this context's rate backend. + pub fn try_ned_bytes(&self, x: &[u8], y: &[u8]) -> InfotheoryResult { + try_ned_rate_backend(x, y, &self.rate_backend) + } + + /// Conservative NED normalization variant under this context's rate backend. + pub fn try_ned_cons_bytes(&self, x: &[u8], y: &[u8]) -> InfotheoryResult { + let (x, y) = aligned_prefix(x, y); + let h_x = self.try_entropy_rate_bytes(x)?; + let h_y = self.try_entropy_rate_bytes(y)?; + let h_xy = self.try_joint_entropy_rate_bytes(x, y)?; + let min_h = h_x.min(h_y); + if h_xy == 0.0 { + Ok(0.0) + } else { + Ok(((h_xy - min_h) / h_xy).clamp(0.0, 1.0)) + } + } + + /// Normalized transform effort (NTE) under this context's rate backend. + pub fn try_nte_bytes(&self, x: &[u8], y: &[u8]) -> InfotheoryResult { + try_nte_rate_backend(x, y, &self.rate_backend) + } + + /// Intrinsic dependence score in `[0,1]` driven by `(H₀(X) - Ĥ(X)) / H₀(X)`, + /// where `H₀` is the order-0 / empirical entropy and `Ĥ` is the entropy rate + /// produced by this context's rate backend. + pub fn try_intrinsic_dependence_bytes(&self, data: &[u8]) -> InfotheoryResult { + let h_empirical = empirical_entropy_bytes(data); + if h_empirical < 1e-9 { + return Ok(0.0); + } + let h_rate = self.try_entropy_rate_bytes(data)?; + Ok(((h_empirical - h_rate) / h_empirical).clamp(0.0, 1.0)) + } + + /// Resistance-to-transformation ratio `I(X;T(X))/H(X)` in `[0,1]` under this context's rate backend. + pub fn try_resistance_to_transformation_bytes( + &self, + x: &[u8], + tx: &[u8], + ) -> InfotheoryResult { + let (x, tx) = aligned_prefix(x, tx); + let h_x = self.try_entropy_rate_bytes(x)?; + if h_x < 1e-9 { + return Ok(0.0); + } + let mi = self.try_mutual_information_bytes(x, tx)?; + Ok((mi / h_x).clamp(0.0, 1.0)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + // Re-exported for trait methods (begin_bit_stream etc.) exercised in + // feature-specific tests. Some narrow backend slices do not use the trait + // directly, but broader bit-session test slices do. + #[allow(unused_imports)] + use crate::prediction::OnlineBitPredictor; + + #[cfg(feature = "backend-ctw")] + fn ctw_checkpoint_depth(session: &RateBackendBitSession) -> usize { + match &session.predictor { + crate::mixture::RateBackendPredictor::Ctw { + checkpoint_depth, .. + } => *checkpoint_depth, + crate::mixture::RateBackendPredictor::FacCtw { + checkpoint_depth, .. + } => *checkpoint_depth, + _ => panic!("expected ctw predictor"), + } + } + + #[cfg(feature = "backend-ctw")] + fn ctw_native_prefix_progress(session: &RateBackendBitSession) -> Option { + match &session.predictor { + crate::mixture::RateBackendPredictor::Ctw { + native_prefix_progress, + .. + } + | crate::mixture::RateBackendPredictor::FacCtw { + native_prefix_progress, + .. + } => *native_prefix_progress, + _ => panic!("expected ctw predictor"), + } + } + + #[cfg(feature = "backend-ctw")] + fn buffered_native_prefix_bits(session: &RateBackendBitSession) -> Option { + match session.prefix.as_ref().map(|prefix| &prefix.kind) { + Some(BufferedBytePrefixKind::NativeMsb { bits, .. }) => Some(*bits), + _ => None, + } + } + + #[cfg(feature = "backend-ctw")] + fn buffered_native_prefix_has_start_checkpoint( + session: &RateBackendBitSession, + ) -> Option { + match session.prefix.as_ref().map(|prefix| &prefix.kind) { + Some(BufferedBytePrefixKind::NativeMsb { + start_checkpoint, .. + }) => Some(start_checkpoint.is_some()), + _ => None, + } + } + + #[cfg(feature = "backend-ctw")] + fn new_ctw_byte_packed_session() -> RateBackendBitSession { + RateBackendBitSession::from_spec( + RateBackend::Ctw { depth: 4 }, + Some(8), + BitStreamSemantics::BytePacked { + order: BitOrder::MsbFirst, + }, + ) + .expect("ctw byte-packed session") + } + + #[cfg(feature = "backend-ctw")] + fn new_fac_ctw_byte_packed_session() -> RateBackendBitSession { + RateBackendBitSession::from_spec( + RateBackend::FacCtw { + base_depth: 4, + num_percept_bits: 8, + encoding_bits: 8, + msb_first: None, + }, + Some(8), + BitStreamSemantics::BytePacked { + order: BitOrder::MsbFirst, + }, + ) + .expect("fac-ctw byte-packed session") + } + + #[cfg(feature = "backend-ctw")] + #[test] + fn frozen_native_byte_completion_releases_start_checkpoint() { + for mut session in [ + new_ctw_byte_packed_session(), + new_fac_ctw_byte_packed_session(), + ] { + for bit in [true, false, true, false, false, true, true, false] { + session + .try_condition_bit(bit) + .expect("condition full frozen native byte"); + } + assert!(session.prefix.is_none()); + assert_eq!(ctw_checkpoint_depth(&session), 0); + assert_eq!(ctw_native_prefix_progress(&session), None); + + let pred = session.predict_bit(); + assert!(pred.p0.is_finite() && pred.p1.is_finite()); + assert_eq!(buffered_native_prefix_bits(&session), Some(0)); + assert_eq!(ctw_native_prefix_progress(&session), Some(0)); + } + } + + #[cfg(feature = "backend-ctw")] + #[test] + fn reset_frozen_discards_inflight_native_prefix_checkpoint() { + let mut session = new_ctw_byte_packed_session(); + session.try_condition_bit(true).expect("conditioning bit"); + assert_eq!(ctw_checkpoint_depth(&session), 1); + + session.reset_frozen(Some(8)).expect("reset frozen"); + assert_eq!(ctw_checkpoint_depth(&session), 0); + } + + #[cfg(feature = "backend-ctw")] + #[test] + fn begin_bit_stream_discards_inflight_native_prefix_checkpoint() { + let mut session = new_ctw_byte_packed_session(); + session.try_condition_bit(true).expect("conditioning bit"); + assert_eq!(ctw_checkpoint_depth(&session), 1); + + session + .begin_bit_stream( + Some(8), + BitStreamSemantics::BytePacked { + order: BitOrder::MsbFirst, + }, + ) + .expect("begin bit stream"); + assert_eq!(ctw_checkpoint_depth(&session), 0); + } + + #[cfg(feature = "backend-ctw")] + #[test] + fn clear_checkpoints_after_empty_native_restore_drops_prefix_cleanly() { + for mut session in [ + new_ctw_byte_packed_session(), + new_fac_ctw_byte_packed_session(), + ] { + let _ = session.predict_bit(); + let checkpoint = session.checkpoint(); + session + .try_condition_bit(true) + .expect("conditioning bit after checkpoint"); + + session + .restore_checkpoint(&checkpoint) + .expect("restore empty native prefix checkpoint"); + assert_eq!(buffered_native_prefix_bits(&session), Some(0)); + assert_eq!(ctw_native_prefix_progress(&session), Some(0)); + + session.clear_checkpoints_if_supported(); + assert!(session.prefix.is_none()); + assert_eq!(ctw_checkpoint_depth(&session), 0); + assert_eq!(ctw_native_prefix_progress(&session), None); + + let pred = session.predict_bit(); + assert!(pred.p0.is_finite() && pred.p1.is_finite()); + assert_eq!(buffered_native_prefix_bits(&session), Some(0)); + assert_eq!(ctw_native_prefix_progress(&session), Some(0)); + } + } + + #[cfg(feature = "backend-ctw")] + #[test] + fn clear_checkpoints_converts_partial_native_prefix_to_mass() { + for mut session in [ + new_ctw_byte_packed_session(), + new_fac_ctw_byte_packed_session(), + ] { + session.try_condition_bit(true).expect("first prefix bit"); + session.try_condition_bit(false).expect("second prefix bit"); + assert_eq!(buffered_native_prefix_bits(&session), Some(2)); + assert_eq!(ctw_native_prefix_progress(&session), Some(2)); + + let before_clear = session.predict_bit(); + session.clear_checkpoints_if_supported(); + + assert!(matches!( + session.prefix.as_ref().map(|prefix| &prefix.kind), + Some(BufferedBytePrefixKind::Mass(_)) + )); + assert_eq!(ctw_checkpoint_depth(&session), 0); + assert_eq!(ctw_native_prefix_progress(&session), None); + + let after_clear = session.predict_bit(); + assert!( + (before_clear.p1 - after_clear.p1).abs() <= 1e-12, + "native-to-mass conversion changed prefix prediction: before={} after={}", + before_clear.p1, + after_clear.p1 + ); + + for bit in [true, false, true, false, true, false] { + session + .try_condition_bit(bit) + .expect("finish converted mass prefix"); + } + session + .finish() + .expect("finish whole byte after conversion"); + } + } + + #[cfg(feature = "backend-ctw")] + #[test] + fn reset_frozen_rolls_back_adaptive_partial_native_prefix() { + for mut session in [ + new_ctw_byte_packed_session(), + new_fac_ctw_byte_packed_session(), + ] { + session.try_observe_bit(true).expect("adaptive prefix bit"); + assert_eq!(ctw_checkpoint_depth(&session), 1); + assert_eq!(ctw_native_prefix_progress(&session), Some(1)); + + session.reset_frozen(Some(8)).expect("reset frozen"); + + assert_eq!(ctw_checkpoint_depth(&session), 0); + assert_eq!(ctw_native_prefix_progress(&session), None); + let mut fresh = match &session.predictor { + crate::mixture::RateBackendPredictor::Ctw { .. } => new_ctw_byte_packed_session(), + crate::mixture::RateBackendPredictor::FacCtw { .. } => { + new_fac_ctw_byte_packed_session() + } + _ => unreachable!("test only constructs CTW-family sessions"), + }; + let reset_prediction = session.predict_bit(); + let fresh_prediction = fresh.predict_bit(); + assert!( + (reset_prediction.p1 - fresh_prediction.p1).abs() <= 1e-12, + "partial adaptive native prefix leaked into reset state: reset={} fresh={}", + reset_prediction.p1, + fresh_prediction.p1 + ); + } + } + + #[cfg(feature = "backend-ctw")] + #[test] + fn discardable_scope_avoids_adaptive_native_prefix_checkpoint() { + for mut session in [ + new_ctw_byte_packed_session(), + new_fac_ctw_byte_packed_session(), + ] { + session.begin_discardable_scope(); + let _ = session.predict_bit(); + session + .try_observe_bit(true) + .expect("discardable adaptive prefix bit"); + + assert_eq!(buffered_native_prefix_bits(&session), Some(1)); + assert_eq!( + buffered_native_prefix_has_start_checkpoint(&session), + Some(false), + "discardable adaptive prefixes must not retain restore-only checkpoints", + ); + assert_eq!(ctw_checkpoint_depth(&session), 0); + + for bit in [false, true, false, true, false, true, false] { + session + .try_observe_bit(bit) + .expect("finish discardable adaptive byte"); + } + assert!(session.prefix.is_none()); + assert_eq!(ctw_checkpoint_depth(&session), 0); + assert_eq!(ctw_native_prefix_progress(&session), None); + } + } + + #[cfg(feature = "backend-ctw")] + #[test] + fn discardable_scope_keeps_frozen_native_prefix_checkpoint() { + let mut session = new_ctw_byte_packed_session(); + session.begin_discardable_scope(); + session + .try_condition_bit(true) + .expect("discardable frozen prefix bit"); + + assert_eq!(buffered_native_prefix_bits(&session), Some(1)); + assert_eq!( + buffered_native_prefix_has_start_checkpoint(&session), + Some(true), + "frozen native prefixes still need a start checkpoint to avoid learning action bytes", + ); + assert_eq!(ctw_checkpoint_depth(&session), 1); + } + + #[cfg(feature = "backend-ctw")] + #[test] + fn begin_bit_stream_rolls_back_adaptive_partial_native_prefix() { + let mut session = new_fac_ctw_byte_packed_session(); + session.try_observe_bit(true).expect("adaptive prefix bit"); + assert_eq!(ctw_checkpoint_depth(&session), 1); + assert_eq!(ctw_native_prefix_progress(&session), Some(1)); + + session + .begin_bit_stream( + Some(8), + BitStreamSemantics::BytePacked { + order: BitOrder::MsbFirst, + }, + ) + .expect("begin bit stream"); + + assert_eq!(ctw_checkpoint_depth(&session), 0); + assert_eq!(ctw_native_prefix_progress(&session), None); + session + .try_observe_bit(false) + .expect("native prefix re-entry after adaptive abort"); + } + + #[cfg(feature = "backend-ctw")] + #[test] + fn restore_checkpoint_discards_abandoned_native_prefix_checkpoint() { + let mut session = new_ctw_byte_packed_session(); + let checkpoint = session.checkpoint(); + assert_eq!(ctw_checkpoint_depth(&session), 1); + + session.try_condition_bit(true).expect("conditioning bit"); + assert_eq!(ctw_checkpoint_depth(&session), 2); + + session + .restore_checkpoint(&checkpoint) + .expect("restore checkpoint"); + assert_eq!(ctw_checkpoint_depth(&session), 1); + } + + #[cfg(feature = "backend-ctw")] + #[test] + fn restore_mid_prefix_checkpoint_rebuilds_frozen_native_rollback_point() { + let mut session = new_ctw_byte_packed_session(); + session.try_condition_bit(true).expect("conditioning bit"); + let checkpoint = session.checkpoint(); + assert_eq!(ctw_checkpoint_depth(&session), 2); + + session + .try_condition_bit(false) + .expect("conditioning second bit"); + session + .restore_checkpoint(&checkpoint) + .expect("restore mid-prefix checkpoint"); + assert_eq!(ctw_checkpoint_depth(&session), 2); + + for bit in [false, true, false, true, false, true, false] { + session + .try_condition_bit(bit) + .expect("finish restored byte"); + } + assert_eq!(ctw_checkpoint_depth(&session), 1); + } + + /// When static native MSB support is advertised but dynamic begin negotiation + /// returns `Ok(false)` (e.g. mixture expert lacks checkpoint rollback), `predict_bit` + /// must fall back to the BytePrefixMass path without debug-only false positives. + #[cfg(feature = "backend-mixture")] + #[test] + fn predict_bit_mass_fallback_when_native_begin_returns_false() { + use crate::mixture::{ + BayesMixture, DEFAULT_MIN_PROB, ExpertConfig, MixtureRuntime, RateBackendPredictor, + }; + + #[derive(Clone)] + struct MockNativeNoCheckpoint; + + impl OnlineBytePredictor for MockNativeNoCheckpoint { + fn log_prob(&mut self, _symbol: u8) -> f64 { + -(256.0f64).ln() + } + + fn update(&mut self, _symbol: u8) {} + + fn has_native_msb_byte_prefix(&self) -> bool { + true + } + } + + let configs = [ExpertConfig::uniform("native", || { + Box::new(MockNativeNoCheckpoint) as Box + })]; + let runtime = MixtureRuntime::Bayes(BayesMixture::new(&configs)); + let mut predictor = RateBackendPredictor::Mixture { runtime }; + predictor.begin_stream(Some(1)).expect("begin_stream"); + + let mut session = RateBackendBitSession { + backend_code: CanonicalBytes::from(b"test-native-fallback".to_vec()), + predictor, + semantics: BitStreamSemantics::BytePacked { + order: BitOrder::MsbFirst, + }, + min_prob: DEFAULT_MIN_PROB, + prefix: None, + discardable_scopes: 0, + }; + let pred = session.predict_bit(); + assert!( + (pred.p0 + pred.p1 - 1.0).abs() < 1e-9, + "mass fallback prediction must normalize: p0={} p1={}", + pred.p0, + pred.p1 + ); + assert!(pred.p1.is_finite()); + assert!(matches!( + session.prefix.as_ref().map(|p| &p.kind), + Some(BufferedBytePrefixKind::Mass(_)) + )); + } + + /// Guards the byte-prefix buffering enum against accidentally embedding a + /// full predictor checkpoint inline in the native-MSB variant. + #[test] + fn record_buffered_byte_prefix_kind_and_related_sizes() { + let byte_prefix_mass = std::mem::size_of::(); + let buffered_kind = std::mem::size_of::(); + let rate_ckpt = std::mem::size_of::(); + let opt_rate_ckpt = std::mem::size_of::>(); + eprintln!( + "BufferedBytePrefixKind sizes: \ + BufferedBytePrefixKind={}B (Mass~{}B inline; NativeMsb Option~{}B; \ + RateBackendPredictorCheckpoint enum={}B). start_checkpoint is boxed; Full variant \ + is Box so compact checkpoint variants stay small.", + buffered_kind, byte_prefix_mass, opt_rate_ckpt, rate_ckpt + ); + // Native-MSB frozen rewinds need a predictor checkpoint, but that + // checkpoint may contain full model state. Keeping it boxed prevents the + // cheaper byte-packed mass path from inheriting that storage cost. + assert_eq!( + buffered_kind, byte_prefix_mass, + "post-box BufferedBytePrefixKind (NativeMsb) must not exceed Mass variant size ({} vs {}); \ + native-MSB rewind checkpoints must stay out-of-line", + buffered_kind, byte_prefix_mass + ); + } +} diff --git a/crates/infotheory/src/api/generation.rs b/crates/infotheory/src/api/generation.rs new file mode 100644 index 00000000..d5215df7 --- /dev/null +++ b/crates/infotheory/src/api/generation.rs @@ -0,0 +1,305 @@ +//! Generation-focused public API surface. + +use super::context::RateBackendSession; +use super::types::{GenerationConfig, GenerationStrategy}; +use crate::error::{InfotheoryError, InfotheoryResult}; +use crate::spec::CompiledRateBackend; + +use crate::with_default_ctx; + +pub(crate) struct GenerationRng { + state: u64, +} + +impl GenerationRng { + pub(crate) fn new(seed: u64) -> Self { + Self { + state: if seed == 0 { + 0xD00D_F00D_CAFE_BABEu64 + } else { + seed + }, + } + } + + fn next_u64(&mut self) -> u64 { + let mut x = self.state; + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + self.state = x; + x + } + + fn next_f64(&mut self) -> f64 { + unit_interval_from_u64(self.next_u64()) + } +} + +#[inline(always)] +fn unit_interval_from_u64(bits: u64) -> f64 { + ((bits >> 11) as f64) * (1.0 / ((1u64 << 53) as f64)) +} + +#[inline(always)] +fn argmax_log_prob_byte(logps: &[f64; 256]) -> u8 { + let mut best_idx = 0usize; + let mut best = f64::NEG_INFINITY; + for (idx, &logp) in logps.iter().enumerate() { + let score = if logp.is_finite() { + logp + } else { + f64::NEG_INFINITY + }; + if score > best { + best = score; + best_idx = idx; + } + } + best_idx as u8 +} + +pub(crate) fn pick_generated_byte( + logps: &[f64; 256], + config: GenerationConfig, + rng: &mut GenerationRng, +) -> u8 { + if matches!(config.strategy, GenerationStrategy::Greedy) + || !config.temperature.is_finite() + || config.temperature <= 0.0 + { + return argmax_log_prob_byte(logps); + } + + let mut entries = [(0u8, f64::NEG_INFINITY); 256]; + for (idx, &logp) in logps.iter().enumerate() { + let scaled = if logp.is_finite() { + logp / config.temperature + } else { + f64::NEG_INFINITY + }; + entries[idx] = (idx as u8, scaled); + } + entries.sort_by(|a, b| b.1.total_cmp(&a.1)); + + let keep_k = if config.top_k == 0 { + entries.len() + } else { + config.top_k.min(entries.len()) + }; + + let top_p = if config.top_p.is_finite() { + config.top_p.clamp(0.0, 1.0) + } else { + 1.0 + }; + + let mut max_logp = f64::NEG_INFINITY; + for &(_, logp) in entries.iter().take(keep_k) { + if logp.is_finite() { + max_logp = max_logp.max(logp); + } + } + if !max_logp.is_finite() { + return argmax_log_prob_byte(logps); + } + + let mut weights = [(0u8, 0.0f64); 256]; + let mut total = 0.0; + for (idx, &(byte, logp)) in entries.iter().take(keep_k).enumerate() { + let w = if logp.is_finite() { + (logp - max_logp).exp() + } else { + 0.0 + }; + weights[idx] = (byte, w); + total += w; + } + if !(total.is_finite()) || total <= 0.0 { + return argmax_log_prob_byte(logps); + } + + let cutoff_count = if top_p >= 1.0 { + keep_k + } else { + let mut cumulative = 0.0; + let mut keep = 0usize; + for &(_, w) in weights.iter().take(keep_k) { + cumulative += w / total; + keep += 1; + if cumulative >= top_p { + break; + } + } + keep.max(1) + }; + + let mut truncated_total = 0.0; + for &(_, w) in weights.iter().take(cutoff_count) { + truncated_total += w; + } + if !(truncated_total.is_finite()) || truncated_total <= 0.0 { + return argmax_log_prob_byte(logps); + } + + let target = rng.next_f64() * truncated_total; + let mut cumulative = 0.0; + let mut picked = weights[0].0; + for &(byte, weight) in weights.iter().take(cutoff_count) { + cumulative += weight; + if cumulative >= target { + picked = byte; + break; + } + } + picked +} + +pub(crate) fn try_generate_rate_backend_chain( + prefix_parts: &[&[u8]], + bytes: usize, + backend: &CompiledRateBackend, + config: GenerationConfig, +) -> InfotheoryResult> { + if bytes == 0 { + return Ok(Vec::new()); + } + + let total = prefix_parts + .iter() + .map(|p| p.len() as u64) + .sum::() + .saturating_add(bytes as u64); + let mut session = + RateBackendSession::from_backend(backend.clone(), Some(total)).map_err(|e| { + InfotheoryError::runtime(format!("rate backend generation init failed: {e}")) + })?; + for &part in prefix_parts { + session.observe(part); + } + let out = session.generate_bytes(bytes, config); + session.finish().map_err(|e| { + InfotheoryError::runtime(format!("rate backend generation finalize failed: {e}")) + })?; + Ok(out) +} + +#[cfg(test)] +#[allow(dead_code)] +pub(crate) fn generate_rate_backend_chain( + prefix_parts: &[&[u8]], + bytes: usize, + backend: &CompiledRateBackend, + config: GenerationConfig, +) -> Vec { + try_generate_rate_backend_chain(prefix_parts, bytes, backend, config) + .expect("generate_rate_backend_chain") +} + +/// Generate a continuation from `prompt` +/// using the current default context and [`GenerationConfig::default()`]. +/// +/// The default is deterministic frozen sampling with seed `42`. +#[inline(always)] +pub fn try_generate_bytes(prompt: &[u8], bytes: usize) -> InfotheoryResult> { + with_default_ctx(|ctx| ctx.try_generate_bytes(prompt, bytes)) +} + +/// Generate a continuation from `prompt` using the current default context. +#[inline(always)] +pub fn try_generate_bytes_with_config( + prompt: &[u8], + bytes: usize, + config: GenerationConfig, +) -> InfotheoryResult> { + with_default_ctx(|ctx| ctx.try_generate_bytes_with_config(prompt, bytes, config)) +} + +/// Generate a continuation after conditioning on an explicit chain of prefix parts +/// using the current default context and [`GenerationConfig::default()`]. +#[inline(always)] +pub fn try_generate_bytes_conditional_chain( + prefix_parts: &[&[u8]], + bytes: usize, +) -> InfotheoryResult> { + with_default_ctx(|ctx| ctx.try_generate_bytes_conditional_chain(prefix_parts, bytes)) +} + +/// Generate a continuation after conditioning on an explicit chain of prefix parts +/// using the current default context. +#[inline(always)] +pub fn try_generate_bytes_conditional_chain_with_config( + prefix_parts: &[&[u8]], + bytes: usize, + config: GenerationConfig, +) -> InfotheoryResult> { + with_default_ctx(|ctx| { + ctx.try_generate_bytes_conditional_chain_with_config(prefix_parts, bytes, config) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn sparse_logps(entries: &[(u8, f64)]) -> [f64; 256] { + let mut logps = [f64::NEG_INFINITY; 256]; + for &(byte, logp) in entries { + logps[byte as usize] = logp; + } + logps + } + + #[test] + fn greedy_generation_picks_argmax() { + let logps = sparse_logps(&[(7, -0.2), (42, -1.0)]); + let mut rng = GenerationRng::new(123); + let picked = pick_generated_byte(&logps, GenerationConfig::greedy_frozen(), &mut rng); + assert_eq!(picked, 7); + } + + #[test] + fn nonpositive_temperature_falls_back_to_argmax() { + let logps = sparse_logps(&[(3, -0.1), (11, -0.3)]); + let mut rng = GenerationRng::new(7); + let mut config = GenerationConfig::sampled_frozen(7); + config.temperature = 0.0; + let picked = pick_generated_byte(&logps, config, &mut rng); + assert_eq!(picked, 3); + } + + #[test] + fn top_k_sampling_respects_truncation() { + let logps = sparse_logps(&[(9, -0.01), (10, -0.02), (11, -0.03)]); + let mut rng = GenerationRng::new(99); + let mut config = GenerationConfig::sampled_frozen(99); + config.top_k = 1; + let picked = pick_generated_byte(&logps, config, &mut rng); + assert_eq!(picked, 9); + } + + #[test] + fn top_p_sampling_keeps_only_minimal_prefix_mass() { + let logps = sparse_logps(&[(5, 0.0), (6, -1.5), (7, -3.0)]); + let mut rng = GenerationRng::new(5); + let mut config = GenerationConfig::sampled_frozen(5); + config.top_p = 0.5; + let picked = pick_generated_byte(&logps, config, &mut rng); + assert_eq!(picked, 5); + } + + #[test] + fn nonfinite_log_probs_do_not_panic() { + let mut logps = [f64::NEG_INFINITY; 256]; + logps[0] = f64::NAN; + let mut rng = GenerationRng::new(17); + let picked = pick_generated_byte(&logps, GenerationConfig::sampled_frozen(17), &mut rng); + assert_eq!(picked, 0); + } + + #[test] + fn unit_interval_mapping_excludes_one() { + assert_eq!(unit_interval_from_u64(0), 0.0); + assert!(unit_interval_from_u64(u64::MAX) < 1.0); + } +} diff --git a/crates/infotheory/src/api/metrics.rs b/crates/infotheory/src/api/metrics.rs new file mode 100644 index 00000000..fa137a34 --- /dev/null +++ b/crates/infotheory/src/api/metrics.rs @@ -0,0 +1,471 @@ +//! Information-theoretic metric and scoring API surface. +//! +//! Functions in this module split cleanly into two families: +//! +//! - **Algorithmic**: take a [`CompiledRateBackend`] (or use the default context +//! backend) and produce entropy-rate / cross-entropy / mutual-information / +//! normalized-distance estimates driven by the rate model. Algorithm-specific +//! parameters (such as ROSA's `max_order`) live inside the backend's variant. +//! - **Empirical** (`empirical_*`): order-0 / IID Shannon plug-in estimators. +//! They treat the input as IID symbols and estimate entropy from observed +//! symbol frequencies. They never take an order parameter; if higher-order +//! structure matters, use a context-aware [`RateBackend`] (e.g. CTW). + +use crate::error::{InfotheoryError, InfotheoryResult}; +use crate::spec::CompiledRateBackend; + +use crate::{aligned_prefix, with_default_ctx}; + +#[inline(always)] +/// Fallible entropy-rate estimate `Ĥ(X)` (bits per symbol) using the default context backend. +pub fn try_entropy_rate_bytes(data: &[u8]) -> InfotheoryResult { + with_default_ctx(|ctx| ctx.try_entropy_rate_bytes(data)) +} + +#[inline(always)] +/// Fallible biased/plugin entropy-rate estimate using the default context backend. +pub fn try_biased_entropy_rate_bytes(data: &[u8]) -> InfotheoryResult { + with_default_ctx(|ctx| ctx.try_biased_entropy_rate_bytes(data)) +} + +/// Mutual information rate estimate under an explicit `backend`. +/// +/// Inputs are aligned to the shared prefix length. +pub fn try_mutual_information_rate_backend( + x: &[u8], + y: &[u8], + backend: &CompiledRateBackend, +) -> InfotheoryResult { + let (x, y) = aligned_prefix(x, y); + if x.is_empty() { + return Ok(0.0); + } + let h_x = try_entropy_rate_backend(x, backend)?; + let h_y = try_entropy_rate_backend(y, backend)?; + let h_xy = try_joint_entropy_rate_backend(x, y, backend)?; + Ok((h_x + h_y - h_xy).max(0.0)) +} + +/// Normalized entropy distance under an explicit `backend`. +/// +/// Returns a value in `[0, 1]` after clamping. +pub fn try_ned_rate_backend( + x: &[u8], + y: &[u8], + backend: &CompiledRateBackend, +) -> InfotheoryResult { + let (x, y) = aligned_prefix(x, y); + if x.is_empty() { + return Ok(0.0); + } + let h_x = try_entropy_rate_backend(x, backend)?; + let h_y = try_entropy_rate_backend(y, backend)?; + let h_xy = try_joint_entropy_rate_backend(x, y, backend)?; + let min_h = h_x.min(h_y); + let max_h = h_x.max(h_y); + if max_h == 0.0 { + Ok(0.0) + } else { + Ok(((h_xy - min_h) / max_h).clamp(0.0, 1.0)) + } +} + +/// Normalized transform effort (variation-of-information form) under an explicit `backend`. +/// +/// Returns a value in `[0, 2]` after clamping. +pub fn try_nte_rate_backend( + x: &[u8], + y: &[u8], + backend: &CompiledRateBackend, +) -> InfotheoryResult { + let (x, y) = aligned_prefix(x, y); + if x.is_empty() { + return Ok(0.0); + } + let h_x = try_entropy_rate_backend(x, backend)?; + let h_y = try_entropy_rate_backend(y, backend)?; + let h_xy = try_joint_entropy_rate_backend(x, y, backend)?; + let max_h = h_x.max(h_y); + if max_h == 0.0 { + Ok(0.0) + } else { + let vi = (h_xy - h_x).max(0.0) + (h_xy - h_y).max(0.0); + Ok((vi / max_h).clamp(0.0, 2.0)) + } +} + +/// Fallible entropy-rate estimate of `data` using the explicit rate `backend`. +pub fn try_entropy_rate_backend( + data: &[u8], + backend: &CompiledRateBackend, +) -> InfotheoryResult { + crate::runtime::try_entropy_rate_backend_direct(data, backend) +} + +/// Fallible biased/plugin entropy rate of `data` using the explicit rate `backend`. +pub fn try_biased_entropy_rate_backend( + data: &[u8], + backend: &CompiledRateBackend, +) -> InfotheoryResult { + if !backend.capabilities().supports_biased_entropy { + Err(InfotheoryError::unsupported( + "biased/plugin entropy is not supported for zpaq rate backends", + )) + } else { + crate::try_frozen_plugin_rate_backend(data, &[data], backend) + } +} + +/// Fallible cross-entropy `H_{train}(test)` — score `test_data` under a model trained on `train_data`. +pub fn try_cross_entropy_rate_backend( + test_data: &[u8], + train_data: &[u8], + backend: &CompiledRateBackend, +) -> InfotheoryResult { + crate::runtime::try_cross_entropy_rate_backend_direct(test_data, train_data, backend) +} + +/// Fallible joint entropy rate `H(X,Y)` using an explicit `backend`. +pub fn try_joint_entropy_rate_backend( + x: &[u8], + y: &[u8], + backend: &CompiledRateBackend, +) -> InfotheoryResult { + crate::runtime::try_joint_entropy_rate_backend_direct(x, y, backend) +} + +/// Empirical (zero-order, IID) Shannon entropy `H₀(X)` in bits/symbol. +/// +/// Treats the input as a sequence of IID byte symbols and returns the plug-in +/// Shannon entropy of the observed byte frequencies. Use a context-aware +/// [`crate::api::RateBackend`] via [`try_entropy_rate_bytes`] for higher-order estimation. +#[inline(always)] +pub fn empirical_entropy_bytes(data: &[u8]) -> f64 { + if data.is_empty() { + return 0.0; + } + + let mut counts = [0u64; 256]; + for &b in data { + counts[b as usize] += 1; + } + + let n = data.len() as f64; + let mut h = 0.0f64; + for &count in &counts { + if count > 0 { + let p = count as f64 / n; + h -= p * p.log2(); + } + } + h +} + +/// Empirical (zero-order, IID) joint Shannon entropy `H₀(X,Y)` over aligned prefixes. +/// +/// Treats aligned `(x[i], y[i])` pairs as IID samples from a joint distribution +/// over 65536 outcomes and returns the plug-in Shannon entropy. +#[inline(always)] +pub fn empirical_joint_entropy_bytes(x: &[u8], y: &[u8]) -> f64 { + let (x, y) = aligned_prefix(x, y); + let n = x.len(); + if n == 0 { + return 0.0; + } + + let mut counts = vec![0u64; 256 * 256]; + for i in 0..n { + let pair_idx = (x[i] as usize) * 256 + (y[i] as usize); + counts[pair_idx] += 1; + } + + let n_f64 = n as f64; + let mut h = 0.0f64; + for &c in &counts { + if c > 0 { + let p = c as f64 / n_f64; + h -= p * p.log2(); + } + } + h +} + +#[inline(always)] +/// Fallible joint entropy-rate estimate `H(X,Y)` with the default context backend. +pub fn try_joint_entropy_rate_bytes(x: &[u8], y: &[u8]) -> InfotheoryResult { + with_default_ctx(|ctx| ctx.try_joint_entropy_rate_bytes(x, y)) +} + +#[inline(always)] +/// Fallible conditional entropy-rate estimate `H(X|Y)` with the default context backend. +pub fn try_conditional_entropy_rate_bytes(x: &[u8], y: &[u8]) -> InfotheoryResult { + with_default_ctx(|ctx| ctx.try_conditional_entropy_rate_bytes(x, y)) +} + +#[inline(always)] +/// Fallible conditional entropy estimate using the default context rate backend. +pub fn try_conditional_entropy_bytes(x: &[u8], y: &[u8]) -> InfotheoryResult { + with_default_ctx(|ctx| ctx.try_conditional_entropy_bytes(x, y)) +} + +#[inline(always)] +/// Fallible mutual-information estimate `I(X;Y)` using the default context rate backend. +pub fn try_mutual_information_bytes(x: &[u8], y: &[u8]) -> InfotheoryResult { + with_default_ctx(|ctx| ctx.try_mutual_information_bytes(x, y)) +} + +/// Empirical (zero-order, IID) mutual information `I₀(X;Y)` from byte histograms. +pub fn empirical_mutual_information_bytes(x: &[u8], y: &[u8]) -> f64 { + let (x, y) = aligned_prefix(x, y); + let h_x = empirical_entropy_bytes(x); + let h_y = empirical_entropy_bytes(y); + let h_xy = empirical_joint_entropy_bytes(x, y); + (h_x + h_y - h_xy).max(0.0) +} + +#[inline(always)] +/// Fallible mutual-information rate estimate with the default context backend. +pub fn try_mutual_information_rate_bytes(x: &[u8], y: &[u8]) -> InfotheoryResult { + with_default_ctx(|ctx| ctx.try_mutual_information_rate_bytes(x, y)) +} + +#[inline(always)] +/// Fallible normalized entropy distance (NED) estimate with the default context backend. +pub fn try_ned_bytes(x: &[u8], y: &[u8]) -> InfotheoryResult { + with_default_ctx(|ctx| ctx.try_ned_bytes(x, y)) +} + +/// Empirical (zero-order, IID) normalized entropy distance: +/// `(H₀(X,Y) - min(H₀(X), H₀(Y))) / max(H₀(X), H₀(Y))`. +pub fn empirical_ned_bytes(x: &[u8], y: &[u8]) -> f64 { + let (x, y) = aligned_prefix(x, y); + let h_x = empirical_entropy_bytes(x); + let h_y = empirical_entropy_bytes(y); + let h_xy = empirical_joint_entropy_bytes(x, y); + let min_h = h_x.min(h_y); + let max_h = h_x.max(h_y); + if max_h == 0.0 { + 0.0 + } else { + ((h_xy - min_h) / max_h).clamp(0.0, 1.0) + } +} + +#[inline(always)] +/// Fallible entropy-rate NED estimate with the default context backend. +pub fn try_ned_rate_bytes(x: &[u8], y: &[u8]) -> InfotheoryResult { + with_default_ctx(|ctx| ctx.try_ned_bytes(x, y)) +} + +#[inline(always)] +/// Fallible constructive NED estimate with the default context backend. +pub fn try_ned_cons_bytes(x: &[u8], y: &[u8]) -> InfotheoryResult { + with_default_ctx(|ctx| ctx.try_ned_cons_bytes(x, y)) +} + +/// Empirical (zero-order, IID) constructive normalized entropy distance: +/// `(H₀(X,Y) - min(H₀(X), H₀(Y))) / H₀(X,Y)`. +pub fn empirical_ned_cons_bytes(x: &[u8], y: &[u8]) -> f64 { + let (x, y) = aligned_prefix(x, y); + let h_x = empirical_entropy_bytes(x); + let h_y = empirical_entropy_bytes(y); + let h_xy = empirical_joint_entropy_bytes(x, y); + let min_h = h_x.min(h_y); + if h_xy == 0.0 { + 0.0 + } else { + ((h_xy - min_h) / h_xy).clamp(0.0, 1.0) + } +} + +#[inline(always)] +/// Fallible entropy-rate constructive NED estimate with the default context backend. +pub fn try_ned_cons_rate_bytes(x: &[u8], y: &[u8]) -> InfotheoryResult { + with_default_ctx(|ctx| ctx.try_ned_cons_bytes(x, y)) +} + +#[inline(always)] +/// Fallible normalized transform-effort (NTE/VI-based) estimate with the default context backend. +pub fn try_nte_bytes(x: &[u8], y: &[u8]) -> InfotheoryResult { + with_default_ctx(|ctx| ctx.try_nte_bytes(x, y)) +} + +/// Empirical (zero-order, IID) NTE estimate using variation of information +/// normalized by `max(H₀(X), H₀(Y))`. +pub fn empirical_nte_bytes(x: &[u8], y: &[u8]) -> f64 { + let (x, y) = aligned_prefix(x, y); + let h_x = empirical_entropy_bytes(x); + let h_y = empirical_entropy_bytes(y); + let h_xy = empirical_joint_entropy_bytes(x, y); + let vi = 2.0 * h_xy - h_x - h_y; + let max_h = h_x.max(h_y); + if max_h == 0.0 { + 0.0 + } else { + (vi / max_h).clamp(0.0, 2.0) + } +} + +#[inline(always)] +/// Fallible entropy-rate NTE estimate with the default context backend. +pub fn try_nte_rate_bytes(x: &[u8], y: &[u8]) -> InfotheoryResult { + with_default_ctx(|ctx| ctx.try_nte_bytes(x, y)) +} + +#[inline(always)] +pub(crate) fn byte_histogram(data: &[u8]) -> [f64; 256] { + let mut counts = [0u64; 256]; + for &b in data { + counts[b as usize] += 1; + } + let n = data.len() as f64; + let mut probs = [0.0f64; 256]; + if n == 0.0 { + return probs; + } + for i in 0..256 { + probs[i] = counts[i] as f64 / n; + } + probs +} + +#[inline(always)] +/// Total variation distance between the byte distributions of `x` and `y`. +/// +/// TVD is an empirical, order-0 quantity by construction. +pub fn tvd_bytes(x: &[u8], y: &[u8]) -> f64 { + if x.is_empty() || y.is_empty() { + return 0.0; + } + let p_x = byte_histogram(x); + let p_y = byte_histogram(y); + + let mut sum = 0.0f64; + for i in 0..256 { + sum += (p_x[i] - p_y[i]).abs(); + } + + (sum / 2.0).clamp(0.0, 1.0) +} + +#[inline(always)] +/// Normalized Hellinger distance between the byte distributions of `x` and `y`. +/// +/// NHD is an empirical, order-0 quantity by construction. +pub fn nhd_bytes(x: &[u8], y: &[u8]) -> f64 { + if x.is_empty() || y.is_empty() { + return 0.0; + } + let p_x = byte_histogram(x); + let p_y = byte_histogram(y); + + let mut bc = 0.0f64; + for i in 0..256 { + bc += (p_x[i] * p_y[i]).sqrt(); + } + + (1.0 - bc).max(0.0).sqrt() +} + +#[inline(always)] +/// Fallible cross-entropy estimate `H_train(test)` using the default context rate backend. +pub fn try_cross_entropy_bytes(test_data: &[u8], train_data: &[u8]) -> InfotheoryResult { + with_default_ctx(|ctx| ctx.try_cross_entropy_bytes(test_data, train_data)) +} + +/// Empirical (zero-order, IID) cross-entropy `H₀_q(p) = -Σ p(x) log₂ q(x)` between +/// the byte histograms of `test_data` (treated as `p`) and `train_data` (treated as `q`). +pub fn empirical_cross_entropy_bytes(test_data: &[u8], train_data: &[u8]) -> f64 { + if test_data.is_empty() { + return 0.0; + } + let p_x = byte_histogram(test_data); + let p_y = byte_histogram(train_data); + let mut h = 0.0f64; + for i in 0..256 { + if p_x[i] > 0.0 { + let q_y = p_y[i].max(1e-12); + h -= p_x[i] * q_y.log2(); + } + } + h +} + +#[inline(always)] +/// Fallible cross-entropy *rate* estimate with the default context backend. +pub fn try_cross_entropy_rate_bytes(test_data: &[u8], train_data: &[u8]) -> InfotheoryResult { + with_default_ctx(|ctx| ctx.try_cross_entropy_rate_bytes(test_data, train_data)) +} + +/// KL divergence `D_KL(P || Q)` between the byte histograms of `x` and `y` (bits). +/// +/// `D_KL` is an empirical, order-0 quantity by construction. +pub fn d_kl_bytes(x: &[u8], y: &[u8]) -> f64 { + if x.is_empty() || y.is_empty() { + return 0.0; + } + let p_x = byte_histogram(x); + let p_y = byte_histogram(y); + let mut d_kl = 0.0f64; + for i in 0..256 { + if p_x[i] > 0.0 { + let q_y = p_y[i].max(1e-12); + d_kl += p_x[i] * (p_x[i] / q_y).log2(); + } + } + d_kl.max(0.0) +} + +/// Jensen-Shannon divergence between the byte histograms of `x` and `y` (bits). +/// +/// JSD is an empirical, order-0 quantity by construction. +pub fn js_div_bytes(x: &[u8], y: &[u8]) -> f64 { + if x.is_empty() || y.is_empty() { + return 0.0; + } + let p_x = byte_histogram(x); + let p_y = byte_histogram(y); + let mut m = [0.0f64; 256]; + for i in 0..256 { + m[i] = 0.5 * (p_x[i] + p_y[i]); + } + + let mut kl_pm = 0.0f64; + let mut kl_qm = 0.0f64; + for i in 0..256 { + if p_x[i] > 0.0 { + kl_pm += p_x[i] * (p_x[i] / m[i]).log2(); + } + if p_y[i] > 0.0 { + kl_qm += p_y[i] * (p_y[i] / m[i]).log2(); + } + } + (0.5 * kl_pm + 0.5 * kl_qm).max(0.0) +} + +#[inline(always)] +/// Fallible intrinsic dependence estimate: +/// `(H₀(X) - Ĥ(X)) / H₀(X)`, where `H₀` is the order-0 / empirical entropy and +/// `Ĥ` is the entropy rate produced by the default context's rate backend. +pub fn try_intrinsic_dependence_bytes(data: &[u8]) -> InfotheoryResult { + with_default_ctx(|ctx| ctx.try_intrinsic_dependence_bytes(data)) +} + +#[inline(always)] +/// Fallible resistance-to-transformation estimate: +/// `I(X; T(X)) / H(X)` for `tx = T(x)`, using the default context rate backend. +pub fn try_resistance_to_transformation_bytes(x: &[u8], tx: &[u8]) -> InfotheoryResult { + with_default_ctx(|ctx| ctx.try_resistance_to_transformation_bytes(x, tx)) +} + +/// Empirical (zero-order, IID) resistance-to-transformation estimate: +/// `I₀(X; T(X)) / H₀(X)` for `tx = T(x)`. +pub fn empirical_resistance_to_transformation_bytes(x: &[u8], tx: &[u8]) -> f64 { + let (x, tx) = aligned_prefix(x, tx); + let h_x = empirical_entropy_bytes(x); + if h_x < 1e-9 { + 0.0 + } else { + (empirical_mutual_information_bytes(x, tx) / h_x).clamp(0.0, 1.0) + } +} diff --git a/crates/infotheory/src/api/mod.rs b/crates/infotheory/src/api/mod.rs new file mode 100644 index 00000000..b52534d3 --- /dev/null +++ b/crates/infotheory/src/api/mod.rs @@ -0,0 +1,65 @@ +//! Public, spec-first infotheory API surface. + +pub(crate) mod compression; +pub(crate) mod context; +pub(crate) mod generation; +pub(crate) mod metrics; +pub(crate) mod paths; +pub(crate) mod types; + +pub use self::context::{ + InfotheoryCtx, RateBackendBitSession, RateBackendBitSessionCheckpoint, RateBackendSession, + get_default_ctx, set_default_ctx, +}; +pub use self::types::{ + CalibratedSpec, CalibrationContextKind, CompressionBackend, GenerationConfig, + GenerationStrategy, GenerationUpdateMode, MAX_MIXTURE_NESTING, MixtureExpertSpec, MixtureKind, + MixtureScheduleMode, MixtureSpec, ParticleSpec, RateBackend, ZpaqMethodSpec, + parse_mixture_kind_name, parse_mixture_schedule_name, validate_compression_backend, + validate_rate_backend, +}; +pub use crate::prediction::{ + BinaryPrediction, BitOrder, BitStreamSemantics, BytePrefixMass, OnlineBitPredictor, + OnlineBytePredictor, +}; +pub use crate::spec::{ + AssetRef, CanonicalBytes, CanonicalJson, CompiledCompressionBackend, CompiledRateBackend, + CompressionBackendCapabilities, MethodBackendFamily, RateBackendCapabilities, + RateBackendTraceStrategy, SpecEnvironment, ValidatedCompressionBackend, ValidatedRateBackend, +}; + +pub use self::compression::{ + NcdComputeOptions, NcdVariant, OperationParallelism, try_compress_bytes_backend, + try_compress_size_backend, try_compress_size_chain_backend, try_decompress_bytes_backend, + try_ncd_bytes_backend, try_ncd_bytes_backend_with_options, try_ncd_bytes_default, + try_ncd_matrix_bytes_backend, try_ncd_matrix_bytes_backend_with_options, + try_ncd_matrix_bytes_default, +}; +pub use self::generation::{ + try_generate_bytes, try_generate_bytes_conditional_chain, + try_generate_bytes_conditional_chain_with_config, try_generate_bytes_with_config, +}; +pub use self::metrics::{ + d_kl_bytes, empirical_cross_entropy_bytes, empirical_entropy_bytes, + empirical_joint_entropy_bytes, empirical_mutual_information_bytes, empirical_ned_bytes, + empirical_ned_cons_bytes, empirical_nte_bytes, empirical_resistance_to_transformation_bytes, + js_div_bytes, nhd_bytes, try_biased_entropy_rate_backend, try_biased_entropy_rate_bytes, + try_conditional_entropy_bytes, try_conditional_entropy_rate_bytes, try_cross_entropy_bytes, + try_cross_entropy_rate_backend, try_cross_entropy_rate_bytes, try_entropy_rate_backend, + try_entropy_rate_bytes, try_intrinsic_dependence_bytes, try_joint_entropy_rate_backend, + try_joint_entropy_rate_bytes, try_mutual_information_bytes, + try_mutual_information_rate_backend, try_mutual_information_rate_bytes, try_ned_bytes, + try_ned_cons_bytes, try_ned_cons_rate_bytes, try_ned_rate_backend, try_ned_rate_bytes, + try_nte_bytes, try_nte_rate_backend, try_nte_rate_bytes, + try_resistance_to_transformation_bytes, tvd_bytes, +}; +pub use self::paths::{ + CompressionPathBatchOptions, try_conditional_entropy_paths, try_cross_entropy_paths, + try_get_bytes_from_paths, try_get_compressed_size_path_backend, + try_get_compressed_sizes_from_paths_backend, + try_get_compressed_sizes_from_paths_backend_with_options, try_js_divergence_paths, + try_kl_divergence_paths, try_mutual_information_paths, try_ncd_matrix_paths_backend, + try_ncd_matrix_paths_backend_with_options, try_ncd_paths_backend, + try_ncd_paths_compiled_backend, try_ncd_paths_compiled_backend_with_options, try_ned_paths, + try_nhd_paths, try_nte_paths, try_tvd_paths, +}; diff --git a/crates/infotheory/src/api/paths.rs b/crates/infotheory/src/api/paths.rs new file mode 100644 index 00000000..28b8f44d --- /dev/null +++ b/crates/infotheory/src/api/paths.rs @@ -0,0 +1,227 @@ +//! Path-oriented convenience API surface. + +use rayon::prelude::*; + +use super::compression::{ + NcdComputeOptions, NcdVariant, OperationParallelism, try_compress_size_backend, + try_ncd_bytes_backend_with_options, try_ncd_matrix_bytes_backend_with_options, +}; +use super::metrics::{ + d_kl_bytes, js_div_bytes, nhd_bytes, try_conditional_entropy_bytes, try_cross_entropy_bytes, + try_mutual_information_bytes, try_ned_bytes, try_nte_bytes, tvd_bytes, +}; +use super::types::CompressionBackend; +use crate::error::{InfotheoryError, InfotheoryResult}; +use crate::spec::CompiledCompressionBackend; +use rayon::ThreadPoolBuilder; + +#[inline(always)] +fn try_read_path_pair(x: &str, y: &str) -> InfotheoryResult<(Vec, Vec)> { + let (bx, by) = rayon::join( + || std::fs::read(x).map_err(InfotheoryError::from), + || std::fs::read(y).map_err(InfotheoryError::from), + ); + Ok((bx?, by?)) +} + +/// Options for backend-first path compression-size operations. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct CompressionPathBatchOptions { + /// Operation-level parallelism policy used while processing the path batch. (external parellization, doesn't affect compression algorithm itself) + pub parallelism: OperationParallelism, +} + +/// Read `path` and return its compressed size (bytes) using `backend`. +pub fn try_get_compressed_size_path_backend( + path: &str, + backend: &CompiledCompressionBackend, +) -> InfotheoryResult { + let data = std::fs::read(path)?; + try_compress_size_backend(&data, backend) +} + +#[inline(always)] +/// Read all files in `paths` in parallel and return their contents. +pub fn try_get_bytes_from_paths(paths: &[&str]) -> InfotheoryResult>> { + paths + .par_iter() + .map(|path| std::fs::read(*path).map_err(InfotheoryError::from)) + .collect() +} + +/// Compute compressed sizes for `paths` using `backend`. +pub fn try_get_compressed_sizes_from_paths_backend( + paths: &[&str], + backend: &CompiledCompressionBackend, +) -> InfotheoryResult> { + try_get_compressed_sizes_from_paths_backend_with_options( + paths, + backend, + CompressionPathBatchOptions::default(), + ) +} + +/// Compute compressed sizes for `paths` using `backend` with explicit +/// operation-level parallelism controls. +pub fn try_get_compressed_sizes_from_paths_backend_with_options( + paths: &[&str], + backend: &CompiledCompressionBackend, + options: CompressionPathBatchOptions, +) -> InfotheoryResult> { + match options.parallelism { + OperationParallelism::Serial => paths + .iter() + .map(|path| try_get_compressed_size_path_backend(path, backend)) + .collect(), + OperationParallelism::Auto => paths + .par_iter() + .map(|path| try_get_compressed_size_path_backend(path, backend)) + .collect(), + OperationParallelism::Threads(threads) => { + if threads <= 1 { + return paths + .iter() + .map(|path| try_get_compressed_size_path_backend(path, backend)) + .collect(); + } + let pool = ThreadPoolBuilder::new() + .num_threads(threads) + .build() + .map_err(|err| { + InfotheoryError::runtime(format!("failed to build rayon pool: {err}")) + })?; + pool.install(|| { + paths + .par_iter() + .map(|path| try_get_compressed_size_path_backend(path, backend)) + .collect() + }) + } + } +} + +/// Compute NCD for two files with an explicit compression backend. +pub fn try_ncd_paths_backend( + x: &str, + y: &str, + backend: &CompressionBackend, + variant: NcdVariant, +) -> InfotheoryResult { + let (bx, by) = rayon::join( + || std::fs::read(x).map_err(InfotheoryError::from), + || std::fs::read(y).map_err(InfotheoryError::from), + ); + let compiled = backend + .compile() + .map_err(|err| InfotheoryError::invalid_backend_config(err.to_string()))?; + try_ncd_bytes_backend_with_options(&bx?, &by?, &compiled, variant, NcdComputeOptions::default()) +} + +#[inline(always)] +/// Compute NCD for two files with an explicit compiled compression backend. +pub fn try_ncd_paths_compiled_backend( + x: &str, + y: &str, + backend: &CompiledCompressionBackend, + variant: NcdVariant, +) -> InfotheoryResult { + try_ncd_paths_compiled_backend_with_options( + x, + y, + backend, + variant, + NcdComputeOptions::default(), + ) +} + +/// Compute NCD for two files with an explicit compiled compression backend and +/// explicit operation-level parallelism controls. +pub fn try_ncd_paths_compiled_backend_with_options( + x: &str, + y: &str, + backend: &CompiledCompressionBackend, + variant: NcdVariant, + options: NcdComputeOptions, +) -> InfotheoryResult { + let (bx, by) = rayon::join( + || std::fs::read(x).map_err(InfotheoryError::from), + || std::fs::read(y).map_err(InfotheoryError::from), + ); + try_ncd_bytes_backend_with_options(&bx?, &by?, backend, variant, options) +} + +/// Compute an `n x n` pairwise NCD matrix (row-major) for file paths with an explicit compiled compression backend. +pub fn try_ncd_matrix_paths_backend( + paths: &[&str], + backend: &CompiledCompressionBackend, + variant: NcdVariant, +) -> InfotheoryResult> { + try_ncd_matrix_paths_backend_with_options(paths, backend, variant, NcdComputeOptions::default()) +} + +/// Compute an `n x n` pairwise NCD matrix (row-major) for file paths with an +/// explicit compiled compression backend and operation-level parallelism +/// controls. +pub fn try_ncd_matrix_paths_backend_with_options( + paths: &[&str], + backend: &CompiledCompressionBackend, + variant: NcdVariant, + options: NcdComputeOptions, +) -> InfotheoryResult> { + let datas = try_get_bytes_from_paths(paths)?; + try_ncd_matrix_bytes_backend_with_options(&datas, backend, variant, options) +} + +/// Compute normalized entropy distance (NED) for two files using the default rate backend. +pub fn try_ned_paths(x: &str, y: &str) -> InfotheoryResult { + let (bx, by) = try_read_path_pair(x, y)?; + try_ned_bytes(&bx, &by) +} + +/// Compute normalized transform effort (NTE) for two files using the default rate backend. +pub fn try_nte_paths(x: &str, y: &str) -> InfotheoryResult { + let (bx, by) = try_read_path_pair(x, y)?; + try_nte_bytes(&bx, &by) +} + +/// Compute total variation distance (TVD) between the byte distributions of two files. +pub fn try_tvd_paths(x: &str, y: &str) -> InfotheoryResult { + let (bx, by) = try_read_path_pair(x, y)?; + Ok(tvd_bytes(&bx, &by)) +} + +/// Compute normalized Hellinger distance (NHD) between the byte distributions of two files. +pub fn try_nhd_paths(x: &str, y: &str) -> InfotheoryResult { + let (bx, by) = try_read_path_pair(x, y)?; + Ok(nhd_bytes(&bx, &by)) +} + +/// Compute mutual information estimate for two files using the default rate backend. +pub fn try_mutual_information_paths(x: &str, y: &str) -> InfotheoryResult { + let (bx, by) = try_read_path_pair(x, y)?; + try_mutual_information_bytes(&bx, &by) +} + +/// Compute conditional entropy estimate `H(X|Y)` for two files using the default rate backend. +pub fn try_conditional_entropy_paths(x: &str, y: &str) -> InfotheoryResult { + let (bx, by) = try_read_path_pair(x, y)?; + try_conditional_entropy_bytes(&bx, &by) +} + +/// Compute cross-entropy estimate `H_train(test)` for two files using the default rate backend. +pub fn try_cross_entropy_paths(x: &str, y: &str) -> InfotheoryResult { + let (bx, by) = try_read_path_pair(x, y)?; + try_cross_entropy_bytes(&bx, &by) +} + +/// Compute KL divergence between the byte histograms of two files. +pub fn try_kl_divergence_paths(x: &str, y: &str) -> InfotheoryResult { + let (bx, by) = try_read_path_pair(x, y)?; + Ok(d_kl_bytes(&bx, &by)) +} + +/// Compute Jensen-Shannon divergence between the byte histograms of two files. +pub fn try_js_divergence_paths(x: &str, y: &str) -> InfotheoryResult { + let (bx, by) = try_read_path_pair(x, y)?; + Ok(js_div_bytes(&bx, &by)) +} diff --git a/crates/infotheory/src/api/types.rs b/crates/infotheory/src/api/types.rs new file mode 100644 index 00000000..499783cc --- /dev/null +++ b/crates/infotheory/src/api/types.rs @@ -0,0 +1,1058 @@ +//! Public type definitions for the spec-first API surface. + +use crate::coders::CoderType; +use crate::error::{InfotheoryError, InfotheoryResult}; +use std::num::NonZeroUsize; +use std::sync::Arc; + +/// Typed ZPAQ method specification. +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum ZpaqMethodSpec { + /// Literal ZPAQ method string. + Literal { + /// ZPAQ method string accepted by the backend. + value: String, + }, +} + +impl ZpaqMethodSpec { + /// Construct a literal ZPAQ method specification. + pub fn literal(value: impl Into) -> Self { + Self::Literal { + value: value.into(), + } + } + + /// Return the canonical method string. + pub fn value(&self) -> &str { + match self { + Self::Literal { value } => value, + } + } +} + +impl From for ZpaqMethodSpec { + fn from(value: String) -> Self { + Self::literal(value) + } +} + +impl From<&str> for ZpaqMethodSpec { + fn from(value: &str) -> Self { + Self::literal(value) + } +} + +/// How generated symbols should update the model state. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[non_exhaustive] +pub enum GenerationUpdateMode { + /// Keep adapting/fitting on generated bytes. + Adaptive, + /// Freeze fitted parameters/statistics and only advance conditioning state. + Frozen, +} + +/// How to pick the next byte from the model distribution. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[non_exhaustive] +pub enum GenerationStrategy { + /// Deterministic argmax over the next-byte distribution. + Greedy, + /// Seeded sampling from the next-byte distribution. + Sample, +} + +/// Generation options shared by the library API and CLI. +#[derive(Clone, Copy, Debug)] +#[non_exhaustive] +pub struct GenerationConfig { + /// Byte-selection strategy. + pub strategy: GenerationStrategy, + /// Whether generated bytes should keep adapting the model. + pub update_mode: GenerationUpdateMode, + /// RNG seed used by [`GenerationStrategy::Sample`]. + pub seed: u64, + /// Softmax temperature for sampling. `<= 0` behaves like greedy. + pub temperature: f64, + /// Optional top-k truncation. `0` disables it. + pub top_k: usize, + /// Optional nucleus truncation. Values `>= 1.0` disable it. + pub top_p: f64, +} + +impl Default for GenerationConfig { + fn default() -> Self { + Self::sampled_frozen(42) + } +} + +impl GenerationConfig { + /// Deterministic frozen continuation. + pub const fn greedy_frozen() -> Self { + Self { + strategy: GenerationStrategy::Greedy, + update_mode: GenerationUpdateMode::Frozen, + seed: 0xD00D_F00D_CAFE_BABEu64, + temperature: 1.0, + top_k: 0, + top_p: 1.0, + } + } + + /// Seeded frozen sampling from the model distribution. + pub const fn sampled_frozen(seed: u64) -> Self { + Self { + strategy: GenerationStrategy::Sample, + update_mode: GenerationUpdateMode::Frozen, + seed, + temperature: 1.0, + top_k: 0, + top_p: 1.0, + } + } +} + +/// Core predictive model class used by the library. +/// +/// `RateBackend` is the shared model class behind entropy-rate estimation, +/// rate-coded compression, generation, and the world-model interface used by +/// MC-AIXI/AIQI planners. +#[derive(Clone)] +#[non_exhaustive] +pub enum RateBackend { + /// ROSA+ suffix-automaton estimator. + /// + /// `max_order < 0` enables ROSA's adaptive order selection over the full + /// suffix automaton; `max_order >= 0` caps the predictive context length. + RosaPlus { + /// Maximum context length used by the ROSA predictor. + /// + /// `< 0` means "adaptive / full SAM"; `>= 0` is a fixed cap. + max_order: i64, + }, + /// Local contiguous match predictor. + Match { + /// Number of retained hash bits for suffix lookup. + hash_bits: usize, + /// Minimum repeat length required before predicting. + min_len: usize, + /// Maximum repeat length used for confidence scaling. + max_len: usize, + /// Residual probability mass left for non-match symbols. + base_mix: f64, + /// Confidence multiplier applied to short-match tapering. + confidence_scale: f64, + }, + /// Sparse/gapped local match predictor. + SparseMatch { + /// Number of retained hash bits for spaced-suffix lookup. + hash_bits: usize, + /// Minimum spaced repeat length required before predicting. + min_len: usize, + /// Maximum spaced repeat length used for confidence scaling. + max_len: usize, + /// Minimum gap between matched bytes. + gap_min: usize, + /// Maximum gap between matched bytes. + gap_max: usize, + /// Residual probability mass left for non-match symbols. + base_mix: f64, + /// Confidence multiplier applied to short-match tapering. + confidence_scale: f64, + }, + /// Pure-Rust bounded-memory PPMD-style model. + Ppmd { + /// Maximum context order. + order: usize, + /// Approximate memory budget in MiB. + memory_mb: usize, + }, + /// Exact online Sequitur grammar backend with byte-level predictive readout. + Sequitur { + /// Maximum number of terminal bytes retained per grammar-derived context. + context_bytes: usize, + }, + #[cfg(feature = "backend-mamba")] + /// Typed Mamba method specification. + MambaMethod { + /// Mamba method specification. + method: crate::mambazip::MethodSpec, + }, + #[cfg(feature = "backend-rwkv")] + /// Typed RWKV7 method specification. + Rwkv7Method { + /// RWKV7 method specification. + method: crate::rwkvzip::MethodSpec, + }, + /// ZPAQ compression-based rate model (streamable methods only). + Zpaq { + /// Typed ZPAQ method specification. + method: ZpaqMethodSpec, + }, + /// Online mixture over `RateBackend` experts. + /// + /// `Bayes`, `Switching`, and `Convex` follow + /// "On Ensemble Techniques for AIXI Approximation"; `FadingBayes`, + /// `Mdl`, and `Neural` are repository extensions. + Mixture { + /// Mixture expert/runtime specification. + spec: Arc, + }, + /// Particle-latent filter ensemble. + Particle { + /// Particle filter specification. + spec: Arc, + }, + /// Calibrated wrapper over another bytewise backend. + Calibrated { + /// Calibration specification. + spec: Arc, + }, + /// Action-Conditional CTW (single context tree). + Ctw { + /// Context tree depth. + depth: usize, + }, + /// Factorized Action-Conditional CTW (k trees for k-bit percepts). + FacCtw { + /// Base context depth. + base_depth: usize, + /// Planner/percept bit cardinality for AIXI-style consumers. + /// + /// Rate-backend byte execution uses `encoding_bits` as the symbol + /// decomposition width. `num_percept_bits` is retained for controller + /// specs whose percept cardinality can differ from byte encoding width. + num_percept_bits: usize, + /// Encoding width in bits for rate execution. + /// + /// Valid compiled FAC-CTW specs require `1..=8`. + encoding_bits: usize, + /// Optional bit order for decomposing symbols. + /// + /// `None` keeps the compatibility default: 8-bit byte symbols use + /// MSB-first ordering, while non-byte widths retain the legacy + /// LSB-first FAC-CTW convention. `Some(true)` selects MSB-first + /// explicitly; `Some(false)` selects legacy LSB-first explicitly. + msb_first: Option, + }, +} + +/// Compression backend used by NCD/compression-size operations. +#[derive(Clone)] +#[non_exhaustive] +pub enum CompressionBackend { + /// ZPAQ compressor with explicit method string. + Zpaq { + /// Typed ZPAQ method specification. + method: ZpaqMethodSpec, + /// Internal ZPAQ compression thread count (`1` => single-threaded). + threads: NonZeroUsize, + }, + #[cfg(feature = "backend-rwkv")] + /// RWKV7 compressor configured by typed method specification. + Rwkv7 { + /// RWKV7 method specification. + method: crate::rwkvzip::MethodSpec, + /// Entropy coder used for coding model PDFs. + coder: CoderType, + }, + /// Generic rate-coded compressor wrapping an arbitrary rate backend. + Rate { + /// Predictive rate backend. + rate_backend: RateBackend, + /// Entropy coder used for coding model PDFs. + coder: CoderType, + /// Framing mode for output payloads. + framing: crate::compression::FramingMode, + }, +} + +/// Shared maximum nesting depth for recursive mixture specifications. +pub const MAX_MIXTURE_NESTING: usize = 8; + +/// Mixture policy kind for rate-backend mixtures. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub enum MixtureKind { + /// Standard Bayesian mixture with fixed expert weights. + Bayes, + /// Bayesian mixture with exponential weight decay. + FadingBayes, + /// Switching mixture using the fixed-share update from + /// "On Ensemble Techniques for AIXI Approximation". + Switching, + /// Online convex mixture with projected-simplex weight updates. + Convex, + /// MDL-style best-expert selector. + Mdl, + /// Bytewise neural logistic mixer (PAQ style adaptation). + Neural, +} + +/// Adaptive schedule family for switching and convex mixtures. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +#[non_exhaustive] +pub enum MixtureScheduleMode { + /// Use the implementation's default exposed parameterization. + /// + /// - `Switching`: constant switch rate `alpha` + /// - `Convex`: step size `alpha / sqrt(t)` + #[default] + Default, + /// Use the theorem schedule from + /// "On Ensemble Techniques for AIXI Approximation". + /// + /// - `Switching`: `alpha_t = 1 / t` + /// - `Convex`: `eta_t = epsilon / sqrt(t)` under this implementation's + /// natural-log gradient, matching the bit-loss schedule analyzed in + /// "On Ensemble Techniques for AIXI Approximation" after + /// accounting for the `1 / ln(2)` factor in the base-2 gradient + /// + /// This preserves configured expert priors; exact theorem hypotheses for + /// switching still additionally require uniform priors. + Theorem, +} + +/// Fixed context families for calibrated PDF wrappers. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub enum CalibrationContextKind { + /// Single global calibration row. + Global, + /// Previous-byte class only. + ByteClass, + /// Text-structure-aware context hash. + Text, + /// Repeat-aware context hash. + Repeat, + /// Joint text/repeat-aware context hash. + TextRepeat, +} + +/// Configuration for a calibrated wrapper rate backend. +#[derive(Clone)] +#[non_exhaustive] +pub struct CalibratedSpec { + /// Base backend whose PDF is calibrated. + pub base: RateBackend, + /// Context family controlling table row selection. + pub context: CalibrationContextKind, + /// Number of probability bins per row. + pub bins: usize, + /// Online learning rate for observed-symbol updates. + pub learning_rate: f64, + /// Symmetric clip applied to calibration weights. + pub bias_clip: f64, +} + +impl CalibratedSpec { + /// Create a new calibrated spec with default parameters. + pub fn new(base: RateBackend, context: CalibrationContextKind) -> Self { + Self { + base, + context, + bins: 33, + learning_rate: 0.02, + bias_clip: 4.0, + } + } + + /// Override the per-row bin count. + #[must_use] + pub fn with_bins(mut self, bins: usize) -> Self { + self.bins = bins; + self + } + + /// Override the online learning rate. + #[must_use] + pub fn with_learning_rate(mut self, learning_rate: f64) -> Self { + self.learning_rate = learning_rate; + self + } + + /// Override the symmetric bias clip. + #[must_use] + pub fn with_bias_clip(mut self, bias_clip: f64) -> Self { + self.bias_clip = bias_clip; + self + } +} + +/// Expert specification for mixture backends. +/// +/// Algorithm-specific configuration (such as ROSA's `max_order`) lives inside +/// the expert's [`RateBackend`] variant; the expert spec itself only carries +/// mixture-level metadata. +#[derive(Clone)] +#[non_exhaustive] +pub struct MixtureExpertSpec { + /// Optional expert display name. + pub name: Option, + /// Log prior weight (natural log). Uniform priors can be `0.0`. + pub log_prior: f64, + /// Underlying backend for this expert. + pub backend: RateBackend, +} + +impl MixtureExpertSpec { + /// Create a new expert spec with a default 0.0 prior and no length limits. + pub fn new(backend: RateBackend) -> Self { + Self { + name: None, + log_prior: 0.0, + backend, + } + } + + /// Set the human-readable expert name. + #[must_use] + pub fn with_name(mut self, name: impl Into) -> Self { + self.name = Some(name.into()); + self + } + + /// Clear any previously set expert name. + #[must_use] + pub fn without_name(mut self) -> Self { + self.name = None; + self + } + + /// Set the log-prior weight (natural log) for this expert. + #[must_use] + pub fn with_log_prior(mut self, log_prior: f64) -> Self { + self.log_prior = log_prior; + self + } +} + +/// Mixture specification for rate-backend mixtures. +#[derive(Clone)] +#[non_exhaustive] +pub struct MixtureSpec { + /// Mixture policy. + pub kind: MixtureKind, + /// Adaptive schedule family for supported mixture kinds. + pub schedule: MixtureScheduleMode, + /// Shared scalar parameter: switch rate for `Switching`, step-size scale for `Convex`, + /// learning rate for `Neural`, and generic alpha for the remaining families. + /// + /// In theorem mode for `Switching` and `Convex`, this field is retained for API + /// compatibility but is not used by the update schedule. + pub alpha: f64, + /// Decay factor for fading Bayes mixtures. + pub decay: Option, + /// Expert list. + pub experts: Vec, +} + +/// Configuration for a particle-latent filter ensemble rate backend. +#[derive(Clone, Debug)] +#[non_exhaustive] +pub struct ParticleSpec { + /// Number of particles in the ensemble. + pub num_particles: usize, + /// Context window length for rolling byte context. + pub context_window: usize, + /// Number of latent update unroll steps per byte. + pub unroll_steps: usize, + /// Number of latent cells per particle. + pub num_cells: usize, + /// Dimensionality of each latent cell. + pub cell_dim: usize, + /// Number of discrete rules for soft routing. + pub num_rules: usize, + /// Hidden dimension for the selector MLP. + pub selector_hidden: usize, + /// Hidden dimension for each rule MLP. + pub rule_hidden: usize, + /// Dimension of per-rule noise input (ignored when deterministic). + pub noise_dim: usize, + /// Whether to use fully deterministic execution (no RNG). + pub deterministic: bool, + /// Whether to inject noise into rule inputs (ignored when deterministic). + pub enable_noise: bool, + /// Base scale for deterministic hash-noise injected into rule inputs. + pub noise_scale: f64, + /// Number of steps over which injected noise linearly anneals to zero. + pub noise_anneal_steps: usize, + /// Learning rate for readout layer SGD. + pub learning_rate_readout: f64, + /// Learning rate for selector MLP SGD. + pub learning_rate_selector: f64, + /// Learning rate for rule MLP SGD. + pub learning_rate_rule: f64, + /// Truncated backpropagation-through-time depth (number of recent steps). + pub bptt_depth: usize, + /// Momentum coefficient for selector/rule online updates (in [0, 1)). + pub optimizer_momentum: f64, + /// Gradient clipping threshold (max abs value per element). + pub grad_clip: f64, + /// Latent cell state clipping threshold (max abs value per element). + pub state_clip: f64, + /// Forgetting factor for particle log-weights (0 = no forgetting). + pub forget_lambda: f64, + /// Effective sample size ratio threshold for resampling (in (0, 1]). + pub resample_threshold: f64, + /// Fraction of particles to mutate after resampling (in [0, 1]). + pub mutate_fraction: f64, + /// Scale of hash-noise perturbation applied during mutation. + pub mutate_scale: f64, + /// Whether mutation also perturbs model parameters (state is always mutated). + pub mutate_model_params: bool, + /// Diagnostics print interval in steps (0 disables particle diagnostics logs). + pub diagnostics_interval: usize, + /// Minimum probability floor for numerical stability. + pub min_prob: f64, + /// Master seed for deterministic initialization and mutation. + pub seed: u64, +} + +impl RateBackend { + /// Returns the current build's implicit default rate backend. + pub fn try_default() -> InfotheoryResult { + crate::runtime::first_enabled_default_rate_backend_spec().ok_or_else(|| { + let mut requested = Vec::new(); + for descriptor in crate::runtime::RATE_BACKEND_REGISTRY { + if let Some(feature) = descriptor.feature + && !requested.contains(&feature) + { + requested.push(feature); + } + } + InfotheoryError::unsupported(format!( + "no default rate backend is available in this build; enable one of: {}", + requested.join(", ") + )) + }) + } + + /// Stable internal backend identity. + pub(crate) fn kind(&self) -> crate::runtime::RateBackendKind { + match self { + RateBackend::RosaPlus { .. } => crate::runtime::RateBackendKind::RosaPlus, + RateBackend::Match { .. } => crate::runtime::RateBackendKind::Match, + RateBackend::SparseMatch { .. } => crate::runtime::RateBackendKind::SparseMatch, + RateBackend::Ppmd { .. } => crate::runtime::RateBackendKind::Ppmd, + RateBackend::Sequitur { .. } => crate::runtime::RateBackendKind::Sequitur, + #[cfg(feature = "backend-mamba")] + RateBackend::MambaMethod { .. } => crate::runtime::RateBackendKind::Mamba, + #[cfg(feature = "backend-rwkv")] + RateBackend::Rwkv7Method { .. } => crate::runtime::RateBackendKind::Rwkv7, + RateBackend::Zpaq { .. } => crate::runtime::RateBackendKind::Zpaq, + RateBackend::Mixture { .. } => crate::runtime::RateBackendKind::Mixture, + RateBackend::Particle { .. } => crate::runtime::RateBackendKind::Particle, + RateBackend::Calibrated { .. } => crate::runtime::RateBackendKind::Calibrated, + RateBackend::Ctw { .. } => crate::runtime::RateBackendKind::Ctw, + RateBackend::FacCtw { .. } => crate::runtime::RateBackendKind::FacCtw, + } + } + + pub(crate) fn descriptor( + &self, + ) -> Result<&'static crate::runtime::RateBackendDescriptor, String> { + crate::runtime::describe_rate_backend_kind(self.kind()) + } + + /// Validate and canonicalize this backend in the provided compilation environment. + pub fn validate_in( + &self, + env: &crate::spec::SpecEnvironment, + ) -> crate::spec::SpecResult { + crate::spec::core::validate_rate_backend_in(self, env) + } + + /// Validate and canonicalize this backend using the default environment. + pub fn validate(&self) -> crate::spec::SpecResult { + self.validate_in(&crate::spec::SpecEnvironment::default()) + } + + /// Validate and compile this backend in the provided compilation environment. + pub fn compile_in( + &self, + env: &crate::spec::SpecEnvironment, + ) -> crate::spec::SpecResult { + self.validate_in(env)?.compile() + } + + /// Validate and compile this backend using the default environment. + pub fn compile(&self) -> crate::spec::SpecResult { + self.validate()?.compile() + } +} + +impl CompressionBackend { + /// Construct a ZPAQ compression backend with default internal threading (`threads = 1`). + pub fn zpaq(method: impl Into) -> Self { + Self::Zpaq { + method: method.into(), + threads: NonZeroUsize::MIN, + } + } + + /// Construct a ZPAQ compression backend with explicit internal thread count. + pub fn zpaq_with_threads(method: impl Into, threads: NonZeroUsize) -> Self { + Self::Zpaq { + method: method.into(), + threads, + } + } + + /// Returns the current build's implicit default compression backend. + pub fn try_default() -> InfotheoryResult { + if crate::runtime::COMPRESSION_BACKEND_REGISTRY + .iter() + .any(|descriptor| { + descriptor.enabled + && descriptor.kind == crate::runtime::CompressionBackendKind::Zpaq + }) + { + return Ok(Self::zpaq("5")); + } + + Ok(CompressionBackend::Rate { + rate_backend: RateBackend::try_default()?, + coder: crate::coders::CoderType::AC, + framing: crate::compression::FramingMode::Raw, + }) + } + + /// Stable internal backend identity. + pub(crate) fn kind(&self) -> crate::runtime::CompressionBackendKind { + match self { + CompressionBackend::Zpaq { .. } => crate::runtime::CompressionBackendKind::Zpaq, + #[cfg(feature = "backend-rwkv")] + CompressionBackend::Rwkv7 { .. } => crate::runtime::CompressionBackendKind::Rwkv7, + CompressionBackend::Rate { coder, .. } => match coder { + crate::coders::CoderType::AC => crate::runtime::CompressionBackendKind::RateAc, + crate::coders::CoderType::RANS => crate::runtime::CompressionBackendKind::RateRans, + }, + } + } + + /// Canonical registry descriptor for this compression backend. + pub(crate) fn descriptor( + &self, + ) -> Result<&'static crate::runtime::CompressionBackendDescriptor, String> { + crate::runtime::describe_compression_backend_kind(self.kind()) + } + + /// Validate and canonicalize this backend in the provided compilation environment. + pub fn validate_in( + &self, + env: &crate::spec::SpecEnvironment, + ) -> crate::spec::SpecResult { + crate::spec::core::validate_compression_backend_in(self, env) + } + + /// Validate and canonicalize this backend using the default environment. + pub fn validate(&self) -> crate::spec::SpecResult { + self.validate_in(&crate::spec::SpecEnvironment::default()) + } + + /// Validate and compile this backend in the provided compilation environment. + pub fn compile_in( + &self, + env: &crate::spec::SpecEnvironment, + ) -> crate::spec::SpecResult { + self.validate_in(env)?.compile() + } + + /// Validate and compile this backend using the default environment. + pub fn compile(&self) -> crate::spec::SpecResult { + self.validate()?.compile() + } +} +/// Parse a mixture kind name with the shared alias table used across CLI, Python, and WASM. +pub fn parse_mixture_kind_name(kind: &str) -> Result { + match kind.trim().to_ascii_lowercase().as_str() { + "bayes" => Ok(MixtureKind::Bayes), + "fading-bayes" => Ok(MixtureKind::FadingBayes), + "switching" => Ok(MixtureKind::Switching), + "convex" => Ok(MixtureKind::Convex), + "mdl" => Ok(MixtureKind::Mdl), + "neural" => Ok(MixtureKind::Neural), + other => Err(format!("unknown mixture kind '{other}'")), + } +} + +/// Parse a mixture schedule mode with the shared alias table used across CLI, Python, and WASM. +pub fn parse_mixture_schedule_name(schedule: &str) -> Result { + match schedule.trim().to_ascii_lowercase().as_str() { + "" | "default" => Ok(MixtureScheduleMode::Default), + "theorem" => Ok(MixtureScheduleMode::Theorem), + other => Err(format!("unknown mixture schedule '{other}'")), + } +} + +impl MixtureSpec { + /// Build a mixture specification from kind and expert list. + pub fn new(kind: MixtureKind, experts: Vec) -> Self { + Self { + kind, + schedule: MixtureScheduleMode::Default, + alpha: 0.01, + decay: None, + experts, + } + } + + /// Set the schedule family. + pub fn with_schedule(mut self, schedule: MixtureScheduleMode) -> Self { + self.schedule = schedule; + self + } + + /// Set the family-specific alpha parameter. + pub fn with_alpha(mut self, alpha: f64) -> Self { + self.alpha = alpha; + self + } + + /// Set fading decay factor. + pub fn with_decay(mut self, decay: f64) -> Self { + self.decay = Some(decay); + self + } + + /// Validate the mixture configuration before building runtime state. + pub fn validate(&self) -> InfotheoryResult<()> { + validate_mixture_spec_with_depth(self, MAX_MIXTURE_NESTING) + .map_err(InfotheoryError::invalid_backend_config) + } + + /// Convert to executable expert configs for runtime mixture evaluation. + pub fn build_experts(&self) -> Vec { + self.experts + .iter() + .map(|spec| { + crate::mixture::ExpertConfig::from_rate_backend( + spec.name.clone(), + spec.log_prior, + spec.backend.clone(), + ) + }) + .collect() + } +} + +fn validate_mixture_spec_with_depth(spec: &MixtureSpec, depth: usize) -> Result<(), String> { + if depth == 0 { + return Err("mixture spec nesting too deep".to_string()); + } + validate_mixture_spec_shallow(spec)?; + for (index, expert) in spec.experts.iter().enumerate() { + validate_rate_backend_with_depth(&expert.backend, depth - 1).map_err(|err| { + if let Some(name) = expert.name.as_deref() { + format!("mixture expert '{name}' invalid: {err}") + } else { + format!("mixture expert #{} invalid: {err}", index + 1) + } + })?; + } + Ok(()) +} + +fn validate_mixture_spec_shallow(spec: &MixtureSpec) -> Result<(), String> { + if spec.experts.is_empty() { + return Err("mixture spec must include at least one expert".to_string()); + } + if !spec.alpha.is_finite() { + return Err("mixture alpha must be finite".to_string()); + } + if spec + .experts + .iter() + .any(|expert| !expert.log_prior.is_finite()) + { + return Err("mixture expert log_prior must be finite".to_string()); + } + if let Some(decay) = spec.decay + && !(decay.is_finite() && decay > 0.0 && decay < 1.0) + { + return Err("mixture decay must be in (0, 1)".to_string()); + } + if matches!(spec.kind, MixtureKind::FadingBayes) && spec.decay.is_none() { + return Err("fading Bayes mixture requires decay".to_string()); + } + if spec.schedule != MixtureScheduleMode::Default + && !matches!(spec.kind, MixtureKind::Switching | MixtureKind::Convex) + { + return Err( + "mixture schedule is only supported for switching and convex mixtures".to_string(), + ); + } + match (spec.kind, spec.schedule) { + (MixtureKind::Switching, MixtureScheduleMode::Default) => { + if !(0.0..=1.0).contains(&spec.alpha) { + return Err("switching mixture alpha must be in [0, 1]".to_string()); + } + } + (MixtureKind::Convex, MixtureScheduleMode::Default) + | (MixtureKind::Neural, MixtureScheduleMode::Default) => { + if spec.alpha <= 0.0 { + return Err("mixture alpha must be > 0".to_string()); + } + } + (MixtureKind::Neural, MixtureScheduleMode::Theorem) => unreachable!(), + _ => {} + } + Ok(()) +} + +fn validate_rate_backend_with_depth(backend: &RateBackend, depth: usize) -> Result<(), String> { + let descriptor = crate::runtime::try_describe_rate_backend(backend) + .map_err(|err| format!("{err} (while validating rate backend)"))?; + if !descriptor.enabled { + let Some(feature) = descriptor.feature else { + return Err(format!( + "internal backend registry mismatch: disabled backend '{}' is missing required feature metadata", + descriptor.canonical + )); + }; + return Err(format!( + "backend '{}' requires infotheory feature '{}'", + descriptor.canonical, feature + )); + } + match backend { + RateBackend::Sequitur { context_bytes } => { + if *context_bytes < 2 { + Err("sequitur context_bytes must be >= 2".to_string()) + } else { + Ok(()) + } + } + RateBackend::Mixture { spec } => validate_mixture_spec_with_depth(spec.as_ref(), depth), + RateBackend::Particle { spec } => spec.validate().map_err(|err| err.to_string()), + RateBackend::Calibrated { spec } => { + if depth == 0 { + return Err("calibrated spec nesting too deep".to_string()); + } + validate_rate_backend_with_depth(&spec.base, depth - 1) + .map_err(|err| format!("calibrated base invalid: {err}")) + } + _ => Ok(()), + } +} + +/// Validate a rate backend, including nested mixture/calibrated subgraphs. +pub fn validate_rate_backend(backend: &RateBackend) -> InfotheoryResult<()> { + validate_rate_backend_with_depth(backend, MAX_MIXTURE_NESTING) + .map_err(InfotheoryError::invalid_backend_config) +} + +/// Validate a compression backend, including nested rate-backed compressors. +pub fn validate_compression_backend(backend: &CompressionBackend) -> InfotheoryResult<()> { + let descriptor = crate::runtime::try_describe_compression_backend(backend) + .map_err(InfotheoryError::invalid_backend_config)?; + if !descriptor.enabled { + let Some(feature) = descriptor.feature else { + return Err(InfotheoryError::invalid_backend_config(format!( + "internal backend registry mismatch: disabled compression backend '{}' is missing required feature metadata", + descriptor.canonical + ))); + }; + return Err(InfotheoryError::invalid_backend_config(format!( + "compression backend '{}' requires infotheory feature '{}'", + descriptor.canonical, feature + ))); + } + if let CompressionBackend::Rate { rate_backend, .. } = backend { + validate_rate_backend(rate_backend).map_err(|err| { + InfotheoryError::invalid_backend_config(format!( + "rate-coded compression backend invalid: {err}" + )) + })?; + } + Ok(()) +} + +impl Default for ParticleSpec { + fn default() -> Self { + Self { + num_particles: 16, + context_window: 32, + unroll_steps: 2, + num_cells: 8, + cell_dim: 32, + num_rules: 4, + selector_hidden: 64, + rule_hidden: 64, + noise_dim: 8, + deterministic: true, + enable_noise: false, + noise_scale: 0.10, + noise_anneal_steps: 8192, + learning_rate_readout: 0.01, + learning_rate_selector: 1e-4, + learning_rate_rule: 3e-4, + bptt_depth: 3, + optimizer_momentum: 0.05, + grad_clip: 1.0, + state_clip: 8.0, + forget_lambda: 0.0, + resample_threshold: 0.5, + mutate_fraction: 0.1, + mutate_scale: 0.01, + mutate_model_params: false, + diagnostics_interval: 0, + min_prob: 2f64.powi(-24), + seed: 42, + } + } +} + +impl ParticleSpec { + /// Validate all fields, returning an error message on failure. + pub fn validate(&self) -> InfotheoryResult<()> { + if self.num_particles == 0 { + return Err(InfotheoryError::invalid_backend_config( + "num_particles must be > 0", + )); + } + if self.context_window == 0 { + return Err(InfotheoryError::invalid_backend_config( + "context_window must be > 0", + )); + } + if self.unroll_steps == 0 { + return Err(InfotheoryError::invalid_backend_config( + "unroll_steps must be > 0", + )); + } + if self.num_cells == 0 { + return Err(InfotheoryError::invalid_backend_config( + "num_cells must be > 0", + )); + } + if self.cell_dim == 0 { + return Err(InfotheoryError::invalid_backend_config( + "cell_dim must be > 0", + )); + } + if self.num_rules == 0 { + return Err(InfotheoryError::invalid_backend_config( + "num_rules must be > 0", + )); + } + if self.selector_hidden == 0 { + return Err(InfotheoryError::invalid_backend_config( + "selector_hidden must be > 0", + )); + } + if self.rule_hidden == 0 { + return Err(InfotheoryError::invalid_backend_config( + "rule_hidden must be > 0", + )); + } + if !self.learning_rate_readout.is_finite() || self.learning_rate_readout < 0.0 { + return Err(InfotheoryError::invalid_backend_config( + "learning_rate_readout must be finite and non-negative", + )); + } + if !self.learning_rate_selector.is_finite() || self.learning_rate_selector < 0.0 { + return Err(InfotheoryError::invalid_backend_config( + "learning_rate_selector must be finite and non-negative", + )); + } + if !self.learning_rate_rule.is_finite() || self.learning_rate_rule < 0.0 { + return Err(InfotheoryError::invalid_backend_config( + "learning_rate_rule must be finite and non-negative", + )); + } + if !self.noise_scale.is_finite() || self.noise_scale < 0.0 { + return Err(InfotheoryError::invalid_backend_config( + "noise_scale must be finite and non-negative", + )); + } + if !self.optimizer_momentum.is_finite() + || self.optimizer_momentum < 0.0 + || self.optimizer_momentum >= 1.0 + { + return Err(InfotheoryError::invalid_backend_config( + "optimizer_momentum must be finite and in [0, 1)", + )); + } + if self.bptt_depth == 0 { + return Err(InfotheoryError::invalid_backend_config( + "bptt_depth must be > 0", + )); + } + if !(self.resample_threshold > 0.0 && self.resample_threshold <= 1.0) { + return Err(InfotheoryError::invalid_backend_config( + "resample_threshold must be in (0, 1]", + )); + } + if !(self.mutate_fraction >= 0.0 && self.mutate_fraction <= 1.0) { + return Err(InfotheoryError::invalid_backend_config( + "mutate_fraction must be in [0, 1]", + )); + } + if !(self.min_prob > 0.0 && self.min_prob < 0.5) { + return Err(InfotheoryError::invalid_backend_config( + "min_prob must be in (0, 0.5)", + )); + } + Ok(()) + } +} + +impl crate::spec::CanonicalJson for RateBackend { + fn to_canonical_json_value(&self) -> crate::spec::SpecResult { + crate::spec::rate_backend_to_json_value(self) + } +} + +impl crate::spec::CanonicalJson for CompressionBackend { + fn to_canonical_json_value(&self) -> crate::spec::SpecResult { + crate::spec::compression_backend_to_json_value(self) + } +} + +impl crate::spec::CanonicalJson for MixtureSpec { + fn to_canonical_json_value(&self) -> crate::spec::SpecResult { + crate::spec::mixture_spec_to_json_value(self) + } +} + +impl crate::spec::CanonicalJson for ParticleSpec { + fn to_canonical_json_value(&self) -> crate::spec::SpecResult { + Ok(crate::spec::particle_spec_to_json_value(self)) + } +} + +impl crate::spec::CanonicalJson for CalibratedSpec { + fn to_canonical_json_value(&self) -> crate::spec::SpecResult { + crate::spec::calibrated_spec_to_json_value(self) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn fading_bayes_decay_rejects_zero_and_one_boundaries() { + let expert = MixtureExpertSpec { + name: Some("placeholder".to_string()), + log_prior: 0.0, + backend: RateBackend::RosaPlus { max_order: -1 }, + }; + + for &decay in &[0.0, 1.0] { + let spec = + MixtureSpec::new(MixtureKind::FadingBayes, vec![expert.clone()]).with_decay(decay); + let err = validate_mixture_spec_shallow(&spec).expect_err("boundary decay should fail"); + assert!( + err.contains("(0, 1)"), + "unexpected error for decay={decay}: {err}" + ); + } + + let valid = MixtureSpec::new(MixtureKind::FadingBayes, vec![expert]).with_decay(0.5); + validate_mixture_spec_shallow(&valid).expect("strictly interior decay should validate"); + } +} diff --git a/crates/infotheory/src/axioms.rs b/crates/infotheory/src/axioms.rs new file mode 100644 index 00000000..81802ab4 --- /dev/null +++ b/crates/infotheory/src/axioms.rs @@ -0,0 +1,262 @@ +//! # Axioms: Mathematical Property Verifiers +//! +//! This module provides generic functions to verify mathematical properties +//! that should hold for any correct implementation of information-theoretic +//! measures. + +// ============================================================================ +// Metric Axioms +// ============================================================================ + +/// Verify that a distance function d(x,x) is close to 0 (allow for small overhead). +pub fn verify_identity(metric: F, x: &[u8], tolerance: f64) -> bool +where + F: Fn(&[u8], &[u8]) -> f64, +{ + let d = metric(x, x); + d.abs() <= tolerance +} + +/// Verify symmetry: d(x,y) ≈ d(y,x). +pub fn verify_symmetry(metric: F, x: &[u8], y: &[u8], tolerance: f64) -> bool +where + F: Fn(&[u8], &[u8]) -> f64, +{ + let d_xy = metric(x, y); + let d_yx = metric(y, x); + (d_xy - d_yx).abs() <= tolerance +} + +/// Verify triangle inequality: d(x,z) ≤ d(x,y) + d(y,z). +pub fn verify_triangle_inequality( + metric: F, + x: &[u8], + y: &[u8], + z: &[u8], + tolerance: f64, +) -> bool +where + F: Fn(&[u8], &[u8]) -> f64, +{ + let d_xy = metric(x, y); + let d_yz = metric(y, z); + let d_xz = metric(x, z); + d_xz <= (d_xy + d_yz + tolerance) +} + +/// Verify non-negativity: d(x,y) ≥ 0. +pub fn verify_non_negativity(metric: F, x: &[u8], y: &[u8]) -> bool +where + F: Fn(&[u8], &[u8]) -> f64, +{ + // Allow tiny floating point errors slightly below zero + metric(x, y) >= -1e-12 +} + +// ============================================================================ +// Information Inequalities +// ============================================================================ + +/// Verify mutual information non-negativity: I(X;Y) ≥ 0. +pub fn verify_mi_nonnegative(mi: F, x: &[u8], y: &[u8]) -> bool +where + F: Fn(&[u8], &[u8]) -> f64, +{ + mi(x, y) >= -1e-12 +} + +/// Verify subadditivity: H(X,Y) ≤ H(X) + H(Y). +/// +/// This is equivalent to I(X;Y) ≥ 0. +pub fn verify_subadditivity( + joint_entropy: FJoint, + empirical_entropy: FEmpirical, + x: &[u8], + y: &[u8], + tolerance: f64, +) -> bool +where + FJoint: Fn(&[u8], &[u8]) -> f64, + FEmpirical: Fn(&[u8]) -> f64, +{ + let h_xy = joint_entropy(x, y); + let h_x = empirical_entropy(x); + let h_y = empirical_entropy(y); + h_xy <= (h_x + h_y + tolerance) +} + +/// Verify conditioning reduces entropy: H(X|Y) ≤ H(X). +pub fn verify_conditioning_reduces_entropy( + conditional_entropy: FCond, + empirical_entropy: FEmpirical, + x: &[u8], + y: &[u8], + tolerance: f64, +) -> bool +where + FCond: Fn(&[u8], &[u8]) -> f64, + FEmpirical: Fn(&[u8]) -> f64, +{ + let h_x_given_y = conditional_entropy(x, y); + let h_x = empirical_entropy(x); + h_x_given_y <= (h_x + tolerance) +} + +/// Verify chain rule: H(X,Y) = H(X) + H(Y|X). +pub fn verify_chain_rule( + joint: FJoint, + empirical: FEmpirical, + conditional: FCond, + x: &[u8], + y: &[u8], + tolerance: f64, +) -> bool +where + FJoint: Fn(&[u8], &[u8]) -> f64, + FEmpirical: Fn(&[u8]) -> f64, + FCond: Fn(&[u8], &[u8]) -> f64, // H(Y|X) +{ + let h_xy = joint(x, y); + let h_x = empirical(x); + let h_y_given_x = conditional(y, x); + + (h_xy - (h_x + h_y_given_x)).abs() <= tolerance +} + +// ============================================================================ +// Bounds +// ============================================================================ + +/// Verify NCD range: 0 ≤ NCD ≤ 1+epsilon. +/// +/// NCD theoretically can slightly exceed 1 due to compression overhead, so we allow +/// a small margin or just check it's not egregiously large. Usually NCD <= 1.1 is safe. +pub fn verify_ncd_bounds(ncd: F, x: &[u8], y: &[u8]) -> bool +where + F: Fn(&[u8], &[u8]) -> f64, +{ + let val = ncd(x, y); + (-1e-12..=1.1).contains(&val) +} + +/// Verify entropy is bounded by log2(alphabet_size). +/// For bytes, max entropy is 8.0 bits/byte. +pub fn verify_entropy_bounds(entropy: F, data: &[u8]) -> bool +where + F: Fn(&[u8]) -> f64, +{ + let h = entropy(data); + (-1e-12..=8.0 + 1e-12).contains(&h) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn hamming_distance(x: &[u8], y: &[u8]) -> f64 { + x.iter().zip(y.iter()).filter(|(a, b)| a != b).count() as f64 + } + + #[test] + fn metric_axiom_verifiers_distinguish_valid_from_invalid_cases() { + let x = b"abc"; + let y = b"abd"; + let z = b"acd"; + + assert!(verify_identity(hamming_distance, x, 0.0)); + assert!(!verify_identity(|_, _| 0.2, x, 0.1)); + + assert!(verify_symmetry(hamming_distance, x, y, 0.0)); + assert!(!verify_symmetry( + |lhs, rhs| { + if lhs == rhs { + 0.0 + } else if lhs == x && rhs == y { + 1.0 + } else { + 3.0 + } + }, + x, + y, + 0.0 + )); + + assert!(verify_triangle_inequality(hamming_distance, x, y, z, 0.0)); + assert!(!verify_triangle_inequality( + |lhs, rhs| { if lhs == x && rhs == z { 5.0 } else { 1.0 } }, + x, + y, + z, + 0.0 + )); + + assert!(verify_non_negativity(hamming_distance, x, y)); + assert!(verify_non_negativity(|_, _| -5e-13, x, y)); + assert!(!verify_non_negativity(|_, _| -1e-6, x, y)); + } + + #[test] + fn information_inequality_verifiers_cover_positive_and_negative_examples() { + let x = b"left"; + let y = b"right"; + + let empirical = |data: &[u8]| data.len() as f64; + let joint = |lhs: &[u8], rhs: &[u8]| (lhs.len() + rhs.len()) as f64 - 0.5; + let conditional = |lhs: &[u8], _rhs: &[u8]| lhs.len() as f64 - 0.25; + + assert!(verify_mi_nonnegative(|_, _| 0.0, x, y)); + assert!(!verify_mi_nonnegative(|_, _| -1e-6, x, y)); + + assert!(verify_subadditivity(joint, empirical, x, y, 0.0)); + assert!(!verify_subadditivity( + |lhs, rhs| (lhs.len() + rhs.len()) as f64 + 2.0, + empirical, + x, + y, + 0.0 + )); + + assert!(verify_conditioning_reduces_entropy( + conditional, + empirical, + x, + y, + 0.0 + )); + assert!(!verify_conditioning_reduces_entropy( + |lhs, _rhs| lhs.len() as f64 + 1.0, + empirical, + x, + y, + 0.0 + )); + + assert!(verify_chain_rule(joint, empirical, conditional, x, y, 0.5)); + assert!(!verify_chain_rule( + |lhs, rhs| (lhs.len() + rhs.len()) as f64 + 3.0, + empirical, + conditional, + x, + y, + 0.0 + )); + } + + #[test] + fn bound_verifiers_allow_expected_slack_only() { + let x = b"x"; + let y = b"y"; + + assert!(verify_ncd_bounds(|_, _| 0.0, x, y)); + assert!(verify_ncd_bounds(|_, _| 1.1, x, y)); + assert!(!verify_ncd_bounds(|_, _| 1.100_001, x, y)); + assert!(!verify_ncd_bounds(|_, _| -1e-6, x, y)); + + assert!(verify_entropy_bounds(|_| 0.0, x)); + assert!(verify_entropy_bounds(|_| 8.0, x)); + assert!(verify_entropy_bounds(|_| -5e-13, x)); + assert!(!verify_entropy_bounds(|_| 8.1, x)); + assert!(!verify_entropy_bounds(|_| -1e-6, x)); + } +} diff --git a/src/backends/calibration.rs b/crates/infotheory/src/backends/calibration.rs similarity index 99% rename from src/backends/calibration.rs rename to crates/infotheory/src/backends/calibration.rs index a1029fe5..3c7d8c78 100644 --- a/src/backends/calibration.rs +++ b/crates/infotheory/src/backends/calibration.rs @@ -1,4 +1,4 @@ -use crate::CalibrationContextKind; +use crate::api::CalibrationContextKind; use crate::backends::text_context::{NeuralContextState, TextContextAnalyzer}; #[derive(Clone, Debug)] diff --git a/src/backends/ctw.rs b/crates/infotheory/src/backends/ctw.rs similarity index 63% rename from src/backends/ctw.rs rename to crates/infotheory/src/backends/ctw.rs index 85208125..9b109e22 100644 --- a/src/backends/ctw.rs +++ b/crates/infotheory/src/backends/ctw.rs @@ -6,20 +6,312 @@ //! semantics while avoiding the `O(depth)` explicit-node blow-up for singleton //! paths. -use std::cell::RefCell; +use std::cell::{Cell, RefCell}; use std::f64; use std::mem::size_of; +use std::sync::OnceLock; type Symbol = bool; +const HISTORY_WORD_BITS: usize = u64::BITS as usize; +#[inline(always)] +fn history_word_len(bits: usize) -> usize { + bits.div_ceil(HISTORY_WORD_BITS) +} + +trait HistoryAccess { + fn len(&self) -> usize; + fn bit(&self, index: usize) -> Symbol; + + #[inline(always)] + fn recent_bit(&self, _depth: usize) -> Option { + None + } + + #[inline(always)] + fn recent_path_bits(&self, _depth: usize, _len: usize) -> Option { + None + } +} + +impl HistoryAccess for [Symbol] { + #[inline(always)] + fn len(&self) -> usize { + <[Symbol]>::len(self) + } + + #[inline(always)] + fn bit(&self, index: usize) -> Symbol { + debug_assert!(index < self.len()); + unsafe { *self.get_unchecked(index) } + } +} + +impl HistoryAccess for Vec { + #[inline(always)] + fn len(&self) -> usize { + self.as_slice().len() + } + + #[inline(always)] + fn bit(&self, index: usize) -> Symbol { + self.as_slice().bit(index) + } +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +struct BitHistory { + words: Vec, + len: usize, + recent: u64, +} + +impl BitHistory { + #[inline(always)] + fn len(&self) -> usize { + self.len + } + + #[inline(always)] + fn is_empty(&self) -> bool { + self.len == 0 + } + + #[inline(always)] + fn memory_usage(&self) -> usize { + self.words.capacity() * size_of::() + } + + #[inline] + fn rebuild_recent(&mut self) { + self.recent = 0; + let tail_len = self.len.min(HISTORY_WORD_BITS); + for depth in 0..tail_len { + let idx = self.len - depth - 1; + if self.bit(idx) { + self.recent |= 1u64 << depth; + } + } + } + + #[inline] + fn reserve_exact(&mut self, additional_bits: usize) { + let required_bits = self.len.saturating_add(additional_bits); + let required_words = history_word_len(required_bits); + if required_words > self.words.capacity() { + self.words + .reserve_exact(required_words.saturating_sub(self.words.len())); + } + } + + #[inline] + fn push(&mut self, bit: Symbol) { + let word_idx = self.len / HISTORY_WORD_BITS; + let bit_idx = self.len % HISTORY_WORD_BITS; + if word_idx == self.words.len() { + self.words.push(0); + } + let mask = 1u64 << bit_idx; + if bit { + self.words[word_idx] |= mask; + } else { + self.words[word_idx] &= !mask; + } + self.recent = (self.recent << 1) | (bit as u64); + self.len += 1; + } + + #[inline] + fn pop(&mut self) -> Option { + if self.len == 0 { + return None; + } + let next_len = self.len - 1; + let word_idx = next_len / HISTORY_WORD_BITS; + let bit_idx = next_len % HISTORY_WORD_BITS; + let mask = 1u64 << bit_idx; + let bit = (self.words[word_idx] & mask) != 0; + if bit { + self.words[word_idx] &= !mask; + } + self.len = next_len; + if bit_idx == 0 { + self.words.truncate(word_idx); + } + self.recent >>= 1; + if self.len >= HISTORY_WORD_BITS { + let exposed_idx = self.len - HISTORY_WORD_BITS; + if self.bit(exposed_idx) { + self.recent |= 1u64 << (HISTORY_WORD_BITS - 1); + } + } + Some(bit) + } + + #[inline] + fn extend_from_slice(&mut self, symbols: &[Symbol]) { + self.reserve_exact(symbols.len()); + for &symbol in symbols { + self.push(symbol); + } + } + + #[inline] + fn truncate(&mut self, new_len: usize) { + if new_len >= self.len { + return; + } + self.len = new_len; + self.words.truncate(history_word_len(new_len)); + let rem = new_len % HISTORY_WORD_BITS; + if rem != 0 { + let mask = (1u64 << rem) - 1; + if let Some(last) = self.words.last_mut() { + *last &= mask; + } + } + self.rebuild_recent(); + } + + #[inline] + fn clear(&mut self) { + self.words.clear(); + self.len = 0; + self.recent = 0; + } + + #[cfg(test)] + fn to_vec(&self) -> Vec { + (0..self.len).map(|idx| self.bit(idx)).collect() + } +} + +impl HistoryAccess for BitHistory { + #[inline(always)] + fn len(&self) -> usize { + self.len + } + + #[inline(always)] + fn bit(&self, index: usize) -> Symbol { + debug_assert!(index < self.len); + let word = unsafe { *self.words.get_unchecked(index / HISTORY_WORD_BITS) }; + ((word >> (index % HISTORY_WORD_BITS)) & 1) != 0 + } + + #[inline(always)] + fn recent_bit(&self, depth: usize) -> Option { + if depth < self.len.min(HISTORY_WORD_BITS) { + Some(((self.recent >> depth) & 1) != 0) + } else { + None + } + } + + #[inline(always)] + fn recent_path_bits(&self, depth: usize, len: usize) -> Option { + let available = self.len.saturating_sub(depth).min(len); + if available == 0 { + return Some(0); + } + if depth + available <= self.len.min(HISTORY_WORD_BITS) { + let mask = if available >= HISTORY_WORD_BITS { + u64::MAX + } else { + (1u64 << available) - 1 + }; + Some((self.recent >> depth) & mask) + } else { + None + } + } +} + +const CTW_LOG_CACHE_LIMIT: usize = 1 << 24; +const CTW_HOT_PREFIX_DEPTH_DEFAULT: usize = 12; +const CTW_LOG_OVERFLOW_CACHE_SLOTS: usize = 1 << 14; + +#[cfg(not(test))] +#[inline(always)] +fn ctw_log_cache_limit() -> usize { + CTW_LOG_CACHE_LIMIT +} + +#[cfg(test)] +#[inline(always)] +fn ctw_log_cache_limit() -> usize { + CTW_TEST_LOG_CACHE_LIMIT.with(|limit| limit.borrow().unwrap_or(CTW_LOG_CACHE_LIMIT)) +} + +fn ctw_hot_prefix_depth_limit() -> usize { + static HOT_PREFIX_DEPTH: OnceLock = OnceLock::new(); + *HOT_PREFIX_DEPTH.get_or_init(|| { + std::env::var("INFOTHEORY_CTW_HOT_PREFIX_DEPTH") + .ok() + .and_then(|raw| raw.parse::().ok()) + .unwrap_or(CTW_HOT_PREFIX_DEPTH_DEFAULT) + }) +} + +#[cfg(not(test))] +#[inline(always)] +fn ctw_log_overflow_cache_slots() -> usize { + CTW_LOG_OVERFLOW_CACHE_SLOTS +} + +#[cfg(test)] +#[inline(always)] +fn ctw_log_overflow_cache_slots() -> usize { + CTW_TEST_LOG_OVERFLOW_CACHE_SLOTS + .with(|slots| slots.borrow().unwrap_or(CTW_LOG_OVERFLOW_CACHE_SLOTS)) +} + +#[cfg(test)] #[inline(always)] fn ensure_log_caches(log_int: &mut Vec, log_half: &mut Vec, upto: usize) { if upto < log_int.len() { return; } + let required_len = upto + 1; + log_int.reserve(required_len - log_int.len()); + log_half.reserve(required_len - log_half.len()); + push_log_cache_entries(log_int, log_half, upto); +} + +#[inline(always)] +fn reserve_bounded_log_cache(cache: &mut Vec, required_len: usize, max_len: usize) { + if cache.capacity() >= required_len { + return; + } + let target_capacity = cache + .capacity() + .saturating_mul(2) + .max(required_len) + .min(max_len); + cache.reserve_exact(target_capacity - cache.len()); +} + +#[inline(always)] +fn ensure_bounded_log_caches( + log_int: &mut Vec, + log_half: &mut Vec, + upto: usize, + limit: usize, +) { + let target = upto.min(limit); + if target < log_int.len() { + return; + } + let required_len = target + 1; + let max_len = limit + 1; + reserve_bounded_log_cache(log_int, required_len, max_len); + reserve_bounded_log_cache(log_half, required_len, max_len); + push_log_cache_entries(log_int, log_half, target); +} + +#[inline(always)] +fn push_log_cache_entries(log_int: &mut Vec, log_half: &mut Vec, upto: usize) { let start = log_int.len(); - log_int.reserve(upto + 1 - start); - log_half.reserve(upto + 1 - start); for n in start..=upto { if n == 0 { log_int.push(f64::NEG_INFINITY); @@ -30,10 +322,164 @@ fn ensure_log_caches(log_int: &mut Vec, log_half: &mut Vec, upto: usiz } } +struct LogCacheSlot { + key: Cell, + value: Cell, +} + +impl LogCacheSlot { + #[inline(always)] + fn empty() -> Self { + Self { + key: Cell::new(usize::MAX), + value: Cell::new(0.0), + } + } +} + +#[inline(always)] +fn ensure_overflow_log_cache(cache: &mut Vec, slots: usize) { + if slots == 0 { + cache.clear(); + cache.shrink_to(0); + return; + } + if cache.len() == slots { + return; + } + cache.clear(); + if cache.capacity() < slots { + cache.reserve_exact(slots); + } + cache.resize_with(slots, LogCacheSlot::empty); + if cache.capacity() > slots { + cache.shrink_to(slots); + } +} + +trait CtLogAccess: Copy { + fn log_int(self, n: usize) -> f64; + fn log_half(self, n: usize) -> f64; +} + +#[derive(Clone, Copy)] +struct CachedLogs<'a> { + log_int: &'a [f64], + log_half: &'a [f64], +} + +impl<'a> CachedLogs<'a> { + #[cfg(test)] + #[inline(always)] + fn new(log_int: &'a [f64], log_half: &'a [f64]) -> Self { + Self { log_int, log_half } + } +} + +impl CtLogAccess for CachedLogs<'_> { + #[inline(always)] + fn log_int(self, n: usize) -> f64 { + debug_assert!(n < self.log_int.len()); + // SAFETY: `CachedLogs` is used only when the shared prefix cache already + // contains every value through the current visit bound. + unsafe { *self.log_int.get_unchecked(n) } + } + + #[inline(always)] + fn log_half(self, n: usize) -> f64 { + debug_assert!(n < self.log_half.len()); + // SAFETY: `CachedLogs` is used only when the shared prefix cache already + // contains every value through the current visit bound. + unsafe { *self.log_half.get_unchecked(n) } + } +} + +#[derive(Clone, Copy)] +struct BoundedLogs<'a> { + log_int: &'a [f64], + log_half: &'a [f64], + overflow_log_int: &'a [LogCacheSlot], + overflow_log_half: &'a [LogCacheSlot], +} + +impl<'a> BoundedLogs<'a> { + #[cfg(test)] + #[inline(always)] + fn new(log_int: &'a [f64], log_half: &'a [f64]) -> Self { + Self { + log_int, + log_half, + overflow_log_int: &[], + overflow_log_half: &[], + } + } + + #[inline(always)] + fn with_overflow( + log_int: &'a [f64], + log_half: &'a [f64], + overflow_log_int: &'a [LogCacheSlot], + overflow_log_half: &'a [LogCacheSlot], + ) -> Self { + Self { + log_int, + log_half, + overflow_log_int, + overflow_log_half, + } + } + + #[inline(always)] + fn lookup_overflow(slots: &'a [LogCacheSlot], n: usize, compute: impl FnOnce() -> f64) -> f64 { + if slots.is_empty() { + return compute(); + } + let len = slots.len(); + let slot_idx = if len.is_power_of_two() { + n & (len - 1) + } else { + n % len + }; + let slot = &slots[slot_idx]; + if slot.key.get() == n { + slot.value.get() + } else { + let value = compute(); + slot.key.set(n); + slot.value.set(value); + value + } + } +} + +impl CtLogAccess for BoundedLogs<'_> { + #[inline(always)] + fn log_int(self, n: usize) -> f64 { + if n < self.log_int.len() { + self.log_int[n] + } else if n == 0 { + f64::NEG_INFINITY + } else { + Self::lookup_overflow(self.overflow_log_int, n, || (n as f64).ln()) + } + } + + #[inline(always)] + fn log_half(self, n: usize) -> f64 { + if n < self.log_half.len() { + self.log_half[n] + } else { + Self::lookup_overflow(self.overflow_log_half, n, || (n as f64 + 0.5).ln()) + } + } +} + #[derive(Default)] struct SharedLogCache { log_int: Vec, log_half: Vec, + overflow_log_int: Vec, + overflow_log_half: Vec, } impl SharedLogCache { @@ -41,17 +487,28 @@ impl SharedLogCache { Self { log_int: vec![f64::NEG_INFINITY], log_half: vec![(0.5f64).ln()], + overflow_log_int: Vec::new(), + overflow_log_half: Vec::new(), } } #[inline(always)] fn ensure(&mut self, upto: usize) { - ensure_log_caches(&mut self.log_int, &mut self.log_half, upto); + let limit = ctw_log_cache_limit(); + ensure_bounded_log_caches(&mut self.log_int, &mut self.log_half, upto, limit); + if upto > limit { + let slots = ctw_log_overflow_cache_slots(); + ensure_overflow_log_cache(&mut self.overflow_log_int, slots); + ensure_overflow_log_cache(&mut self.overflow_log_half, slots); + } } #[inline(always)] fn memory_usage(&self) -> usize { - self.log_int.capacity() * size_of::() + self.log_half.capacity() * size_of::() + self.log_int.capacity() * size_of::() + + self.log_half.capacity() * size_of::() + + self.overflow_log_int.capacity() * size_of::() + + self.overflow_log_half.capacity() * size_of::() } } @@ -60,12 +517,40 @@ thread_local! { RefCell::new(SharedLogCache::new()); } +#[cfg(test)] +thread_local! { + static CTW_TEST_LOG_CACHE_LIMIT: RefCell> = const { RefCell::new(None) }; +} + +#[cfg(test)] +thread_local! { + static CTW_TEST_LOG_OVERFLOW_CACHE_SLOTS: RefCell> = const { RefCell::new(None) }; +} + +#[inline] +fn with_shared_cached_logs(upto: usize, f: impl FnOnce(CachedLogs<'_>) -> R) -> R { + debug_assert!(upto <= ctw_log_cache_limit()); + CTW_SHARED_LOG_CACHE.with(|cache_cell| { + let mut cache = cache_cell.borrow_mut(); + cache.ensure(upto); + f(CachedLogs { + log_int: &cache.log_int, + log_half: &cache.log_half, + }) + }) +} + #[inline] -fn with_shared_log_cache(upto: usize, f: impl FnOnce(&[f64], &[f64]) -> R) -> R { +fn with_shared_bounded_logs(upto: usize, f: impl FnOnce(BoundedLogs<'_>) -> R) -> R { CTW_SHARED_LOG_CACHE.with(|cache_cell| { let mut cache = cache_cell.borrow_mut(); cache.ensure(upto); - f(&cache.log_int, &cache.log_half) + f(BoundedLogs::with_overflow( + &cache.log_int, + &cache.log_half, + &cache.overflow_log_int, + &cache.overflow_log_half, + )) }) } @@ -83,20 +568,43 @@ fn shared_log_cache_lens() -> (usize, usize) { }) } +#[cfg(test)] +#[inline] +fn shared_log_overflow_cache_lens() -> (usize, usize) { + CTW_SHARED_LOG_CACHE.with(|cache_cell| { + let cache = cache_cell.borrow(); + (cache.overflow_log_int.len(), cache.overflow_log_half.len()) + }) +} + +#[cfg(test)] +pub(crate) fn reset_shared_log_cache_for_test() { + CTW_SHARED_LOG_CACHE.with(|cache_cell| { + *cache_cell.borrow_mut() = SharedLogCache::new(); + }); +} + #[inline(always)] -fn history_symbol(history: &[Symbol], depth: usize) -> Symbol { +fn history_symbol(history: &H, depth: usize) -> Symbol { + if let Some(bit) = history.recent_bit(depth) { + return bit; + } let idx = history.len().wrapping_sub(depth + 1); if depth < history.len() { - unsafe { *history.get_unchecked(idx) } + history.bit(idx) } else { false } } #[inline(always)] -unsafe fn history_at_or_zero(history_ptr: *const Symbol, history_len: isize, idx: isize) -> Symbol { +fn history_at_or_zero( + history: &H, + history_len: isize, + idx: isize, +) -> Symbol { if idx >= 0 && idx < history_len { - *history_ptr.add(idx as usize) + history.bit(idx as usize) } else { false } @@ -342,7 +850,7 @@ impl SegmentPayload { } #[inline(always)] - fn from_path(history: &[Symbol], depth: usize, len: u32) -> Option { + fn from_path(history: &H, depth: usize, len: u32) -> Option { if len > SEG_EXACT_MAX_LEN { return None; } @@ -395,8 +903,6 @@ enum PreparedEnd { #[derive(Clone, Copy, Debug, PartialEq)] struct PreparedStep { source: ExistingSource, - counts: [u32; 2], - kt_log_prob: f64, span: u32, sibling_weight: f64, has_sibling: u8, @@ -454,7 +960,10 @@ fn low_bits_mask_u64(len: u32) -> u64 { } #[inline(always)] -fn path_bits_from_history(history: &[Symbol], depth: usize, len: usize) -> u64 { +fn path_bits_from_history(history: &H, depth: usize, len: usize) -> u64 { + if let Some(bits) = history.recent_path_bits(depth, len) { + return bits; + } let history_len = history.len(); let available = history_len.saturating_sub(depth).min(len); if available == 0 { @@ -464,7 +973,7 @@ fn path_bits_from_history(history: &[Symbol], depth: usize, len: usize) -> u64 { let mut bits = 0u64; let mut hist_idx = history_len - depth - 1; for offset in 0..available { - bits |= (unsafe { *history.get_unchecked(hist_idx) } as u64) << offset; + bits |= (history.bit(hist_idx) as u64) << offset; if hist_idx == 0 { break; } @@ -522,6 +1031,9 @@ fn predict_ratio_kt_one(counts: [u32; 2]) -> f64 { #[inline(always)] fn update_weighted_log_prob_non_leaf(kt_log_prob: f64, log_prob_w0: f64, log_prob_w1: f64) -> f64 { let child_log_prob = log_prob_w0 + log_prob_w1; + if child_log_prob.to_bits() == kt_log_prob.to_bits() { + return clamp_log_prob(kt_log_prob); + } let delta = child_log_prob - kt_log_prob; let log_prob_weighted = if delta >= 0.0 { child_log_prob + (-delta).exp().ln_1p() - std::f64::consts::LN_2 @@ -586,6 +1098,9 @@ fn combined_weight_ratio_internal( ) -> (f64, f64) { let kt_ratio = predict_ratio_kt(counts, sym_idx); let child_log_prob = path_child_log_prob + sibling_log_prob; + if child_log_prob.to_bits() == kt_log_prob.to_bits() { + return (clamp_log_prob(kt_log_prob), 0.5 * (kt_ratio + child_ratio)); + } let delta = child_log_prob - kt_log_prob; if delta >= 0.0 { let x = (-delta).exp(); @@ -612,6 +1127,9 @@ fn combined_weight_ratio_internal_one( ) -> (f64, f64) { let kt_ratio = predict_ratio_kt_one(counts); let child_log_prob = path_child_log_prob + sibling_log_prob; + if child_log_prob.to_bits() == kt_log_prob.to_bits() { + return (clamp_log_prob(kt_log_prob), 0.5 * (kt_ratio + child_ratio)); + } let delta = child_log_prob - kt_log_prob; if delta >= 0.0 { let x = (-delta).exp(); @@ -652,6 +1170,10 @@ fn unary_chain_log_weight_precomputed( } #[inline(always)] +// The CTW ratio transform is a hot scalar kernel; grouping these independent +// numeric inputs into a temporary struct would add ceremony without clarifying +// ownership, invariants, or call-site meaning. +#[allow(clippy::too_many_arguments)] fn unary_chain_ratio_transform_precomputed( kt_log_prob: f64, counts: [u32; 2], @@ -668,6 +1190,12 @@ fn unary_chain_ratio_transform_precomputed( { return (clamp_log_prob(kt_log_prob), kt_ratio); } + if kt_log_prob.to_bits() == continuation_log_prob.to_bits() { + return ( + clamp_log_prob(kt_log_prob), + (1.0 - alpha) * kt_ratio + alpha * continuation_ratio, + ); + } let delta = continuation_log_prob - kt_log_prob; if delta >= 0.0 { @@ -701,6 +1229,12 @@ fn unary_chain_ratio_transform_precomputed_one( { return (clamp_log_prob(kt_log_prob), kt_ratio); } + if kt_log_prob.to_bits() == continuation_log_prob.to_bits() { + return ( + clamp_log_prob(kt_log_prob), + (1.0 - alpha) * kt_ratio + alpha * continuation_ratio, + ); + } let delta = continuation_log_prob - kt_log_prob; if delta >= 0.0 { @@ -728,7 +1262,11 @@ fn predict_ratio_internal( sym_idx: usize, ) -> f64 { let kt_ratio = predict_ratio_kt(counts, sym_idx); - let delta = path_child_log_prob + sibling_log_prob - kt_log_prob; + let child_log_prob = path_child_log_prob + sibling_log_prob; + if child_log_prob.to_bits() == kt_log_prob.to_bits() { + return 0.5 * (kt_ratio + child_ratio); + } + let delta = child_log_prob - kt_log_prob; if delta >= 0.0 { let inv_rho = (-delta).exp(); (kt_ratio * inv_rho + child_ratio) / (1.0 + inv_rho) @@ -747,7 +1285,11 @@ fn predict_ratio_internal_one( child_ratio: f64, ) -> f64 { let kt_ratio = predict_ratio_kt_one(counts); - let delta = path_child_log_prob + sibling_log_prob - kt_log_prob; + let child_log_prob = path_child_log_prob + sibling_log_prob; + if child_log_prob.to_bits() == kt_log_prob.to_bits() { + return 0.5 * (kt_ratio + child_ratio); + } + let delta = child_log_prob - kt_log_prob; if delta >= 0.0 { let inv_rho = (-delta).exp(); (kt_ratio * inv_rho + child_ratio) / (1.0 + inv_rho) @@ -758,19 +1300,23 @@ fn predict_ratio_internal_one( } #[inline(always)] -fn path_edge_at_depth(history: &[Symbol], history_len: usize, depth: usize) -> bool { +fn path_edge_at_depth( + history: &H, + history_len: usize, + depth: usize, +) -> bool { if depth < history_len { - history[history_len - depth - 1] + history.bit(history_len - depth - 1) } else { false } } #[inline(always)] -fn segment_edge_from_parts( +fn segment_edge_from_parts( segment: CtSegment, offset: usize, - history: &[Symbol], + history: &H, history_len: usize, ) -> bool { match segment.payload.mode() { @@ -779,7 +1325,7 @@ fn segment_edge_from_parts( if segment.payload.anchor_or_const() as usize >= offset { let hist_idx = segment.payload.anchor_or_const() as usize - offset; if hist_idx < history_len { - let raw = history[hist_idx]; + let raw = history.bit(hist_idx); if segment.payload.mode() == SEG_MODE_HISTORY_INVERT { !raw } else { @@ -798,10 +1344,10 @@ fn segment_edge_from_parts( } #[inline(always)] -fn first_segment_mismatch( +fn first_segment_mismatch( segment: CtSegment, depth: usize, - history: &[Symbol], + history: &H, comparable_len: usize, ) -> Option<(usize, bool, bool)> { if comparable_len == 0 { @@ -815,16 +1361,13 @@ fn first_segment_mismatch( comparable_len, ), SEG_MODE_HISTORY | SEG_MODE_HISTORY_INVERT => { - let history_ptr = history.as_ptr(); let history_len = history.len() as isize; let mut path_hist_idx = history_len - depth as isize - 1; let mut seg_hist_idx = segment.payload.anchor_or_const() as isize; let invert = segment.payload.mode() == SEG_MODE_HISTORY_INVERT; for offset in 0..comparable_len { - let path_edge = - unsafe { history_at_or_zero(history_ptr, history_len, path_hist_idx) }; - let existing_raw = - unsafe { history_at_or_zero(history_ptr, history_len, seg_hist_idx) }; + let path_edge = history_at_or_zero(history, history_len, path_hist_idx); + let existing_raw = history_at_or_zero(history, history_len, seg_hist_idx); let existing_edge = if invert { !existing_raw } else { existing_raw }; if existing_edge != path_edge { return Some((offset, path_edge, existing_edge)); @@ -835,13 +1378,11 @@ fn first_segment_mismatch( None } SEG_MODE_CONST => { - let history_ptr = history.as_ptr(); let history_len = history.len() as isize; let mut path_hist_idx = history_len - depth as isize - 1; let existing_edge = segment.payload.const_bit(); for offset in 0..comparable_len { - let path_edge = - unsafe { history_at_or_zero(history_ptr, history_len, path_hist_idx) }; + let path_edge = history_at_or_zero(history, history_len, path_hist_idx); if existing_edge != path_edge { return Some((offset, path_edge, existing_edge)); } @@ -854,9 +1395,8 @@ fn first_segment_mismatch( } #[inline] -fn apply_update_to_state_raw( - log_int: &[f64], - log_half: &[f64], +fn apply_update_to_state_raw( + logs: L, symbol_count: &mut [u32; 2], log_prob_kt: &mut f64, sym_idx: usize, @@ -864,10 +1404,8 @@ fn apply_update_to_state_raw( let total_before = (symbol_count[0] + symbol_count[1]) as usize; let sym_before = symbol_count[sym_idx] as usize; debug_assert!(sym_before <= total_before); - debug_assert!(sym_before < log_half.len()); - debug_assert!(total_before + 1 < log_int.len()); - let log_half_before = unsafe { *log_half.get_unchecked(sym_before) }; - let log_total_after = unsafe { *log_int.get_unchecked(total_before + 1) }; + let log_half_before = logs.log_half(sym_before); + let log_total_after = logs.log_int(total_before + 1); *log_prob_kt += log_half_before - log_total_after; if *log_prob_kt > 1.0e-10 { *log_prob_kt = 0.0; @@ -878,9 +1416,8 @@ fn apply_update_to_state_raw( } #[inline] -fn apply_revert_to_state_raw( - log_int: &[f64], - log_half: &[f64], +fn apply_revert_to_state_raw( + logs: L, symbol_count: &mut [u32; 2], log_prob_kt: &mut f64, sym_idx: usize, @@ -888,10 +1425,8 @@ fn apply_revert_to_state_raw( let total = (symbol_count[0] + symbol_count[1]) as usize; let sym_count = symbol_count[sym_idx] as usize; if sym_count > 0 && total > 0 { - debug_assert!(sym_count - 1 < log_half.len()); - debug_assert!(total < log_int.len()); - let log_half_before = unsafe { *log_half.get_unchecked(sym_count - 1) }; - let log_total = unsafe { *log_int.get_unchecked(total) }; + let log_half_before = logs.log_half(sym_count - 1); + let log_total = logs.log_int(total); *log_prob_kt -= log_half_before - log_total; symbol_count[sym_idx] -= 1; } @@ -1112,7 +1647,12 @@ impl CtArena { } #[inline(always)] - fn segment_edge(&self, segment_idx: SegmentIndex, offset: u32, history: &[Symbol]) -> usize { + fn segment_edge( + &self, + segment_idx: SegmentIndex, + offset: u32, + history: &(impl HistoryAccess + ?Sized), + ) -> usize { let segment = self.segments[segment_idx.get()]; segment_edge_from_parts(segment, offset as usize, history, history.len()) as usize } @@ -1228,9 +1768,13 @@ impl CtArena { } } + // These fields are the exact segment state plus insertion context. Keeping + // them as scalar arguments avoids building a transient descriptor on this + // path-compression hot path. + #[allow(clippy::too_many_arguments)] fn prepend_or_alloc_segment( &mut self, - history: &[Symbol], + history: &(impl HistoryAccess + ?Sized), depth: usize, symbol_count: [u32; 2], log_prob_kt: f64, @@ -1340,7 +1884,6 @@ struct CtEngine { impl CtEngine { const RESERVE_MIN_NODES: usize = 4 * 1024; const RESERVE_MAX_NODES: usize = 1 << 18; - const HOT_PREFIX_DEPTH: usize = 10; fn new(depth: usize) -> Self { let mut arena = CtArena::with_capacity(1024.min(1 << depth.min(16))); @@ -1380,12 +1923,74 @@ impl CtEngine { #[inline(always)] fn hot_prefix_depth(&self) -> usize { - self.max_depth.min(Self::HOT_PREFIX_DEPTH) + self.max_depth.min(ctw_hot_prefix_depth_limit()) } - fn clear(&mut self) { - self.arena.clear(); - self.root = self.arena.alloc_node(); + #[inline(always)] + fn push_prepared_segment_step( + &mut self, + segment_idx: SegmentIndex, + offset: usize, + sibling_weight: f64, + has_sibling: u8, + ) { + let span = (offset + 1) as u32; + self.prepared_steps.push(PreparedStep { + source: ExistingSource::Segment(segment_idx, offset as u32), + span, + sibling_weight, + has_sibling, + }); + self.prepared_levels += span as usize; + } + + #[inline(always)] + fn walk_prepared_exact_segment( + &mut self, + segment_idx: SegmentIndex, + segment: CtSegment, + depth: usize, + path_bits: u64, + ) -> Option<(usize, ExistingSource)> { + let seg_len = segment.len() as usize; + let terminal_offset = self.max_depth.saturating_sub(depth); + let comparable_len = seg_len.min(terminal_offset); + if let Some((offset, _, _)) = + first_exact_segment_mismatch(segment.payload.exact_bits(), path_bits, comparable_len) + { + self.push_prepared_segment_step( + segment_idx, + offset, + self.arena + .segment_continuation_weight(segment_idx, offset as u32), + 1, + ); + self.prepared_end = PreparedEnd::MismatchAtCurrentSegment; + return None; + } + + if terminal_offset < seg_len { + self.push_prepared_segment_step(segment_idx, terminal_offset, 0.0, 0); + return None; + } + + if segment.tail.is_none() { + self.push_prepared_segment_step(segment_idx, seg_len - 1, 0.0, 0); + self.prepared_end = PreparedEnd::MissingAfterCurrent; + return None; + } + + self.push_prepared_segment_step(segment_idx, seg_len - 1, 0.0, 0); + let tail = segment.tail; + Some(( + depth + seg_len, + Self::child_to_existing_source(tail).unwrap_or(ExistingSource::None), + )) + } + + fn clear(&mut self) { + self.arena.clear(); + self.root = self.arena.alloc_node(); self.levels.fill(LevelState::default()); self.detaches.clear(); self.prepared_steps.clear(); @@ -1419,8 +2024,21 @@ impl CtEngine { } #[inline] - fn with_logs(&mut self, upto: usize, f: impl FnOnce(&mut Self, &[f64], &[f64]) -> R) -> R { - with_shared_log_cache(upto, |log_int, log_half| f(self, log_int, log_half)) + fn with_cached_logs( + &mut self, + upto: usize, + f: impl FnOnce(&mut Self, CachedLogs<'_>) -> R, + ) -> R { + with_shared_cached_logs(upto, |logs| f(self, logs)) + } + + #[inline] + fn with_bounded_logs( + &mut self, + upto: usize, + f: impl FnOnce(&mut Self, BoundedLogs<'_>) -> R, + ) -> R { + with_shared_bounded_logs(upto, |logs| f(self, logs)) } #[inline] @@ -1441,10 +2059,27 @@ impl CtEngine { ) } + #[inline(always)] + fn source_counts_and_kt_log_prob(&self, source: ExistingSource) -> ([u32; 2], f64) { + match source { + ExistingSource::Node(node_idx) => { + let slot = node_idx.get(); + let node = unsafe { *self.arena.nodes.get_unchecked(slot) }; + (node.symbol_count, node.log_prob_kt) + } + ExistingSource::Segment(segment_idx, _) => { + let slot = segment_idx.get(); + let segment = unsafe { *self.arena.segments.get_unchecked(slot) }; + (segment.symbol_count, segment.log_prob_kt) + } + ExistingSource::None => unreachable!("prepared step should never store None"), + } + } + fn build_missing_segment_path( &mut self, depth: usize, - history: &[Symbol], + history: &(impl HistoryAccess + ?Sized), sym_idx: usize, singleton_log_prob_kt: f64, ) -> ChildRef { @@ -1500,7 +2135,7 @@ impl CtEngine { fn build_missing_path( &mut self, depth: usize, - history: &[Symbol], + history: &(impl HistoryAccess + ?Sized), sym_idx: usize, singleton_log_prob_kt: f64, ) -> ChildRef { @@ -1623,18 +2258,17 @@ impl CtEngine { fn child_to_existing_source(child: ChildRef) -> Option { if let Some(node) = child.as_node() { Some(ExistingSource::Node(node)) - } else if let Some(segment) = child.as_segment() { - Some(ExistingSource::Segment(segment, 0)) } else { - None + child + .as_segment() + .map(|segment| ExistingSource::Segment(segment, 0)) } } #[inline(always)] - fn update_source_state( + fn update_source_state( &mut self, - log_int: &[f64], - log_half: &[f64], + logs: L, source: ExistingSource, sym_idx: usize, ) { @@ -1643,13 +2277,7 @@ impl CtEngine { let slot = node_idx.get(); let mut counts = self.arena.nodes[slot].symbol_count; let mut log_prob_kt = self.arena.nodes[slot].log_prob_kt; - apply_update_to_state_raw( - log_int, - log_half, - &mut counts, - &mut log_prob_kt, - sym_idx, - ); + apply_update_to_state_raw(logs, &mut counts, &mut log_prob_kt, sym_idx); self.arena.nodes[slot].symbol_count = counts; self.arena.nodes[slot].log_prob_kt = log_prob_kt; } @@ -1657,13 +2285,7 @@ impl CtEngine { let slot = segment_idx.get(); let mut counts = self.arena.segments[slot].symbol_count; let mut log_prob_kt = self.arena.segments[slot].log_prob_kt; - apply_update_to_state_raw( - log_int, - log_half, - &mut counts, - &mut log_prob_kt, - sym_idx, - ); + apply_update_to_state_raw(logs, &mut counts, &mut log_prob_kt, sym_idx); self.arena.segments[slot].symbol_count = counts; self.arena.segments[slot].log_prob_kt = log_prob_kt; } @@ -1700,7 +2322,7 @@ impl CtEngine { fn attach_missing_after_prepared_path( &mut self, - history: &[Symbol], + history: &(impl HistoryAccess + ?Sized), sym_idx: usize, singleton_log_prob_kt: f64, ) { @@ -1731,7 +2353,7 @@ impl CtEngine { fn replace_prepared_child( &mut self, - history: &[Symbol], + history: &(impl HistoryAccess + ?Sized), step_index: usize, current_start_depth: usize, new_child: ChildRef, @@ -1755,17 +2377,16 @@ impl CtEngine { } } - fn update_prepared_mismatch( + fn update_prepared_mismatch( &mut self, - log_int: &[f64], - log_half: &[f64], - history: &[Symbol], + logs: L, + history: &(impl HistoryAccess + ?Sized), sym_idx: usize, singleton_log_prob_kt: f64, ) -> ChildRef { let last_index = self.prepared_steps.len() - 1; for idx in 0..last_index { - self.update_source_state(log_int, log_half, self.prepared_steps[idx].source, sym_idx); + self.update_source_state(logs, self.prepared_steps[idx].source, sym_idx); } let last_step = self.prepared_steps[last_index]; @@ -1808,13 +2429,7 @@ impl CtEngine { self.build_missing_path(node_depth + 1, history, sym_idx, singleton_log_prob_kt); let mut updated_counts = original.symbol_count; let mut updated_log_prob_kt = original.log_prob_kt; - apply_update_to_state_raw( - log_int, - log_half, - &mut updated_counts, - &mut updated_log_prob_kt, - sym_idx, - ); + apply_update_to_state_raw(logs, &mut updated_counts, &mut updated_log_prob_kt, sym_idx); let branch = self .arena @@ -1851,11 +2466,10 @@ impl CtEngine { self.arena.child(self.root, root_edge) } - fn update_prepared_cached_path( + fn update_prepared_cached_path( &mut self, - log_int: &[f64], - log_half: &[f64], - history: &[Symbol], + logs: L, + history: &(impl HistoryAccess + ?Sized), sym_idx: usize, singleton_log_prob_kt: f64, ) { @@ -1894,15 +2508,9 @@ impl CtEngine { let step = self.prepared_steps[idx]; match step.source { ExistingSource::Node(node_idx) => { - let mut counts = step.counts; - let mut log_prob_kt = step.kt_log_prob; - apply_update_to_state_raw( - log_int, - log_half, - &mut counts, - &mut log_prob_kt, - sym_idx, - ); + let (mut counts, mut log_prob_kt) = + self.source_counts_and_kt_log_prob(step.source); + apply_update_to_state_raw(logs, &mut counts, &mut log_prob_kt, sym_idx); let weighted = if idx == last_index && self.prepared_end == PreparedEnd::MaxDepth { debug_assert_eq!(step.has_sibling, 0); @@ -1922,15 +2530,9 @@ impl CtEngine { child_weight = weighted; } ExistingSource::Segment(segment_idx, offset) => { - let mut counts = step.counts; - let mut log_prob_kt = step.kt_log_prob; - apply_update_to_state_raw( - log_int, - log_half, - &mut counts, - &mut log_prob_kt, - sym_idx, - ); + let (mut counts, mut log_prob_kt) = + self.source_counts_and_kt_log_prob(step.source); + apply_update_to_state_raw(logs, &mut counts, &mut log_prob_kt, sym_idx); let slot = segment_idx.get(); let weighted = if idx == last_index && self.prepared_end == PreparedEnd::MaxDepth { @@ -1939,7 +2541,7 @@ impl CtEngine { clamp_log_prob(log_prob_kt) } else { let (alpha, log_alpha, log_one_minus_alpha) = - self.segment_constants(self.arena.segments[slot].len()); + self.segment_constants(step.span); unary_chain_log_weight_precomputed( log_prob_kt, child_weight, @@ -1958,13 +2560,12 @@ impl CtEngine { } } - fn update_child_fast( + fn update_child_fast( &mut self, - log_int: &[f64], - log_half: &[f64], + logs: L, child: ChildRef, depth: usize, - history: &[Symbol], + history: &(impl HistoryAccess + ?Sized), sym_idx: usize, singleton_log_prob_kt: f64, ) -> ChildRef { @@ -1980,8 +2581,7 @@ impl CtEngine { let path_edge = history_symbol(history, depth) as usize; let next = self.arena.child(node_idx, path_edge); let updated = self.update_child_fast( - log_int, - log_half, + logs, next, depth + 1, history, @@ -1994,7 +2594,7 @@ impl CtEngine { } let mut counts = self.arena.nodes[node_idx.get()].symbol_count; let mut log_prob_kt = self.arena.nodes[node_idx.get()].log_prob_kt; - apply_update_to_state_raw(log_int, log_half, &mut counts, &mut log_prob_kt, sym_idx); + apply_update_to_state_raw(logs, &mut counts, &mut log_prob_kt, sym_idx); self.arena.nodes[node_idx.get()].symbol_count = counts; self.arena.nodes[node_idx.get()].log_prob_kt = log_prob_kt; self.arena.recompute_node_weight(node_idx); @@ -2006,13 +2606,7 @@ impl CtEngine { let seg_len = original.len() as usize; let mut updated_counts = original.symbol_count; let mut updated_log_prob_kt = original.log_prob_kt; - apply_update_to_state_raw( - log_int, - log_half, - &mut updated_counts, - &mut updated_log_prob_kt, - sym_idx, - ); + apply_update_to_state_raw(logs, &mut updated_counts, &mut updated_log_prob_kt, sym_idx); let depth_budget = self.max_depth.saturating_sub(depth); let comparable_len = if original.tail.is_none() { @@ -2092,8 +2686,7 @@ impl CtEngine { let tail = original.tail; let updated_tail = self.update_child_fast( - log_int, - log_half, + logs, tail, depth + seg_len, history, @@ -2109,13 +2702,16 @@ impl CtEngine { ChildRef::from_segment(segment_idx) } - fn update_child_fast_exact( + // Exact-mode updates thread together the log table, path state, and KT + // singleton value; a wrapper would only hide the data dependencies in this + // inner CTW update kernel. + #[allow(clippy::too_many_arguments)] + fn update_child_fast_exact( &mut self, - log_int: &[f64], - log_half: &[f64], + logs: L, child: ChildRef, depth: usize, - history: &[Symbol], + history: &(impl HistoryAccess + ?Sized), path_bits: u64, sym_idx: usize, singleton_log_prob_kt: f64, @@ -2138,8 +2734,7 @@ impl CtEngine { let path_edge = (path_bits & 1) as usize; let next = self.arena.child(node_idx, path_edge); let updated = self.update_child_fast_exact( - log_int, - log_half, + logs, next, depth + 1, history, @@ -2151,12 +2746,23 @@ impl CtEngine { self.arena.set_child(node_idx, path_edge, updated); } } - let mut counts = self.arena.nodes[node_idx.get()].symbol_count; - let mut log_prob_kt = self.arena.nodes[node_idx.get()].log_prob_kt; - apply_update_to_state_raw(log_int, log_half, &mut counts, &mut log_prob_kt, sym_idx); - self.arena.nodes[node_idx.get()].symbol_count = counts; - self.arena.nodes[node_idx.get()].log_prob_kt = log_prob_kt; - self.arena.recompute_node_weight(node_idx); + let slot = node_idx.get(); + let mut counts = self.arena.nodes[slot].symbol_count; + let mut log_prob_kt = self.arena.nodes[slot].log_prob_kt; + apply_update_to_state_raw(logs, &mut counts, &mut log_prob_kt, sym_idx); + let [left, right] = self.arena.nodes[slot].children; + let weighted = if left.is_none() && right.is_none() { + clamp_log_prob(log_prob_kt) + } else { + // Safety: `left`/`right` come from this node's stored children, so any + // non-none child index is arena-owned and in-bounds for this arena. + let w0 = unsafe { self.arena.child_ref_weighted_unchecked(left) }; + let w1 = unsafe { self.arena.child_ref_weighted_unchecked(right) }; + update_weighted_log_prob_non_leaf(log_prob_kt, w0, w1) + }; + self.arena.nodes[slot].symbol_count = counts; + self.arena.nodes[slot].log_prob_kt = log_prob_kt; + self.arena.nodes[slot].log_prob_weighted = weighted; return ChildRef::from_node(node_idx); } @@ -2164,8 +2770,7 @@ impl CtEngine { let original = self.arena.segments[segment_idx.get()]; if !original.payload.is_exact() { return self.update_child_fast( - log_int, - log_half, + logs, child, depth, history, @@ -2177,13 +2782,7 @@ impl CtEngine { let seg_len = original.len() as usize; let mut updated_counts = original.symbol_count; let mut updated_log_prob_kt = original.log_prob_kt; - apply_update_to_state_raw( - log_int, - log_half, - &mut updated_counts, - &mut updated_log_prob_kt, - sym_idx, - ); + apply_update_to_state_raw(logs, &mut updated_counts, &mut updated_log_prob_kt, sym_idx); let depth_budget = self.max_depth.saturating_sub(depth); let comparable_len = if original.tail.is_none() { @@ -2273,8 +2872,7 @@ impl CtEngine { let tail = original.tail; let updated_tail = self.update_child_fast_exact( - log_int, - log_half, + logs, tail, depth + seg_len, history, @@ -2292,20 +2890,18 @@ impl CtEngine { } #[inline(always)] - fn update_root_child( + fn update_root_child( &mut self, - log_int: &[f64], - log_half: &[f64], + logs: L, child: ChildRef, - history: &[Symbol], + history: &(impl HistoryAccess + ?Sized), sym_idx: usize, singleton_log_prob_kt: f64, ) -> ChildRef { if self.max_depth <= SEG_EXACT_MAX_LEN as usize { let path_bits = path_bits_from_history(history, 1, self.max_depth); self.update_child_fast_exact( - log_int, - log_half, + logs, child, 1, history, @@ -2314,19 +2910,11 @@ impl CtEngine { singleton_log_prob_kt, ) } else { - self.update_child_fast( - log_int, - log_half, - child, - 1, - history, - sym_idx, - singleton_log_prob_kt, - ) + self.update_child_fast(logs, child, 1, history, sym_idx, singleton_log_prob_kt) } } - fn collect_existing_levels(&mut self, history: &[Symbol]) -> ChildRef { + fn collect_existing_levels(&mut self, history: &(impl HistoryAccess + ?Sized)) -> ChildRef { if self.max_depth == 0 { self.detaches.clear(); return ChildRef::NONE; @@ -2417,7 +3005,7 @@ impl CtEngine { old_child } - fn rebuild_path_subtree(&mut self, history: &[Symbol]) -> ChildRef { + fn rebuild_path_subtree(&mut self, history: &(impl HistoryAccess + ?Sized)) -> ChildRef { let mut built = ChildRef::NONE; for depth in (1..=self.max_depth).rev() { @@ -2486,20 +3074,19 @@ impl CtEngine { } } - fn update_with_logs( + fn update_with_logs( &mut self, - log_int: &[f64], - log_half: &[f64], + logs: L, sym: Symbol, - history: &[Symbol], + history: &(impl HistoryAccess + ?Sized), ) { let sym_idx = sym as usize; - let singleton_log_prob_kt = log_half[0] - log_int[1]; + let singleton_log_prob_kt = logs.log_half(0) - logs.log_int(1); { let slot = self.root.get(); let mut counts = self.arena.nodes[slot].symbol_count; let mut log_prob_kt = self.arena.nodes[slot].log_prob_kt; - apply_update_to_state_raw(log_int, log_half, &mut counts, &mut log_prob_kt, sym_idx); + apply_update_to_state_raw(logs, &mut counts, &mut log_prob_kt, sym_idx); self.arena.nodes[slot].symbol_count = counts; self.arena.nodes[slot].log_prob_kt = log_prob_kt; } @@ -2507,140 +3094,146 @@ impl CtEngine { if self.max_depth > 0 { let root_edge = history_symbol(history, 0) as usize; let old_child = self.arena.child(self.root, root_edge); - let new_child = self.update_root_child( - log_int, - log_half, - old_child, - history, - sym_idx, - singleton_log_prob_kt, - ); + let new_child = + self.update_root_child(logs, old_child, history, sym_idx, singleton_log_prob_kt); self.arena.set_child(self.root, root_edge, new_child); } self.arena.recompute_node_weight(self.root); } - fn update(&mut self, sym: Symbol, history: &[Symbol]) { + fn update(&mut self, sym: Symbol, history: &(impl HistoryAccess + ?Sized)) { let upto = self.root_visits() + 1; - self.with_logs(upto, |this, log_int, log_half| { - this.update_with_logs(log_int, log_half, sym, history); - }); + if upto <= ctw_log_cache_limit() { + self.with_cached_logs(upto, |this, logs| { + this.update_with_logs(logs, sym, history); + }); + } else { + self.with_bounded_logs(upto, |this, logs| { + this.update_with_logs(logs, sym, history); + }); + } } - fn update_prepared(&mut self, sym: Symbol, history: &[Symbol], use_prepared: bool) { - let upto = self.root_visits() + 1; - let sym_idx = sym as usize; - self.with_logs(upto, |this, log_int, log_half| { - let singleton_log_prob_kt = log_half[0] - log_int[1]; - { - let slot = this.root.get(); - let mut counts = this.arena.nodes[slot].symbol_count; - let mut log_prob_kt = this.arena.nodes[slot].log_prob_kt; - apply_update_to_state_raw( - log_int, - log_half, - &mut counts, - &mut log_prob_kt, - sym_idx, - ); - this.arena.nodes[slot].symbol_count = counts; - this.arena.nodes[slot].log_prob_kt = log_prob_kt; - } + fn update_prepared_with_logs( + &mut self, + logs: L, + history: &(impl HistoryAccess + ?Sized), + sym_idx: usize, + use_prepared: bool, + ) { + let singleton_log_prob_kt = logs.log_half(0) - logs.log_int(1); + { + let slot = self.root.get(); + let mut counts = self.arena.nodes[slot].symbol_count; + let mut log_prob_kt = self.arena.nodes[slot].log_prob_kt; + apply_update_to_state_raw(logs, &mut counts, &mut log_prob_kt, sym_idx); + self.arena.nodes[slot].symbol_count = counts; + self.arena.nodes[slot].log_prob_kt = log_prob_kt; + } - if this.max_depth > 0 { - let root_edge = history_symbol(history, 0) as usize; - let old_child = this.arena.child(this.root, root_edge); - let new_child = if use_prepared { - match this.prepared_end { - PreparedEnd::MissingAtRoot => { - this.build_missing_path(1, history, sym_idx, singleton_log_prob_kt) - } - PreparedEnd::MaxDepth | PreparedEnd::MissingAfterCurrent => { - if !this.prepared_steps.is_empty() { - this.update_prepared_cached_path( - log_int, - log_half, - history, - sym_idx, - singleton_log_prob_kt, - ); - } - old_child + if self.max_depth > 0 { + let root_edge = history_symbol(history, 0) as usize; + let old_child = self.arena.child(self.root, root_edge); + let new_child = if use_prepared { + match self.prepared_end { + PreparedEnd::MissingAtRoot => { + self.build_missing_path(1, history, sym_idx, singleton_log_prob_kt) + } + PreparedEnd::MaxDepth | PreparedEnd::MissingAfterCurrent => { + if !self.prepared_steps.is_empty() { + self.update_prepared_cached_path( + logs, + history, + sym_idx, + singleton_log_prob_kt, + ); } - PreparedEnd::MismatchAtCurrentSegment => this.update_prepared_mismatch( - log_int, - log_half, - history, - sym_idx, - singleton_log_prob_kt, - ), + old_child } - } else { - this.update_root_child( - log_int, - log_half, - old_child, - history, - sym_idx, - singleton_log_prob_kt, - ) - }; - this.arena.set_child(this.root, root_edge, new_child); - } + PreparedEnd::MismatchAtCurrentSegment => { + self.update_prepared_mismatch(logs, history, sym_idx, singleton_log_prob_kt) + } + } + } else { + self.update_root_child(logs, old_child, history, sym_idx, singleton_log_prob_kt) + }; + self.arena.set_child(self.root, root_edge, new_child); + } - this.arena.recompute_node_weight(this.root); - }); + self.arena.recompute_node_weight(self.root); } - fn revert(&mut self, sym: Symbol, history: &[Symbol]) { - let upto = self.root_visits(); + fn update_prepared( + &mut self, + sym: Symbol, + history: &(impl HistoryAccess + ?Sized), + use_prepared: bool, + ) { + let upto = self.root_visits() + 1; let sym_idx = sym as usize; - self.with_logs(upto, |this, log_int, log_half| { - let old_child = this.collect_existing_levels(history); - - { - let slot = this.root.get(); - let mut counts = this.arena.nodes[slot].symbol_count; - let mut log_prob_kt = this.arena.nodes[slot].log_prob_kt; - apply_revert_to_state_raw( - log_int, - log_half, - &mut counts, - &mut log_prob_kt, - sym_idx, - ); - this.arena.nodes[slot].symbol_count = counts; - this.arena.nodes[slot].log_prob_kt = log_prob_kt; - } + if upto <= ctw_log_cache_limit() { + self.with_cached_logs(upto, |this, logs| { + this.update_prepared_with_logs(logs, history, sym_idx, use_prepared); + }); + } else { + self.with_bounded_logs(upto, |this, logs| { + this.update_prepared_with_logs(logs, history, sym_idx, use_prepared); + }); + } + } - for level in &mut this.levels { - let mut counts = level.symbol_count; - let mut log_prob_kt = level.log_prob_kt; - apply_revert_to_state_raw( - log_int, - log_half, - &mut counts, - &mut log_prob_kt, - sym_idx, - ); - level.symbol_count = counts; - level.log_prob_kt = log_prob_kt; - } + fn revert_with_logs( + &mut self, + logs: L, + history: &(impl HistoryAccess + ?Sized), + sym_idx: usize, + ) { + let old_child = self.collect_existing_levels(history); - if this.max_depth > 0 { - let new_child = this.rebuild_path_subtree(history); - let root_edge = history_symbol(history, 0) as usize; - this.apply_detaches(); - this.arena.free_child_ref(old_child); - this.arena.set_child(this.root, root_edge, new_child); - } + { + let slot = self.root.get(); + let mut counts = self.arena.nodes[slot].symbol_count; + let mut log_prob_kt = self.arena.nodes[slot].log_prob_kt; + apply_revert_to_state_raw(logs, &mut counts, &mut log_prob_kt, sym_idx); + self.arena.nodes[slot].symbol_count = counts; + self.arena.nodes[slot].log_prob_kt = log_prob_kt; + } - this.arena.recompute_node_weight(this.root); - }); + for level in &mut self.levels { + let mut counts = level.symbol_count; + let mut log_prob_kt = level.log_prob_kt; + apply_revert_to_state_raw(logs, &mut counts, &mut log_prob_kt, sym_idx); + level.symbol_count = counts; + level.log_prob_kt = log_prob_kt; + } + + if self.max_depth > 0 { + let new_child = self.rebuild_path_subtree(history); + let root_edge = history_symbol(history, 0) as usize; + self.apply_detaches(); + self.arena.free_child_ref(old_child); + self.arena.set_child(self.root, root_edge, new_child); + } + + self.arena.recompute_node_weight(self.root); + } + + fn revert(&mut self, sym: Symbol, history: &(impl HistoryAccess + ?Sized)) { + let upto = self.root_visits(); + let sym_idx = sym as usize; + if upto <= ctw_log_cache_limit() { + self.with_cached_logs(upto, |this, logs| { + this.revert_with_logs(logs, history, sym_idx); + }); + } else { + self.with_bounded_logs(upto, |this, logs| { + this.revert_with_logs(logs, history, sym_idx); + }); + } } - fn predict(&mut self, sym: Symbol, history: &[Symbol]) -> f64 { + fn predict(&mut self, sym: Symbol, history: &(impl HistoryAccess + ?Sized)) -> f64 { self.prepared_steps.clear(); self.prepared_levels = 0; self.prepared_end = PreparedEnd::MaxDepth; @@ -2667,14 +3260,9 @@ impl CtEngine { match source { ExistingSource::None => break, ExistingSource::Node(node_idx) => { - let slot = node_idx.get(); - let counts = self.arena.nodes[slot].symbol_count; - let kt_log_prob = self.arena.nodes[slot].log_prob_kt; if depth == self.max_depth { self.prepared_steps.push(PreparedStep { source: ExistingSource::Node(node_idx), - counts, - kt_log_prob, span: 1, sibling_weight: 0.0, has_sibling: 0, @@ -2686,8 +3274,6 @@ impl CtEngine { let sibling = self.arena.child(node_idx, path_edge ^ 1); self.prepared_steps.push(PreparedStep { source: ExistingSource::Node(node_idx), - counts, - kt_log_prob, span: 1, sibling_weight: self.arena.child_ref_weighted(sibling), has_sibling: sibling.is_some() as u8, @@ -2703,9 +3289,26 @@ impl CtEngine { } ExistingSource::Segment(segment_idx, _) => { let segment = self.arena.segments[segment_idx.get()]; + if segment.payload.is_exact() { + let path_bits = path_bits_from_history( + history, + depth, + self.max_depth.saturating_sub(depth), + ); + if let Some((next_depth, next_source)) = + self.walk_prepared_exact_segment(segment_idx, segment, depth, path_bits) + { + source = next_source; + if matches!(source, ExistingSource::None) { + self.prepared_end = PreparedEnd::MissingAfterCurrent; + break 'walk; + } + depth = next_depth; + continue 'walk; + } + break 'walk; + } let seg_len = segment.len() as usize; - let counts = segment.symbol_count; - let kt_log_prob = segment.log_prob_kt; for offset in 0..seg_len { let node_depth = depth + offset; let span = (offset + 1) as u32; @@ -2713,8 +3316,6 @@ impl CtEngine { if node_depth == self.max_depth { self.prepared_steps.push(PreparedStep { source: ExistingSource::Segment(segment_idx, offset as u32), - counts, - kt_log_prob, span, sibling_weight: 0.0, has_sibling: 0, @@ -2726,8 +3327,6 @@ impl CtEngine { if offset + 1 >= seg_len && segment.tail.is_none() { self.prepared_steps.push(PreparedStep { source: ExistingSource::Segment(segment_idx, offset as u32), - counts, - kt_log_prob, span, sibling_weight: 0.0, has_sibling: 0, @@ -2743,8 +3342,6 @@ impl CtEngine { if path_edge != existing_edge { self.prepared_steps.push(PreparedStep { source: ExistingSource::Segment(segment_idx, offset as u32), - counts, - kt_log_prob, span, sibling_weight: self .arena @@ -2762,8 +3359,6 @@ impl CtEngine { self.prepared_steps.push(PreparedStep { source: ExistingSource::Segment(segment_idx, offset as u32), - counts, - kt_log_prob, span, sibling_weight: 0.0, has_sibling: 0, @@ -2795,14 +3390,12 @@ impl CtEngine { } let last_step = *self.prepared_steps.last().unwrap(); - let last_counts = last_step.counts; - let last_kt_log_prob = last_step.kt_log_prob; - let (mut child_weight, mut ratio) = if self.prepared_end == PreparedEnd::MaxDepth - && self.prepared_levels == self.max_depth + let (last_counts, last_kt_log_prob) = self.source_counts_and_kt_log_prob(last_step.source); + let (mut child_weight, mut ratio) = if (self.prepared_end == PreparedEnd::MaxDepth + && self.prepared_levels == self.max_depth) + || last_step.has_sibling == 0 { (last_kt_log_prob, predict_ratio_kt(last_counts, sym_idx)) - } else if last_step.has_sibling == 0 { - (last_kt_log_prob, predict_ratio_kt(last_counts, sym_idx)) } else { combined_weight_ratio_internal( last_kt_log_prob, @@ -2814,30 +3407,29 @@ impl CtEngine { ) }; - if let ExistingSource::Segment(_, _) = last_step.source { - if last_step.span > 1 { - let (alpha, log_alpha, log_one_minus_alpha) = - self.segment_constants(last_step.span - 1); - (child_weight, ratio) = unary_chain_ratio_transform_precomputed( - last_step.kt_log_prob, - last_step.counts, - child_weight, - ratio, - alpha, - log_alpha, - log_one_minus_alpha, - sym_idx, - ); - } + if matches!(last_step.source, ExistingSource::Segment(_, _)) && last_step.span > 1 { + let (alpha, log_alpha, log_one_minus_alpha) = + self.segment_constants(last_step.span - 1); + (child_weight, ratio) = unary_chain_ratio_transform_precomputed( + last_kt_log_prob, + last_counts, + child_weight, + ratio, + alpha, + log_alpha, + log_one_minus_alpha, + sym_idx, + ); } for idx in (0..self.prepared_steps.len() - 1).rev() { let step = self.prepared_steps[idx]; + let (step_counts, step_kt_log_prob) = self.source_counts_and_kt_log_prob(step.source); match step.source { ExistingSource::Node(_) => { (child_weight, ratio) = combined_weight_ratio_internal( - step.kt_log_prob, - step.counts, + step_kt_log_prob, + step_counts, child_weight, step.sibling_weight, ratio, @@ -2847,8 +3439,8 @@ impl CtEngine { ExistingSource::Segment(_, _) => { let (alpha, log_alpha, log_one_minus_alpha) = self.segment_constants(step.span); (child_weight, ratio) = unary_chain_ratio_transform_precomputed( - step.kt_log_prob, - step.counts, + step_kt_log_prob, + step_counts, child_weight, ratio, alpha, @@ -2873,7 +3465,7 @@ impl CtEngine { ) } - fn predict_one(&mut self, history: &[Symbol]) -> f64 { + fn predict_one(&mut self, history: &(impl HistoryAccess + ?Sized)) -> f64 { self.prepared_steps.clear(); self.prepared_levels = 0; self.prepared_end = PreparedEnd::MaxDepth; @@ -2900,14 +3492,9 @@ impl CtEngine { match source { ExistingSource::None => break, ExistingSource::Node(node_idx) => { - let slot = node_idx.get(); - let counts = self.arena.nodes[slot].symbol_count; - let kt_log_prob = self.arena.nodes[slot].log_prob_kt; if depth == self.max_depth { self.prepared_steps.push(PreparedStep { source: ExistingSource::Node(node_idx), - counts, - kt_log_prob, span: 1, sibling_weight: 0.0, has_sibling: 0, @@ -2919,8 +3506,6 @@ impl CtEngine { let sibling = self.arena.child(node_idx, path_edge ^ 1); self.prepared_steps.push(PreparedStep { source: ExistingSource::Node(node_idx), - counts, - kt_log_prob, span: 1, sibling_weight: self.arena.child_ref_weighted(sibling), has_sibling: sibling.is_some() as u8, @@ -2936,9 +3521,26 @@ impl CtEngine { } ExistingSource::Segment(segment_idx, _) => { let segment = self.arena.segments[segment_idx.get()]; + if segment.payload.is_exact() { + let path_bits = path_bits_from_history( + history, + depth, + self.max_depth.saturating_sub(depth), + ); + if let Some((next_depth, next_source)) = + self.walk_prepared_exact_segment(segment_idx, segment, depth, path_bits) + { + source = next_source; + if matches!(source, ExistingSource::None) { + self.prepared_end = PreparedEnd::MissingAfterCurrent; + break 'walk; + } + depth = next_depth; + continue 'walk; + } + break 'walk; + } let seg_len = segment.len() as usize; - let counts = segment.symbol_count; - let kt_log_prob = segment.log_prob_kt; for offset in 0..seg_len { let node_depth = depth + offset; let span = (offset + 1) as u32; @@ -2946,8 +3548,6 @@ impl CtEngine { if node_depth == self.max_depth { self.prepared_steps.push(PreparedStep { source: ExistingSource::Segment(segment_idx, offset as u32), - counts, - kt_log_prob, span, sibling_weight: 0.0, has_sibling: 0, @@ -2959,8 +3559,6 @@ impl CtEngine { if offset + 1 >= seg_len && segment.tail.is_none() { self.prepared_steps.push(PreparedStep { source: ExistingSource::Segment(segment_idx, offset as u32), - counts, - kt_log_prob, span, sibling_weight: 0.0, has_sibling: 0, @@ -2976,8 +3574,6 @@ impl CtEngine { if path_edge != existing_edge { self.prepared_steps.push(PreparedStep { source: ExistingSource::Segment(segment_idx, offset as u32), - counts, - kt_log_prob, span, sibling_weight: self .arena @@ -2995,8 +3591,6 @@ impl CtEngine { self.prepared_steps.push(PreparedStep { source: ExistingSource::Segment(segment_idx, offset as u32), - counts, - kt_log_prob, span, sibling_weight: 0.0, has_sibling: 0, @@ -3027,14 +3621,12 @@ impl CtEngine { } let last_step = *self.prepared_steps.last().unwrap(); - let last_counts = last_step.counts; - let last_kt_log_prob = last_step.kt_log_prob; - let (mut child_weight, mut ratio) = if self.prepared_end == PreparedEnd::MaxDepth - && self.prepared_levels == self.max_depth + let (last_counts, last_kt_log_prob) = self.source_counts_and_kt_log_prob(last_step.source); + let (mut child_weight, mut ratio) = if (self.prepared_end == PreparedEnd::MaxDepth + && self.prepared_levels == self.max_depth) + || last_step.has_sibling == 0 { (last_kt_log_prob, predict_ratio_kt_one(last_counts)) - } else if last_step.has_sibling == 0 { - (last_kt_log_prob, predict_ratio_kt_one(last_counts)) } else { combined_weight_ratio_internal_one( last_kt_log_prob, @@ -3051,8 +3643,8 @@ impl CtEngine { let (alpha, log_alpha, log_one_minus_alpha) = self.segment_constants(last_step.span - 1); (child_weight, ratio) = unary_chain_ratio_transform_precomputed_one( - last_step.kt_log_prob, - last_step.counts, + last_kt_log_prob, + last_counts, child_weight, ratio, alpha, @@ -3063,11 +3655,12 @@ impl CtEngine { for idx in (0..self.prepared_steps.len() - 1).rev() { let step = self.prepared_steps[idx]; + let (step_counts, step_kt_log_prob) = self.source_counts_and_kt_log_prob(step.source); match step.source { ExistingSource::Node(_) => { (child_weight, ratio) = combined_weight_ratio_internal_one( - step.kt_log_prob, - step.counts, + step_kt_log_prob, + step_counts, child_weight, step.sibling_weight, ratio, @@ -3076,8 +3669,8 @@ impl CtEngine { ExistingSource::Segment(_, _) => { let (alpha, log_alpha, log_one_minus_alpha) = self.segment_constants(step.span); (child_weight, ratio) = unary_chain_ratio_transform_precomputed_one( - step.kt_log_prob, - step.counts, + step_kt_log_prob, + step_counts, child_weight, ratio, alpha, @@ -3109,13 +3702,99 @@ impl CtEngine { + self.detaches.capacity() * size_of::() + self.prepared_steps.capacity() * size_of::() } + + #[cfg(any(test, feature = "research-tooling"))] + fn scratch_memory_usage(&self) -> usize { + self.segment_alpha.capacity() * size_of::() + + self.segment_log_alpha.capacity() * size_of::() + + self.segment_log_one_minus_alpha.capacity() * size_of::() + + self.levels.capacity() * size_of::() + + self.detaches.capacity() * size_of::() + + self.prepared_steps.capacity() * size_of::() + } + + #[cfg(any(test, feature = "research-tooling"))] + fn telemetry(&self, bit_index: usize) -> FacContextTreeTreeTelemetry { + let mut exact_segments: usize = 0; + let mut history_segments: usize = 0; + let mut history_invert_segments: usize = 0; + let mut const_segments: usize = 0; + let mut segment_bits: u64 = 0; + let mut max_segment_len: u32 = 0; + for segment in &self.arena.segments { + let len = segment.len(); + segment_bits = segment_bits.saturating_add(len as u64); + max_segment_len = max_segment_len.max(len); + match segment.payload.mode() { + SEG_MODE_EXACT => exact_segments = exact_segments.saturating_add(1), + SEG_MODE_HISTORY => history_segments = history_segments.saturating_add(1), + SEG_MODE_HISTORY_INVERT => { + history_invert_segments = history_invert_segments.saturating_add(1); + } + SEG_MODE_CONST => const_segments = const_segments.saturating_add(1), + _ => unreachable!("invalid ctw segment payload mode"), + } + } + + let node_payload_bytes = self.arena.nodes.len() * size_of::(); + let node_bytes = self.arena.nodes.capacity() * size_of::(); + let segment_payload_bytes = self.arena.segments.len() * size_of::(); + let segment_bytes = self.arena.segments.capacity() * size_of::(); + let free_list_bytes = self.arena.free_nodes.capacity() * size_of::() + + self.arena.free_segments.capacity() * size_of::(); + let scratch_bytes = self.scratch_memory_usage(); + let arena_slack_bytes = node_bytes + .saturating_sub(node_payload_bytes) + .saturating_add(segment_bytes.saturating_sub(segment_payload_bytes)); + + FacContextTreeTreeTelemetry { + bit_index, + max_depth: self.max_depth, + root_visits: self.root_visits(), + nodes_len: self.arena.nodes.len(), + nodes_capacity: self.arena.nodes.capacity(), + segments_len: self.arena.segments.len(), + segments_capacity: self.arena.segments.capacity(), + free_nodes_len: self.arena.free_nodes.len(), + free_nodes_capacity: self.arena.free_nodes.capacity(), + free_segments_len: self.arena.free_segments.len(), + free_segments_capacity: self.arena.free_segments.capacity(), + node_bytes, + node_payload_bytes, + segment_bytes, + segment_payload_bytes, + free_list_bytes, + scratch_bytes, + total_bytes: node_bytes + .saturating_add(segment_bytes) + .saturating_add(free_list_bytes) + .saturating_add(scratch_bytes), + arena_slack_bytes, + exact_segments, + history_segments, + history_invert_segments, + const_segments, + segment_bits, + max_segment_len, + } + } } /// A Context Tree for binary sequence prediction. #[derive(Clone)] pub struct ContextTree { engine: CtEngine, - history: Vec, + history: BitHistory, + history_version: u64, + prepared_valid: bool, + prepared_history_len: usize, + prepared_history_version: u64, +} + +#[derive(Clone)] +pub(crate) struct ContextTreeLifecycleSnapshot { + history: BitHistory, + history_version: u64, } impl ContextTree { @@ -3123,21 +3802,62 @@ impl ContextTree { pub fn new(depth: usize) -> Self { Self { engine: CtEngine::new(depth), - history: Vec::new(), + history: BitHistory::default(), + history_version: 0, + prepared_valid: false, + prepared_history_len: 0, + prepared_history_version: 0, } } + #[inline] + fn bump_history_version(&mut self) { + self.history_version = self.history_version.wrapping_add(1); + } + + #[inline] + fn clear_prepared_prediction(&mut self) { + self.prepared_valid = false; + } + + #[inline] + fn prepared_prediction_matches_history(&self) -> bool { + self.prepared_valid + && self.prepared_history_len == self.history.len() + && self.prepared_history_version == self.history_version + } + /// Reset tree parameters and clear conditioning history. pub fn clear(&mut self) { self.history.clear(); self.engine.clear(); + self.history_version = 0; + self.clear_prepared_prediction(); + self.prepared_history_len = 0; + self.prepared_history_version = 0; + } + + #[inline] + pub(crate) fn reserve_for_symbols(&mut self, total_symbols: usize) { + if total_symbols == 0 { + return; + } + self.engine.reserve_for_symbols(total_symbols); + self.history.reserve_exact(total_symbols); } #[inline] /// Observe one binary symbol and update the model. pub fn update(&mut self, sym: Symbol) { - self.engine.update(sym, &self.history); + let use_prepared = self.prepared_prediction_matches_history(); + self.clear_prepared_prediction(); + if use_prepared { + self.engine.update_prepared(sym, &self.history, true); + } else { + self.engine.update(sym, &self.history); + } self.history.push(sym); + self.bump_history_version(); } #[inline] @@ -3146,38 +3866,81 @@ impl ContextTree { let Some(last_sym) = self.history.pop() else { return; }; + self.clear_prepared_prediction(); self.engine.revert(last_sym, &self.history); + self.bump_history_version(); } #[inline] /// Append external symbols to history without touching model state. pub fn update_history(&mut self, symbols: &[Symbol]) { + if symbols.is_empty() { + return; + } + self.clear_prepared_prediction(); self.history.extend_from_slice(symbols); + self.bump_history_version(); } #[inline] /// Remove one history symbol without reverting model statistics. pub fn revert_history(&mut self) { - self.history.pop(); + if self.history.pop().is_some() { + self.clear_prepared_prediction(); + self.bump_history_version(); + } } /// Truncate the stored history to `new_size` symbols. pub fn truncate_history(&mut self, new_size: usize) { if new_size < self.history.len() { + self.clear_prepared_prediction(); self.history.truncate(new_size); + self.bump_history_version(); + } + } + + /// Capture rollback state for stream lifecycle transactions. + /// + /// This clones the full conditioning history, so the allocation and copy are + /// O(history length). It is intended for stream lifecycle boundaries rather + /// than per-symbol speculative prediction. + pub(crate) fn lifecycle_snapshot(&self) -> ContextTreeLifecycleSnapshot { + ContextTreeLifecycleSnapshot { + history: self.history.clone(), + history_version: self.history_version, } } + pub(crate) fn restore_lifecycle_snapshot(&mut self, snapshot: ContextTreeLifecycleSnapshot) { + self.history = snapshot.history; + self.history_version = snapshot.history_version; + self.clear_prepared_prediction(); + } + #[inline] /// Predict `P(sym | history)` under current weighted CTW model. pub fn predict(&mut self, sym: Symbol) -> f64 { - self.engine.predict(sym, &self.history) + let prob = self.engine.predict(sym, &self.history); + self.prepared_valid = true; + self.prepared_history_len = self.history.len(); + self.prepared_history_version = self.history_version; + prob + } + + #[inline] + pub(crate) fn predict_one(&mut self) -> f64 { + let prob = self.engine.predict_one(&self.history); + self.prepared_valid = true; + self.prepared_history_len = self.history.len(); + self.prepared_history_version = self.history_version; + prob } #[inline] /// Predict probability of symbol `true`. pub fn predict_sym_prob(&mut self) -> f64 { - self.predict(true) + self.predict_one() } #[inline] @@ -3207,6 +3970,13 @@ struct ContextTreeCore { prepared_history_version: u64, } +#[derive(Clone, Copy)] +struct ContextTreeCorePreparedSnapshot { + prepared_valid: bool, + prepared_history_len: usize, + prepared_history_version: u64, +} + impl ContextTreeCore { fn new(depth: usize) -> Self { Self { @@ -3230,29 +4000,49 @@ impl ContextTreeCore { } #[inline] - fn update(&mut self, sym: Symbol, shared_history: &[Symbol]) { + fn update_predicted( + &mut self, + sym: Symbol, + shared_history: &(impl HistoryAccess + ?Sized), + history_version: u64, + ) { + let use_prepared = self.prepared_valid + && self.prepared_history_len == shared_history.len() + && self.prepared_history_version == history_version; self.prepared_valid = false; - self.engine.update(sym, shared_history); + self.engine + .update_prepared(sym, shared_history, use_prepared); } #[inline] - fn update_predicted(&mut self, sym: Symbol, shared_history: &[Symbol], history_version: u64) { + fn update_predicted_with_logs( + &mut self, + logs: L, + sym: Symbol, + shared_history: &(impl HistoryAccess + ?Sized), + history_version: u64, + ) { let use_prepared = self.prepared_valid && self.prepared_history_len == shared_history.len() && self.prepared_history_version == history_version; self.prepared_valid = false; self.engine - .update_prepared(sym, shared_history, use_prepared); + .update_prepared_with_logs(logs, shared_history, sym as usize, use_prepared); } #[inline] - fn revert(&mut self, last_sym: Symbol, shared_history: &[Symbol]) { + fn revert(&mut self, last_sym: Symbol, shared_history: &(impl HistoryAccess + ?Sized)) { self.prepared_valid = false; self.engine.revert(last_sym, shared_history); } #[inline] - fn predict(&mut self, sym: Symbol, shared_history: &[Symbol], history_version: u64) -> f64 { + fn predict( + &mut self, + sym: Symbol, + shared_history: &(impl HistoryAccess + ?Sized), + history_version: u64, + ) -> f64 { let prob = self.engine.predict(sym, shared_history); self.prepared_valid = true; self.prepared_history_len = shared_history.len(); @@ -3261,7 +4051,11 @@ impl ContextTreeCore { } #[inline] - fn predict_one(&mut self, shared_history: &[Symbol], history_version: u64) -> f64 { + fn predict_one( + &mut self, + shared_history: &(impl HistoryAccess + ?Sized), + history_version: u64, + ) -> f64 { let prob = self.engine.predict_one(shared_history); self.prepared_valid = true; self.prepared_history_len = shared_history.len(); @@ -3273,18 +4067,180 @@ impl ContextTreeCore { fn get_log_block_probability(&self) -> f64 { self.engine.get_log_block_probability() } + + #[inline] + fn prepared_snapshot(&self) -> ContextTreeCorePreparedSnapshot { + ContextTreeCorePreparedSnapshot { + prepared_valid: self.prepared_valid, + prepared_history_len: self.prepared_history_len, + prepared_history_version: self.prepared_history_version, + } + } + + #[inline] + fn restore_prepared_snapshot(&mut self, snapshot: ContextTreeCorePreparedSnapshot) { + self.prepared_valid = snapshot.prepared_valid; + self.prepared_history_len = snapshot.prepared_history_len; + self.prepared_history_version = snapshot.prepared_history_version; + } } /// Factorized Action-Conditional Context Tree Weighting. #[derive(Clone)] pub struct FacContextTree { trees: Vec, - shared_history: Vec, + shared_history: BitHistory, base_depth: usize, num_bits: usize, shared_history_version: u64, } +#[derive(Clone)] +pub(crate) struct FacContextTreeLifecycleSnapshot { + shared_history: BitHistory, + shared_history_version: u64, + prepared: Vec, +} + +/// Approximate heap-memory breakdown for a [`FacContextTree`]. +#[cfg(any(test, feature = "research-tooling"))] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct FacContextTreeMemoryUsage { + /// Bytes owned by per-bit CTW tree engines, including arenas and scratch buffers. + pub tree_bytes: usize, + /// Bytes held by the thread-local shared CTW logarithm cache. + pub shared_log_cache_bytes: usize, + /// Bytes reserved for the factorized shared history buffer. + pub shared_history_bytes: usize, +} + +/// Per-tree CTW arena and scratch telemetry. +#[cfg(any(test, feature = "research-tooling"))] +#[derive(Clone, Debug, PartialEq, Eq)] +#[non_exhaustive] +pub struct FacContextTreeTreeTelemetry { + /// Factorized bit position for this tree. + pub bit_index: usize, + /// Maximum context depth for this tree. + pub max_depth: usize, + /// Number of symbols observed by this tree root. + pub root_visits: usize, + /// Number of allocated explicit nodes. + pub nodes_len: usize, + /// Reserved explicit-node capacity. + pub nodes_capacity: usize, + /// Number of allocated unary path segments. + pub segments_len: usize, + /// Reserved unary-segment capacity. + pub segments_capacity: usize, + /// Number of node slots currently on the free list. + pub free_nodes_len: usize, + /// Reserved free-node list capacity. + pub free_nodes_capacity: usize, + /// Number of segment slots currently on the free list. + pub free_segments_len: usize, + /// Reserved free-segment list capacity. + pub free_segments_capacity: usize, + /// Reserved explicit-node bytes. + pub node_bytes: usize, + /// Explicit-node payload bytes at current length. + pub node_payload_bytes: usize, + /// Reserved segment bytes. + pub segment_bytes: usize, + /// Unary-segment payload bytes at current length. + pub segment_payload_bytes: usize, + /// Reserved free-list bytes. + pub free_list_bytes: usize, + /// Reserved engine scratch bytes. + pub scratch_bytes: usize, + /// Total reserved tree bytes for this tree. + pub total_bytes: usize, + /// Reserved tree bytes currently unused by arena capacity. + pub arena_slack_bytes: usize, + /// Number of exact-bit segment payloads. + pub exact_segments: usize, + /// Number of history-anchor segment payloads. + pub history_segments: usize, + /// Number of inverted history-anchor segment payloads. + pub history_invert_segments: usize, + /// Number of constant-bit segment payloads. + pub const_segments: usize, + /// Sum of segment lengths in bits. + pub segment_bits: u64, + /// Maximum segment length in bits. + pub max_segment_len: u32, +} + +/// Detailed FAC-CTW memory telemetry. +#[cfg(any(test, feature = "research-tooling"))] +#[derive(Clone, Debug, PartialEq, Eq)] +#[non_exhaustive] +pub struct FacContextTreeTelemetry { + /// Base depth used to construct tree 0. + pub base_depth: usize, + /// Number of factorized bit trees. + pub num_bits: usize, + /// Shared history length in bits. + pub shared_history_len_bits: usize, + /// Shared history reserved capacity in bits. + pub shared_history_capacity_bits: usize, + /// Reserved bytes for shared history. + pub shared_history_bytes: usize, + /// Shared-history payload bytes at current length. + pub shared_history_payload_bytes: usize, + /// Shared-history reserved slack bytes. + pub shared_history_slack_bytes: usize, + /// Reserved bytes for shared log caches. + pub shared_log_cache_bytes: usize, + /// Sum of per-tree reserved bytes. + pub tree_bytes: usize, + /// Sum of live per-tree node and segment payload bytes. + pub tree_payload_bytes: usize, + /// Sum of per-tree arena slack bytes. + pub tree_arena_slack_bytes: usize, + /// Total reserved bytes reported by FAC memory accounting. + pub total_bytes: usize, + /// Total reserved slack bytes attributable to allocator headroom. + pub total_slack_bytes: usize, + /// Sum of allocated explicit nodes across trees. + pub nodes_len: usize, + /// Sum of explicit-node capacity across trees. + pub nodes_capacity: usize, + /// Sum of allocated unary path segments across trees. + pub segments_len: usize, + /// Sum of unary-segment capacity across trees. + pub segments_capacity: usize, + /// Sum of node slots currently on free lists. + pub free_nodes_len: usize, + /// Sum of segment slots currently on free lists. + pub free_segments_len: usize, + /// Sum of exact-bit segment payloads across trees. + pub exact_segments: usize, + /// Sum of history-anchor segment payloads across trees. + pub history_segments: usize, + /// Sum of inverted history-anchor segment payloads across trees. + pub history_invert_segments: usize, + /// Sum of constant-bit segment payloads across trees. + pub const_segments: usize, + /// Sum of segment lengths in bits across trees. + pub segment_bits: u64, + /// Maximum segment length observed across trees. + pub max_segment_len: u32, + /// Per-tree telemetry. + pub trees: Vec, +} + +#[cfg(any(test, feature = "research-tooling"))] +impl FacContextTreeMemoryUsage { + /// Total approximate heap memory in bytes. + #[inline] + pub fn total_bytes(self) -> usize { + self.tree_bytes + .saturating_add(self.shared_log_cache_bytes) + .saturating_add(self.shared_history_bytes) + } +} + impl FacContextTree { /// Create a factorized CTW stack over `num_percept_bits` bit positions. /// @@ -3295,7 +4251,7 @@ impl FacContextTree { .collect(); Self { trees, - shared_history: Vec::new(), + shared_history: BitHistory::default(), base_depth, num_bits: num_percept_bits, shared_history_version: 0, @@ -3336,7 +4292,11 @@ impl FacContextTree { /// Update one bit position with a binary symbol. pub fn update(&mut self, sym: Symbol, bit_index: usize) { debug_assert!(bit_index < self.num_bits); - self.trees[bit_index].update(sym, &self.shared_history); + self.trees[bit_index].update_predicted( + sym, + &self.shared_history, + self.shared_history_version, + ); self.shared_history.push(sym); self.bump_shared_history_version(); } @@ -3358,19 +4318,77 @@ impl FacContextTree { .iter() .all(|tree| tree.engine.root_visits() + 1 == upto) ); - with_shared_log_cache(upto, |log_int, log_half| { - for bit_idx in 0..8usize { - let bit = ((byte >> (7 - bit_idx)) & 1) == 1; - let tree = &mut self.trees[bit_idx]; - tree.prepared_valid = false; - tree.engine - .update_with_logs(log_int, log_half, bit, &self.shared_history); - self.shared_history.push(bit); - } - }); + if upto <= ctw_log_cache_limit() { + with_shared_cached_logs(upto, |logs| { + for bit_idx in 0..8usize { + let bit = ((byte >> (7 - bit_idx)) & 1) == 1; + let tree = &mut self.trees[bit_idx]; + tree.prepared_valid = false; + tree.engine + .update_with_logs(logs, bit, &self.shared_history); + self.shared_history.push(bit); + } + }); + } else { + with_shared_bounded_logs(upto, |logs| { + for bit_idx in 0..8usize { + let bit = ((byte >> (7 - bit_idx)) & 1) == 1; + let tree = &mut self.trees[bit_idx]; + tree.prepared_valid = false; + tree.engine + .update_with_logs(logs, bit, &self.shared_history); + self.shared_history.push(bit); + } + }); + } self.bump_shared_history_version(); } + #[inline] + fn log_prob_update_byte_msb_with_logs(&mut self, logs: L, byte: u8) -> f64 { + let mut logp = 0.0; + for bit_idx in 0..8usize { + let bit = ((byte >> (7 - bit_idx)) & 1) == 1; + let p = + self.trees[bit_idx].predict(bit, &self.shared_history, self.shared_history_version); + if p.is_finite() && p > 0.0 { + logp += p.ln(); + } else { + logp = f64::NEG_INFINITY; + } + self.trees[bit_idx].update_predicted_with_logs( + logs, + bit, + &self.shared_history, + self.shared_history_version, + ); + self.shared_history.push(bit); + self.bump_shared_history_version(); + } + logp + } + + #[inline] + /// Return the MSB-first log probability of `byte` and then update the model. + pub fn log_prob_update_byte_msb(&mut self, byte: u8) -> f64 { + debug_assert_eq!(self.num_bits, 8); + let upto = self.trees[0].engine.root_visits() + 1; + debug_assert!( + self.trees + .iter() + .all(|tree| tree.engine.root_visits() + 1 == upto) + ); + if upto <= ctw_log_cache_limit() { + with_shared_cached_logs(upto, |logs| { + self.log_prob_update_byte_msb_with_logs(logs, byte) + }) + } else { + with_shared_bounded_logs(upto, |logs| { + self.log_prob_update_byte_msb_with_logs(logs, byte) + }) + } + } + #[inline] /// Update all active bit positions from one byte, least-significant bit first. pub fn update_byte_lsb(&mut self, byte: u8) { @@ -3382,16 +4400,29 @@ impl FacContextTree { .take(bits) .all(|tree| tree.engine.root_visits() + 1 == upto) ); - with_shared_log_cache(upto, |log_int, log_half| { - for bit_idx in 0..bits { - let bit = ((byte >> bit_idx) & 1) == 1; - let tree = &mut self.trees[bit_idx]; - tree.prepared_valid = false; - tree.engine - .update_with_logs(log_int, log_half, bit, &self.shared_history); - self.shared_history.push(bit); - } - }); + if upto <= ctw_log_cache_limit() { + with_shared_cached_logs(upto, |logs| { + for bit_idx in 0..bits { + let bit = ((byte >> bit_idx) & 1) == 1; + let tree = &mut self.trees[bit_idx]; + tree.prepared_valid = false; + tree.engine + .update_with_logs(logs, bit, &self.shared_history); + self.shared_history.push(bit); + } + }); + } else { + with_shared_bounded_logs(upto, |logs| { + for bit_idx in 0..bits { + let bit = ((byte >> bit_idx) & 1) == 1; + let tree = &mut self.trees[bit_idx]; + tree.prepared_valid = false; + tree.engine + .update_with_logs(logs, bit, &self.shared_history); + self.shared_history.push(bit); + } + }); + } self.bump_shared_history_version(); } @@ -3464,6 +4495,33 @@ impl FacContextTree { self.bump_shared_history_version(); } + /// Capture rollback state for stream lifecycle transactions. + /// + /// This clones the shared conditioning history and per-tree prepared-prefix + /// state, so the allocation and copy are O(shared history length + number of + /// FAC component trees). It is intended for stream lifecycle boundaries + /// rather than per-symbol speculative prediction. + pub(crate) fn lifecycle_snapshot(&self) -> FacContextTreeLifecycleSnapshot { + FacContextTreeLifecycleSnapshot { + shared_history: self.shared_history.clone(), + shared_history_version: self.shared_history_version, + prepared: self + .trees + .iter() + .map(ContextTreeCore::prepared_snapshot) + .collect(), + } + } + + pub(crate) fn restore_lifecycle_snapshot(&mut self, snapshot: FacContextTreeLifecycleSnapshot) { + debug_assert_eq!(self.trees.len(), snapshot.prepared.len()); + self.shared_history = snapshot.shared_history; + self.shared_history_version = snapshot.shared_history_version; + for (tree, prepared) in self.trees.iter_mut().zip(snapshot.prepared) { + tree.restore_prepared_snapshot(prepared); + } + } + #[inline] /// Sum of per-tree log block probabilities. pub fn get_log_block_probability(&self) -> f64 { @@ -3482,6 +4540,94 @@ impl FacContextTree { self.shared_history_version = 0; } + /// Approximate heap-memory usage broken down by CTW component. + #[cfg(any(test, feature = "research-tooling"))] + pub fn memory_usage_breakdown(&self) -> FacContextTreeMemoryUsage { + let tree_mem: usize = self.trees.iter().map(|t| t.engine.memory_usage()).sum(); + let log_cache_mem = self + .trees + .first() + .map(|t| t.engine.log_cache_memory_usage()) + .unwrap_or(0); + let history_mem = self.shared_history.memory_usage(); + FacContextTreeMemoryUsage { + tree_bytes: tree_mem, + shared_log_cache_bytes: log_cache_mem, + shared_history_bytes: history_mem, + } + } + + /// Detailed CTW arena, segment, scratch, history, and log-cache telemetry. + #[cfg(any(test, feature = "research-tooling"))] + pub fn telemetry(&self) -> FacContextTreeTelemetry { + let usage = self.memory_usage_breakdown(); + let trees: Vec = self + .trees + .iter() + .enumerate() + .map(|(bit_index, tree)| tree.engine.telemetry(bit_index)) + .collect(); + let shared_history_payload_bytes = + self.shared_history.len().div_ceil(HISTORY_WORD_BITS) * size_of::(); + let shared_history_slack_bytes = usage + .shared_history_bytes + .saturating_sub(shared_history_payload_bytes); + let nodes_len = trees.iter().map(|tree| tree.nodes_len).sum(); + let nodes_capacity = trees.iter().map(|tree| tree.nodes_capacity).sum(); + let segments_len = trees.iter().map(|tree| tree.segments_len).sum(); + let segments_capacity = trees.iter().map(|tree| tree.segments_capacity).sum(); + let free_nodes_len = trees.iter().map(|tree| tree.free_nodes_len).sum(); + let free_segments_len = trees.iter().map(|tree| tree.free_segments_len).sum(); + let exact_segments = trees.iter().map(|tree| tree.exact_segments).sum(); + let history_segments = trees.iter().map(|tree| tree.history_segments).sum(); + let history_invert_segments = trees.iter().map(|tree| tree.history_invert_segments).sum(); + let const_segments = trees.iter().map(|tree| tree.const_segments).sum(); + let segment_bits = trees.iter().map(|tree| tree.segment_bits).sum(); + let max_segment_len = trees + .iter() + .map(|tree| tree.max_segment_len) + .max() + .unwrap_or(0); + let tree_payload_bytes = trees + .iter() + .map(|tree| { + tree.node_payload_bytes + .saturating_add(tree.segment_payload_bytes) + }) + .sum(); + let tree_arena_slack_bytes: usize = trees.iter().map(|tree| tree.arena_slack_bytes).sum(); + let total_slack_bytes = tree_arena_slack_bytes.saturating_add(shared_history_slack_bytes); + + FacContextTreeTelemetry { + base_depth: self.base_depth, + num_bits: self.num_bits, + shared_history_len_bits: self.shared_history.len(), + shared_history_capacity_bits: self.shared_history.words.capacity() * HISTORY_WORD_BITS, + shared_history_bytes: usage.shared_history_bytes, + shared_history_payload_bytes, + shared_history_slack_bytes, + shared_log_cache_bytes: usage.shared_log_cache_bytes, + tree_bytes: usage.tree_bytes, + tree_payload_bytes, + tree_arena_slack_bytes, + total_bytes: usage.total_bytes(), + total_slack_bytes, + nodes_len, + nodes_capacity, + segments_len, + segments_capacity, + free_nodes_len, + free_segments_len, + exact_segments, + history_segments, + history_invert_segments, + const_segments, + segment_bits, + max_segment_len, + trees, + } + } + /// Approximate heap memory usage in bytes. pub fn memory_usage(&self) -> usize { let tree_mem: usize = self.trees.iter().map(|t| t.engine.memory_usage()).sum(); @@ -3490,8 +4636,234 @@ impl FacContextTree { .first() .map(|t| t.engine.log_cache_memory_usage()) .unwrap_or(0); - let history_mem = self.shared_history.capacity() * size_of::(); - tree_mem + log_cache_mem + history_mem + let history_mem = self.shared_history.memory_usage(); + tree_mem + .saturating_add(log_cache_mem) + .saturating_add(history_mem) + } +} + +#[inline] +fn compact_symbol_msb_shift(bits_per_symbol: usize, bit_idx: usize) -> usize { + bits_per_symbol.saturating_sub(1).saturating_sub(bit_idx) +} + +#[inline] +pub(crate) fn ctw_symbol_bit_msb(symbol: u8, bits_per_symbol: usize, bit_idx: usize) -> bool { + let bits = bits_per_symbol.clamp(1, 8); + let shift = compact_symbol_msb_shift(bits, bit_idx); + ((symbol >> shift) & 1) == 1 +} + +#[inline] +pub(crate) fn ctw_log_prob_msb( + tree: &mut ContextTree, + symbol: u8, + bits_per_symbol: usize, + min_prob: f64, +) -> f64 { + let bits = bits_per_symbol.clamp(1, 8); + let mut logp = 0.0; + for bit_idx in 0..bits { + let bit = ctw_symbol_bit_msb(symbol, bits, bit_idx); + let p = tree.predict(bit); + if p.is_finite() && p > 0.0 { + logp += p.ln(); + } else { + logp = f64::NEG_INFINITY; + } + tree.update(bit); + } + for _ in 0..bits { + tree.revert(); + } + if logp.is_finite() { + logp.max(min_prob.ln()) + } else { + min_prob.ln() + } +} + +#[inline] +pub(crate) fn ctw_log_prob_update_msb( + tree: &mut ContextTree, + symbol: u8, + bits_per_symbol: usize, + min_prob: f64, +) -> f64 { + let bits = bits_per_symbol.clamp(1, 8); + let mut logp = 0.0; + for bit_idx in 0..bits { + let bit = ctw_symbol_bit_msb(symbol, bits, bit_idx); + let p = tree.predict(bit); + if p.is_finite() && p > 0.0 { + logp += p.ln(); + } else { + logp = f64::NEG_INFINITY; + } + tree.update(bit); + } + if logp.is_finite() { + logp.max(min_prob.ln()) + } else { + min_prob.ln() + } +} + +#[inline] +pub(crate) fn ctw_log_prob_update_lsb( + tree: &mut FacContextTree, + symbol: u8, + bits_per_symbol: usize, + min_prob: f64, +) -> f64 { + let mut logp = 0.0; + for bit_idx in 0..bits_per_symbol { + let bit = ((symbol >> bit_idx) & 1) == 1; + let p = tree.predict(bit, bit_idx); + if p.is_finite() && p > 0.0 { + logp += p.ln(); + } else { + logp = f64::NEG_INFINITY; + } + tree.update_predicted(bit, bit_idx); + } + if logp.is_finite() { + logp.max(min_prob.ln()) + } else { + min_prob.ln() + } +} + +pub(crate) fn fill_ctw_tree_log_probs( + tree: &mut ContextTree, + bits_per_symbol: usize, + min_logp: f64, + out: &mut [f64; 256], +) { + let bits = bits_per_symbol.clamp(1, 8); + let patterns = 1usize << bits; + let mut pattern_logps = [f64::NEG_INFINITY; 256]; + let log_before = tree.get_log_block_probability(); + + fn rec( + tree: &mut ContextTree, + depth: usize, + bits: usize, + log_before: f64, + min_logp: f64, + symbol_acc: u8, + pattern_logps: &mut [f64; 256], + ) { + if depth == bits { + let pat = symbol_acc as usize; + let logp = (tree.get_log_block_probability() - log_before).max(min_logp); + pattern_logps[pat] = logp; + return; + } + + for bit in [false, true] { + tree.update(bit); + let shift = compact_symbol_msb_shift(bits, depth); + let next_symbol = if bit { + symbol_acc | (1u8 << shift) + } else { + symbol_acc + }; + rec( + tree, + depth + 1, + bits, + log_before, + min_logp, + next_symbol, + pattern_logps, + ); + tree.revert(); + } + } + + rec(tree, 0, bits, log_before, min_logp, 0, &mut pattern_logps); + + if bits == 8 { + out.copy_from_slice(&pattern_logps); + } else { + let aliases = 1usize << (8 - bits); + let alias_ln = (aliases as f64).ln(); + let mask = patterns - 1; + for byte in 0..256usize { + out[byte] = pattern_logps[byte & mask] - alias_ln; + } + } +} + +pub(crate) fn fill_fac_tree_log_probs( + tree: &mut FacContextTree, + bits_per_symbol: usize, + msb_first: bool, + min_logp: f64, + out: &mut [f64; 256], +) { + struct RecParams { + bits: usize, + msb_first: bool, + log_before: f64, + min_logp: f64, + } + + let bits = bits_per_symbol.clamp(1, 8); + let patterns = 1usize << bits; + let mut pattern_logps = [f64::NEG_INFINITY; 256]; + let params = RecParams { + bits, + msb_first, + log_before: tree.get_log_block_probability(), + min_logp, + }; + + fn rec( + tree: &mut FacContextTree, + depth: usize, + params: &RecParams, + symbol_acc: u8, + pattern_logps: &mut [f64; 256], + ) { + if depth == params.bits { + let pat = symbol_acc as usize; + let logp = (tree.get_log_block_probability() - params.log_before).max(params.min_logp); + pattern_logps[pat] = logp; + return; + } + + for bit in [false, true] { + tree.update(bit, depth); + let mut next_symbol = symbol_acc; + if params.msb_first { + // Keep sub-byte MSB symbols packed into the low pattern range so the + // alias expansion below can index them via `byte & mask`. + let shift = compact_symbol_msb_shift(params.bits, depth); + if bit { + next_symbol |= 1u8 << shift; + } + } else if bit { + next_symbol |= 1u8 << depth; + } + rec(tree, depth + 1, params, next_symbol, pattern_logps); + tree.revert(depth); + } + } + + rec(tree, 0, ¶ms, 0, &mut pattern_logps); + + if bits == 8 { + out.copy_from_slice(&pattern_logps); + } else { + let aliases = 1usize << (8 - bits); + let alias_ln = (aliases as f64).ln(); + let mask = patterns - 1; + for byte in 0..256usize { + out[byte] = pattern_logps[byte & mask] - alias_ln; + } } } @@ -3499,6 +4871,48 @@ impl FacContextTree { mod tests { use super::*; + struct ScopedLogCacheLimit { + previous: Option, + } + + struct ScopedLogOverflowCacheSlots { + previous: Option, + } + + impl ScopedLogCacheLimit { + fn set(limit: usize) -> Self { + CTW_TEST_LOG_CACHE_LIMIT.with(|cell| { + let previous = cell.replace(Some(limit)); + Self { previous } + }) + } + } + + impl ScopedLogOverflowCacheSlots { + fn set(slots: usize) -> Self { + CTW_TEST_LOG_OVERFLOW_CACHE_SLOTS.with(|cell| { + let previous = cell.replace(Some(slots)); + Self { previous } + }) + } + } + + impl Drop for ScopedLogCacheLimit { + fn drop(&mut self) { + CTW_TEST_LOG_CACHE_LIMIT.with(|cell| { + cell.replace(self.previous); + }); + } + } + + impl Drop for ScopedLogOverflowCacheSlots { + fn drop(&mut self) { + CTW_TEST_LOG_OVERFLOW_CACHE_SLOTS.with(|cell| { + cell.replace(self.previous); + }); + } + } + #[derive(Clone)] struct RefNode { children: [Option>; 2], @@ -3565,8 +4979,7 @@ mod tests { self.max_depth, &self.history, sym_idx, - &self.log_int, - &self.log_half, + CachedLogs::new(&self.log_int, &self.log_half), ); self.history.push(sym); } @@ -3584,8 +4997,7 @@ mod tests { self.max_depth, &self.history, sym_idx, - &self.log_int, - &self.log_half, + CachedLogs::new(&self.log_int, &self.log_half), ); } @@ -3610,7 +5022,7 @@ mod tests { if reached_max_depth && idx == deepest { continue; } - let child_weight = if idx + 1 <= deepest { + let child_weight = if idx < deepest { entries[idx + 1].log_prob_weighted } else { 0.0 @@ -3631,14 +5043,13 @@ mod tests { self.root.log_prob_weighted } - fn update_node( + fn update_node( node: &mut RefNode, depth: usize, max_depth: usize, - history: &[Symbol], + history: &(impl HistoryAccess + ?Sized), sym_idx: usize, - log_int: &[f64], - log_half: &[f64], + logs: L, ) { if depth < max_depth { let edge = history_symbol(history, depth) as usize; @@ -3651,41 +5062,25 @@ mod tests { max_depth, history, sym_idx, - log_int, - log_half, + logs, ); } - apply_update_to_state_raw( - log_int, - log_half, - &mut node.symbol_count, - &mut node.log_prob_kt, - sym_idx, - ); + apply_update_to_state_raw(logs, &mut node.symbol_count, &mut node.log_prob_kt, sym_idx); Self::recompute(node); } - fn revert_node( + fn revert_node( node: &mut RefNode, depth: usize, max_depth: usize, - history: &[Symbol], + history: &(impl HistoryAccess + ?Sized), sym_idx: usize, - log_int: &[f64], - log_half: &[f64], + logs: L, ) -> bool { if depth < max_depth { let edge = history_symbol(history, depth) as usize; let remove_child = if let Some(child) = node.children[edge].as_deref_mut() { - Self::revert_node( - child, - depth + 1, - max_depth, - history, - sym_idx, - log_int, - log_half, - ) + Self::revert_node(child, depth + 1, max_depth, history, sym_idx, logs) } else { false }; @@ -3693,13 +5088,7 @@ mod tests { node.children[edge] = None; } } - apply_revert_to_state_raw( - log_int, - log_half, - &mut node.symbol_count, - &mut node.log_prob_kt, - sym_idx, - ); + apply_revert_to_state_raw(logs, &mut node.symbol_count, &mut node.log_prob_kt, sym_idx); Self::recompute(node); node.symbol_count[0] + node.symbol_count[1] == 0 } @@ -3708,7 +5097,7 @@ mod tests { node: &RefNode, depth: usize, max_depth: usize, - history: &[Symbol], + history: &(impl HistoryAccess + ?Sized), entries: &mut Vec, ) -> bool { let sibling_weight = if depth < max_depth { @@ -3792,7 +5181,10 @@ mod tests { assert!(diff <= 1e-12 * scale, "a={a} b={b} diff={diff}"); } - fn child_after_hot_prefix(tree: &ContextTree, history_before_update: &[Symbol]) -> ChildRef { + fn child_after_hot_prefix( + tree: &ContextTree, + history_before_update: &(impl HistoryAccess + ?Sized), + ) -> ChildRef { let hot_prefix_depth = tree.engine.hot_prefix_depth(); if hot_prefix_depth == 0 { return ChildRef::NONE; @@ -3840,9 +5232,303 @@ mod tests { assert_eq!(std::mem::size_of::(), 40); } + #[test] + fn log_lookup_matches_direct_log_formulas_past_cache_limit() { + let log_int = vec![f64::NEG_INFINITY, 0.0, (2.0f64).ln()]; + let log_half = vec![(0.5f64).ln(), (1.5f64).ln(), (2.5f64).ln()]; + let lookup = BoundedLogs::new(&log_int, &log_half); + + for n in 0..16usize { + let expected_int = if n == 0 { + f64::NEG_INFINITY + } else { + (n as f64).ln() + }; + assert_eq!(lookup.log_int(n).to_bits(), expected_int.to_bits()); + assert_eq!( + lookup.log_half(n).to_bits(), + (n as f64 + 0.5).ln().to_bits() + ); + } + } + + #[test] + fn log_lookup_overflow_cache_reuses_hot_exact_values() { + let slots = [LogCacheSlot::empty(), LogCacheSlot::empty()]; + let misses = Cell::new(0usize); + let first = BoundedLogs::lookup_overflow(&slots, 17, || { + misses.set(misses.get() + 1); + (17.0f64).ln() + }); + let second = BoundedLogs::lookup_overflow(&slots, 17, || { + misses.set(misses.get() + 1); + f64::NAN + }); + + assert_eq!(first.to_bits(), (17.0f64).ln().to_bits()); + assert_eq!(second.to_bits(), first.to_bits()); + assert_eq!(misses.get(), 1); + } + + #[test] + fn shared_log_cache_respects_test_limit() { + reset_shared_log_cache_for_test(); + let _limit = ScopedLogCacheLimit::set(3); + let _overflow_slots = ScopedLogOverflowCacheSlots::set(4); + with_shared_bounded_logs(32, |lookup| { + assert_eq!(lookup.log_int(32).to_bits(), (32.0f64).ln().to_bits()); + assert_eq!(lookup.log_half(32).to_bits(), (32.5f64).ln().to_bits()); + }); + + let (log_int_len, log_half_len) = shared_log_cache_lens(); + let (overflow_log_int_len, overflow_log_half_len) = shared_log_overflow_cache_lens(); + assert!(log_int_len <= 4, "log_int_len={log_int_len}"); + assert!(log_half_len <= 4, "log_half_len={log_half_len}"); + assert_eq!(overflow_log_int_len, 4); + assert_eq!(overflow_log_half_len, 4); + } + + #[test] + fn fac_ctw_memory_usage_breakdown_sums_to_existing_total() { + let mut fac = FacContextTree::new(7, 8); + let payload = b"ctw memory usage breakdown payload"; + for &byte in payload { + fac.update_byte_msb(byte); + } + + let usage = fac.memory_usage_breakdown(); + assert_eq!(fac.memory_usage(), usage.total_bytes()); + assert!(usage.tree_bytes > 0); + assert!(usage.shared_log_cache_bytes > 0); + assert!(usage.shared_history_bytes > 0); + + let telemetry = fac.telemetry(); + assert_eq!(telemetry.total_bytes, usage.total_bytes()); + assert_eq!(telemetry.tree_bytes, usage.tree_bytes); + assert_eq!( + telemetry.shared_log_cache_bytes, + usage.shared_log_cache_bytes + ); + assert_eq!(telemetry.shared_history_bytes, usage.shared_history_bytes); + assert_eq!(telemetry.shared_history_len_bits, payload.len() * 8); + assert_eq!(telemetry.trees.len(), 8); + assert_eq!( + telemetry.nodes_len, + telemetry + .trees + .iter() + .map(|tree| tree.nodes_len) + .sum::() + ); + assert_eq!( + telemetry.segments_len, + telemetry + .trees + .iter() + .map(|tree| tree.segments_len) + .sum::() + ); + assert_eq!( + telemetry.segments_len, + telemetry.exact_segments + + telemetry.history_segments + + telemetry.history_invert_segments + + telemetry.const_segments + ); + } + + #[test] + fn bit_history_preserves_logical_bits_and_packs_memory() { + let mut history = BitHistory::default(); + let bits: Vec = (0..130usize).map(|idx| (idx * 17 + 5) % 7 < 3).collect(); + history.extend_from_slice(&bits); + assert_eq!(history.len(), bits.len()); + assert_eq!(history.to_vec(), bits); + assert_eq!(history.memory_usage(), 3 * size_of::()); + for depth in 0..160usize { + assert_eq!( + history_symbol(&history, depth), + history_symbol(&bits, depth) + ); + for len in [0usize, 1, 2, 7, 31, 32, 33, 63, 64] { + assert_eq!( + path_bits_from_history(&history, depth, len), + path_bits_from_history(&bits, depth, len), + "depth={depth} len={len}", + ); + } + } + + let last = history.pop(); + assert_eq!(last, bits.last().copied()); + assert_eq!(history.len(), bits.len() - 1); + let popped_bits = &bits[..bits.len() - 1]; + for depth in 0..160usize { + assert_eq!( + history_symbol(&history, depth), + history_symbol(popped_bits, depth) + ); + for len in [0usize, 1, 2, 7, 31, 32, 33, 63, 64] { + assert_eq!( + path_bits_from_history(&history, depth, len), + path_bits_from_history(popped_bits, depth, len), + "after pop depth={depth} len={len}", + ); + } + } + + history.truncate(65); + assert_eq!(history.to_vec(), bits[..65].to_vec()); + assert_eq!(history.memory_usage(), 3 * size_of::()); + + history.clear(); + assert!(history.is_empty()); + assert_eq!(history.memory_usage(), 3 * size_of::()); + } + + #[test] + fn fac_ctw_history_memory_is_bit_packed() { + let mut fac = FacContextTree::new(4, 8); + fac.reserve_for_symbols(1_000); + let usage = fac.memory_usage_breakdown(); + assert_eq!( + usage.shared_history_bytes, + history_word_len(8_000) * size_of::() + ); + } + + #[test] + fn fac_ctw_memory_usage_breakdown_reports_bounded_log_cache_component() { + reset_shared_log_cache_for_test(); + let _limit = ScopedLogCacheLimit::set(2); + let _overflow_slots = ScopedLogOverflowCacheSlots::set(4); + let mut fac = FacContextTree::new(7, 8); + for &byte in b"bounded log cache memory component payload" { + fac.update_byte_msb(byte); + } + + let usage = fac.memory_usage_breakdown(); + let (log_int_len, log_half_len) = shared_log_cache_lens(); + let (overflow_log_int_len, overflow_log_half_len) = shared_log_overflow_cache_lens(); + assert!(log_int_len <= 3, "log_int_len={log_int_len}"); + assert!(log_half_len <= 3, "log_half_len={log_half_len}"); + assert_eq!(overflow_log_int_len, 4); + assert_eq!(overflow_log_half_len, 4); + assert!( + usage.shared_log_cache_bytes <= 16 * size_of::() + 32 * size_of::() + ); + assert_eq!(fac.memory_usage(), usage.total_bytes()); + } + + #[test] + fn bounded_log_lookup_preserves_context_tree_updates_and_reverts() { + let mut bounded = ContextTree::new(9); + let mut unbounded = bounded.clone(); + let stream = b"bounded exact log lookup context-tree parity payload"; + + { + let _limit = ScopedLogCacheLimit::set(3); + for &byte in stream { + for bit_idx in 0..8usize { + let bit = ((byte >> (7 - bit_idx)) & 1) == 1; + bounded.update(bit); + } + } + } + + for &byte in stream { + for bit_idx in 0..8usize { + let bit = ((byte >> (7 - bit_idx)) & 1) == 1; + unbounded.update(bit); + } + } + + assert_eq!( + bounded.get_log_block_probability().to_bits(), + unbounded.get_log_block_probability().to_bits() + ); + for &sym in &[false, true] { + assert_eq!( + bounded.predict(sym).to_bits(), + unbounded.predict(sym).to_bits() + ); + } + + { + let _limit = ScopedLogCacheLimit::set(3); + for _ in 0..16usize { + bounded.revert(); + } + } + for _ in 0..16usize { + unbounded.revert(); + } + + assert_eq!( + bounded.get_log_block_probability().to_bits(), + unbounded.get_log_block_probability().to_bits() + ); + for &sym in &[false, true] { + assert_eq!( + bounded.predict(sym).to_bits(), + unbounded.predict(sym).to_bits() + ); + } + } + + #[test] + fn bounded_log_lookup_preserves_fac_byte_fast_paths() { + let mut bounded_msb = FacContextTree::new(7, 8); + let mut unbounded_msb = bounded_msb.clone(); + let mut bounded_lsb = FacContextTree::new(7, 5); + let mut unbounded_lsb = bounded_lsb.clone(); + let stream = b"bounded exact log lookup fac byte parity payload"; + + { + let _limit = ScopedLogCacheLimit::set(2); + for &byte in stream { + bounded_msb.update_byte_msb(byte); + bounded_lsb.update_byte_lsb(byte); + } + } + for &byte in stream { + unbounded_msb.update_byte_msb(byte); + unbounded_lsb.update_byte_lsb(byte); + } + + assert_eq!( + bounded_msb.get_log_block_probability().to_bits(), + unbounded_msb.get_log_block_probability().to_bits() + ); + assert_eq!( + bounded_lsb.get_log_block_probability().to_bits(), + unbounded_lsb.get_log_block_probability().to_bits() + ); + for bit_idx in 0..bounded_msb.num_bits() { + assert_eq!( + bounded_msb.predict(false, bit_idx).to_bits(), + unbounded_msb.predict(false, bit_idx).to_bits() + ); + assert_eq!( + bounded_msb.predict_one(bit_idx).to_bits(), + unbounded_msb.predict_one(bit_idx).to_bits() + ); + } + for bit_idx in 0..bounded_lsb.num_bits() { + assert_eq!( + bounded_lsb.predict(false, bit_idx).to_bits(), + unbounded_lsb.predict(false, bit_idx).to_bits() + ); + assert_eq!( + bounded_lsb.predict_one(bit_idx).to_bits(), + unbounded_lsb.predict_one(bit_idx).to_bits() + ); + } + } + #[test] fn context_tree_singleton_paths_use_hot_prefix_nodes() { - let mut tree = ContextTree::new(12); + let mut tree = ContextTree::new(13); tree.update(false); let hot_prefix_depth = tree.engine.hot_prefix_depth(); @@ -3873,7 +5559,7 @@ mod tests { #[test] fn context_tree_missing_path_tail_uses_exact_segment_payloads() { - let mut tree = ContextTree::new(12); + let mut tree = ContextTree::new(13); tree.update(true); let child = tree.engine.arena.child(tree.engine.root, 0); let mut current = child.as_node().expect("hot-prefix node"); @@ -3932,8 +5618,11 @@ mod tests { .expect("history-backed segment tail"); let first_segment = tree.engine.arena.segments[first_segment.get()]; assert_eq!(first_segment.payload.mode(), SEG_MODE_HISTORY); - assert_eq!(first_segment.payload.len(), 69); - for offset in [0usize, 1, 7, 31, 68] { + assert_eq!( + first_segment.payload.len() as usize, + tree.engine.max_depth - tree.engine.hot_prefix_depth() - 1 + ); + for offset in [0usize, 1, 7, 31, 66] { assert_eq!( segment_edge_from_parts( first_segment, @@ -4185,6 +5874,32 @@ mod tests { } } + #[test] + fn fac_ctw_log_prob_update_byte_msb_matches_manual_fast_path() { + let mut batched = FacContextTree::new(6, 8); + for &byte in b"log prob update byte msb regression payload" { + let mut manual = batched.clone(); + let observed = batched.log_prob_update_byte_msb(byte); + let mut expected = 0.0; + for bit_idx in 0..8usize { + let bit = ((byte >> (7 - bit_idx)) & 1) == 1; + let p = manual.predict(bit, bit_idx); + if p.is_finite() && p > 0.0 { + expected += p.ln(); + } else { + expected = f64::NEG_INFINITY; + } + manual.update_predicted(bit, bit_idx); + } + assert_eq!(observed.to_bits(), expected.to_bits()); + assert_eq!(batched.shared_history, manual.shared_history); + assert_eq!( + batched.get_log_block_probability().to_bits(), + manual.get_log_block_probability().to_bits(), + ); + } + } + #[test] fn fac_ctw_update_byte_lsb_matches_bit_updates() { let mut by_byte = FacContextTree::new(6, 5); @@ -4284,7 +5999,7 @@ mod tests { #[test] fn fac_ctw_update_predicted_ignores_stale_cache_after_reset_and_rewrite() { assert_update_predicted_matches_fresh_after_history_rewrite(|fac| { - let mut rewritten = fac.shared_history.clone(); + let mut rewritten = fac.shared_history.to_vec(); for bit in &mut rewritten { *bit = !*bit; } @@ -4296,7 +6011,7 @@ mod tests { #[test] fn fac_ctw_update_predicted_ignores_stale_cache_after_revert_and_rewrite() { assert_update_predicted_matches_fresh_after_history_rewrite(|fac| { - let original = fac.shared_history.clone(); + let original = fac.shared_history.to_vec(); let keep = original.len() / 3; let remove = original.len() - keep; let mut rewritten_suffix = original[keep..].to_vec(); @@ -4393,6 +6108,67 @@ mod tests { assert_close(fac.predict(true, 3), p1_before); } + #[test] + fn ctw_inline_node_weight_recompute_matches_recompute_node_weight() { + fn inline_weight(arena: &CtArena, idx: NodeIndex) -> f64 { + let slot = idx.get(); + let node = arena.nodes[slot]; + let [left, right] = node.children; + if left.is_none() && right.is_none() { + clamp_log_prob(node.log_prob_kt) + } else { + let w0 = arena.child_ref_weighted(left); + let w1 = arena.child_ref_weighted(right); + update_weighted_log_prob_non_leaf(node.log_prob_kt, w0, w1) + } + } + + let mut arena = CtArena::new(); + let parent = arena.alloc_node_with_state([3, 5], -1.75); + let left_node = arena.alloc_node_with_state([2, 1], -0.25); + let right_node = arena.alloc_node_with_state([1, 2], -0.50); + arena.nodes[left_node.get()].log_prob_weighted = -0.333_333_333_f64; + arena.nodes[right_node.get()].log_prob_weighted = -0.777_777_777_f64; + + let seg = arena.alloc_segment(); + arena.segments[seg.get()].head_log_prob_weighted = -0.125_f64; + + let cases = [ + (ChildRef::NONE, ChildRef::NONE, -2.0_f64), + ( + ChildRef::from_node(left_node), + ChildRef::from_node(right_node), + -1.0_f64, + ), + ( + ChildRef::from_node(left_node), + ChildRef::from_segment(seg), + -0.625_f64, + ), + ( + ChildRef::from_segment(seg), + ChildRef::from_node(right_node), + -0.3125_f64, + ), + ]; + + for (left, right, kt) in cases { + let slot = parent.get(); + arena.nodes[slot].children = [left, right]; + arena.nodes[slot].log_prob_kt = kt; + arena.nodes[slot].log_prob_weighted = f64::NAN; + + let expected = inline_weight(&arena, parent); + arena.recompute_node_weight(parent); + let actual = arena.nodes[slot].log_prob_weighted; + assert_eq!( + actual.to_bits(), + expected.to_bits(), + "node-weight recompute mismatch for children=({left:?},{right:?}) kt={kt}" + ); + } + } + #[test] fn fac_ctw_predict_matches_update_ratio() { let mut fac = FacContextTree::new(6, 8); @@ -4503,42 +6279,34 @@ mod tests { rec(tree, bits, 0); } - fn byte_log_prob(tree: &mut FacContextTree, symbol: u8, msb_first: bool, bits: usize) -> f64 { - let before = tree.get_log_block_probability(); + fn fac_symbol_bit(symbol: u8, msb_first: bool, bits: usize, bit_idx: usize) -> bool { if msb_first { - for bit_idx in 0..bits { - let bit = ((symbol >> (7 - bit_idx)) & 1) == 1; - tree.update(bit, bit_idx); - } - let after = tree.get_log_block_probability(); - for bit_idx in (0..bits).rev() { - tree.revert(bit_idx); - } - after - before + ctw_symbol_bit_msb(symbol, bits, bit_idx) } else { - for bit_idx in 0..bits { - let bit = ((symbol >> bit_idx) & 1) == 1; - tree.update(bit, bit_idx); - } - let after = tree.get_log_block_probability(); - for bit_idx in (0..bits).rev() { - tree.revert(bit_idx); - } - after - before + ((symbol >> bit_idx) & 1) == 1 } } + fn symbol_log_prob(tree: &mut FacContextTree, symbol: u8, msb_first: bool, bits: usize) -> f64 { + let before = tree.get_log_block_probability(); + for bit_idx in 0..bits { + let bit = fac_symbol_bit(symbol, msb_first, bits, bit_idx); + tree.update(bit, bit_idx); + } + let after = tree.get_log_block_probability(); + for bit_idx in (0..bits).rev() { + tree.revert(bit_idx); + } + after - before + } + fn assert_symbol_scan_then_update_matches_plain(msb_first: bool) { let bits = 8usize; let mut with_scan = FacContextTree::new(7, bits); let mut plain = with_scan.clone(); for &byte in b"pdf then update parity payload" { for bit_idx in 0..bits { - let bit = if msb_first { - ((byte >> (7 - bit_idx)) & 1) == 1 - } else { - ((byte >> bit_idx) & 1) == 1 - }; + let bit = fac_symbol_bit(byte, msb_first, bits, bit_idx); with_scan.update(bit, bit_idx); plain.update(bit, bit_idx); } @@ -4548,18 +6316,14 @@ mod tests { let observed = b'n'; for bit_idx in 0..bits { - let bit = if msb_first { - ((observed >> (7 - bit_idx)) & 1) == 1 - } else { - ((observed >> bit_idx) & 1) == 1 - }; + let bit = fac_symbol_bit(observed, msb_first, bits, bit_idx); with_scan.update(bit, bit_idx); plain.update(bit, bit_idx); } for sym in 0u8..=255u8 { - let lp_scan = byte_log_prob(&mut with_scan, sym, msb_first, bits); - let lp_plain = byte_log_prob(&mut plain, sym, msb_first, bits); + let lp_scan = symbol_log_prob(&mut with_scan, sym, msb_first, bits); + let lp_plain = symbol_log_prob(&mut plain, sym, msb_first, bits); let diff = (lp_scan - lp_plain).abs(); assert!( diff < 1e-12, @@ -4568,6 +6332,53 @@ mod tests { } } + fn assert_fill_fac_tree_log_probs_matches_direct_symbol_probs(msb_first: bool, bits: usize) { + let bits = bits.clamp(1, 8); + let min_logp = 1e-12f64.ln(); + let patterns = 1usize << bits; + let alias_ln = if bits == 8 { + 0.0 + } else { + ((1usize << (8 - bits)) as f64).ln() + }; + let training = [0x0u8, 0x3, 0x5, 0x6, 0x9, 0xA, 0xC, 0xF, 0x7, 0x1]; + let mut tree = FacContextTree::new(7, bits); + for &symbol in &training { + for bit_idx in 0..bits { + tree.update(fac_symbol_bit(symbol, msb_first, bits, bit_idx), bit_idx); + } + } + + let log_before = tree.get_log_block_probability(); + let mut predict_zero_before = vec![0.0; bits]; + let mut predict_one_before = vec![0.0; bits]; + for bit_idx in 0..bits { + predict_zero_before[bit_idx] = tree.predict(false, bit_idx); + predict_one_before[bit_idx] = tree.predict(true, bit_idx); + } + + let mut out = [0.0; 256]; + fill_fac_tree_log_probs(&mut tree, bits, msb_first, min_logp, &mut out); + + assert_close(tree.get_log_block_probability(), log_before); + for bit_idx in 0..bits { + assert_close(tree.predict(false, bit_idx), predict_zero_before[bit_idx]); + assert_close(tree.predict(true, bit_idx), predict_one_before[bit_idx]); + } + + let mask = patterns - 1; + for (byte, &actual) in out.iter().enumerate() { + let symbol = if bits == 8 { + byte as u8 + } else { + (byte & mask) as u8 + }; + let expected = + symbol_log_prob(&mut tree, symbol, msb_first, bits).max(min_logp) - alias_ln; + assert_close(actual, expected); + } + } + #[test] fn fac_ctw_symbol_scan_then_update_matches_plain_msb() { assert_symbol_scan_then_update_matches_plain(true); @@ -4577,4 +6388,14 @@ mod tests { fn fac_ctw_symbol_scan_then_update_matches_plain_lsb() { assert_symbol_scan_then_update_matches_plain(false); } + + #[test] + fn fill_fac_tree_log_probs_matches_direct_symbol_probs_for_subbyte_msb() { + assert_fill_fac_tree_log_probs_matches_direct_symbol_probs(true, 4); + } + + #[test] + fn fill_fac_tree_log_probs_matches_direct_symbol_probs_for_subbyte_lsb() { + assert_fill_fac_tree_log_probs_matches_direct_symbol_probs(false, 4); + } } diff --git a/crates/infotheory/src/backends/fixed_gemv.rs b/crates/infotheory/src/backends/fixed_gemv.rs new file mode 100644 index 00000000..045bc5ad --- /dev/null +++ b/crates/infotheory/src/backends/fixed_gemv.rs @@ -0,0 +1,317 @@ +//! Shared crate-private fixed-shape GEMV specializations for neural backends. +//! +//! The current consumer is RWKV7. Shapes are intentionally limited to the +//! benchmark-stable hot path so we can demand a strict binary-size gate. + +use wide::f32x8; + +const LANES: usize = 8; + +#[inline(always)] +/// Load one SIMD lane group from `ptr`. +/// +/// # Safety +/// +/// `ptr..ptr.add(LANES)` must be valid for reading initialized `f32` values. +/// The pointer need not be aligned for `f32x8`; this helper performs an +/// unaligned SIMD load. +unsafe fn load8(ptr: *const f32) -> f32x8 { + // SAFETY: the caller supplies a readable lane group; `read_unaligned` + // removes any extra SIMD alignment requirement. + unsafe { ptr.cast::().read_unaligned() } +} + +#[inline(always)] +/// Store one SIMD lane group to `ptr`. +/// +/// # Safety +/// +/// `ptr..ptr.add(LANES)` must be valid for writing `f32` values. The pointer +/// need not be aligned for `f32x8`; this helper performs an unaligned SIMD +/// store. +unsafe fn store8(ptr: *mut f32, v: f32x8) { + // SAFETY: the caller supplies a writable lane group; `write_unaligned` + // removes any extra SIMD alignment requirement. + unsafe { ptr.cast::().write_unaligned(v) } +} + +#[inline(always)] +/// Specialized matrix-vector multiply for a compile-time matrix shape. +/// +/// Computes `y = A @ x` for an `A` matrix with shape +/// `ROWS x (CHUNKS * LANES)`. +/// +/// # Safety +/// +/// `a` must be valid for reading `ROWS * CHUNKS * LANES` initialized `f32` +/// values, `x` must be valid for reading `CHUNKS * LANES` initialized `f32` +/// values, and `y` must be valid for writing `ROWS` `f32` values. `y` must not +/// overlap `a` or `x` for the duration of the call. `ROWS` must be divisible +/// by the fixed row batch; this is asserted when the specialization is +/// monomorphized. +unsafe fn gemv_fixed( + a: *const f32, + x: *const f32, + y: *mut f32, +) { + const ROW_BATCH: usize = 4; + const { + assert!(ROWS.is_multiple_of(ROW_BATCH)); + } + const fn cols_for() -> usize { + CHUNKS * LANES + } + let cols = cols_for::(); + let mut r = 0usize; + while r < ROWS { + // SAFETY: `ROWS` is asserted divisible by `ROW_BATCH`, so each loop + // entry starts a complete row batch within the backing matrix. + let row0 = unsafe { a.add(r * cols) }; + let row1 = unsafe { a.add((r + 1) * cols) }; + let row2 = unsafe { a.add((r + 2) * cols) }; + let row3 = unsafe { a.add((r + 3) * cols) }; + + let mut sum0 = f32x8::ZERO; + let mut sum1 = f32x8::ZERO; + let mut sum2 = f32x8::ZERO; + let mut sum3 = f32x8::ZERO; + + let mut c = 0usize; + while c < cols { + // SAFETY: `cols == CHUNKS * LANES`, so every `c` visited here + // starts a complete lane group within `x` and each row. + let xv = unsafe { load8(x.add(c)) }; + sum0 += unsafe { load8(row0.add(c)) } * xv; + sum1 += unsafe { load8(row1.add(c)) } * xv; + sum2 += unsafe { load8(row2.add(c)) } * xv; + sum3 += unsafe { load8(row3.add(c)) } * xv; + c += LANES; + } + + // SAFETY: the loop guard proves the four output rows are in bounds, + // and the caller guarantees `y` is writable for `ROWS` values. + unsafe { + *y.add(r) = sum0.reduce_add(); + *y.add(r + 1) = sum1.reduce_add(); + *y.add(r + 2) = sum2.reduce_add(); + *y.add(r + 3) = sum3.reduce_add(); + } + r += ROW_BATCH; + } +} + +#[inline(always)] +/// Specialized transposed matrix-vector multiply for a compile-time shape. +/// +/// Computes `y = A^T @ x` for an `A` matrix with shape +/// `ROWS x (CHUNKS * LANES)`. +/// +/// # Safety +/// +/// `a` must be valid for reading `ROWS * CHUNKS * LANES` initialized `f32` +/// values, `x` must be valid for reading `ROWS` initialized `f32` values, and +/// `y` must be valid for writing `CHUNKS * LANES` `f32` values. `y` must not +/// overlap `a` or `x` for the duration of the call. +unsafe fn gemv_t_fixed( + a: *const f32, + x: *const f32, + y: *mut f32, +) { + const fn cols_for() -> usize { + CHUNKS * LANES + } + let cols = cols_for::(); + let mut c = 0usize; + while c < cols { + // SAFETY: `cols == CHUNKS * LANES`, so each `c` starts a complete + // output lane group within `y`. + unsafe { store8(y.add(c), f32x8::ZERO) }; + c += LANES; + } + + let mut r = 0usize; + while r < ROWS { + // SAFETY: `r < ROWS`, and the caller guarantees `x` is readable for + // every row coordinate. + let x_r = f32x8::splat(unsafe { *x.add(r) }); + // SAFETY: `r < ROWS`, and the matrix contract covers the full row. + let row = unsafe { a.add(r * cols) }; + let mut c = 0usize; + while c < cols { + // SAFETY: `cols == CHUNKS * LANES`, so every `c` starts a complete + // lane group within the row and output vector. + let yv = unsafe { load8(y.add(c)) }; + let av = unsafe { load8(row.add(c)) }; + unsafe { store8(y.add(c), yv + av * x_r) }; + c += LANES; + } + r += 1; + } +} + +#[inline(always)] +/// Try a fixed-shape `y = A @ x` specialization. +/// +/// Returns `false` without touching memory when `(rows, cols)` is not one of +/// the curated hot shapes. +/// +/// # Safety +/// +/// For supported shapes, `a` must be valid for reading `rows * cols` +/// initialized `f32` values, `x` must be valid for reading `cols` initialized +/// `f32` values, and `y` must be valid for writing `rows` `f32` values. `y` +/// must not overlap `a` or `x` for the duration of the call. +pub(crate) unsafe fn try_gemv( + a: *const f32, + x: *const f32, + y: *mut f32, + rows: usize, + cols: usize, +) -> bool { + match (rows, cols) { + // SAFETY: the matched runtime shape exactly equals the const-generic + // shape, and this function's caller supplies the backing buffers. + (256, 64) => unsafe { gemv_fixed::<256, 8>(a, x, y) }, + (64, 64) => unsafe { gemv_fixed::<64, 8>(a, x, y) }, + (16, 64) => unsafe { gemv_fixed::<16, 8>(a, x, y) }, + (64, 16) => unsafe { gemv_fixed::<64, 2>(a, x, y) }, + _ => return false, + } + true +} + +#[inline(always)] +/// Try a fixed-shape `y = A^T @ x` specialization. +/// +/// Returns `false` without touching memory when `(rows, cols)` is not one of +/// the curated hot shapes. +/// +/// # Safety +/// +/// For supported shapes, `a` must be valid for reading `rows * cols` +/// initialized `f32` values, `x` must be valid for reading `rows` initialized +/// `f32` values, and `y` must be valid for writing `cols` `f32` values. `y` +/// must not overlap `a` or `x` for the duration of the call. +pub(crate) unsafe fn try_gemv_t( + a: *const f32, + x: *const f32, + y: *mut f32, + rows: usize, + cols: usize, +) -> bool { + match (rows, cols) { + // SAFETY: the matched runtime shape exactly equals the const-generic + // shape, and this function's caller supplies the backing buffers. + (256, 64) => unsafe { gemv_t_fixed::<256, 8>(a, x, y) }, + (64, 64) => unsafe { gemv_t_fixed::<64, 8>(a, x, y) }, + (16, 64) => unsafe { gemv_t_fixed::<16, 8>(a, x, y) }, + (64, 16) => unsafe { gemv_t_fixed::<64, 2>(a, x, y) }, + _ => return false, + } + true +} + +#[cfg(test)] +mod tests { + use super::{try_gemv, try_gemv_t}; + + #[derive(Clone, Copy)] + struct Lcg(u64); + + impl Lcg { + fn new(seed: u64) -> Self { + Self(seed) + } + + fn next_f32(&mut self) -> f32 { + self.0 = self + .0 + .wrapping_mul(6364136223846793005) + .wrapping_add(1442695040888963407); + let bits = ((self.0 >> 40) as u32) | 0x3f80_0000; + f32::from_bits(bits) - 1.0 + } + } + + fn fill_centered(dst: &mut [f32], rng: &mut Lcg, scale: f32) { + for x in dst { + *x = (rng.next_f32() * 2.0 - 1.0) * scale; + } + } + + fn gemv_scalar(a: &[f32], x: &[f32], rows: usize, cols: usize) -> Vec { + let mut y = vec![0.0; rows]; + for r in 0..rows { + let row = &a[r * cols..(r + 1) * cols]; + let mut acc = 0.0f32; + for c in 0..cols { + acc += row[c] * x[c]; + } + y[r] = acc; + } + y + } + + fn gemv_t_scalar(a: &[f32], x: &[f32], rows: usize, cols: usize) -> Vec { + let mut y = vec![0.0; cols]; + for r in 0..rows { + let row = &a[r * cols..(r + 1) * cols]; + for c in 0..cols { + y[c] += row[c] * x[r]; + } + } + y + } + + fn assert_close(lhs: &[f32], rhs: &[f32], tol: f32) { + assert_eq!(lhs.len(), rhs.len()); + for idx in 0..lhs.len() { + let diff = (lhs[idx] - rhs[idx]).abs(); + assert!( + diff <= tol, + "mismatch at {idx}: lhs={} rhs={} diff={diff}", + lhs[idx], + rhs[idx] + ); + } + } + + fn check_shape(rows: usize, cols: usize) { + let mut rng = Lcg::new(((rows as u64) << 32) ^ (cols as u64) ^ 0xC0FFEEu64); + let mut a = vec![0.0; rows * cols]; + let mut x = vec![0.0; cols]; + let mut xt = vec![0.0; rows]; + fill_centered(&mut a, &mut rng, 0.8); + fill_centered(&mut x, &mut rng, 0.5); + fill_centered(&mut xt, &mut rng, 0.4); + + let mut y = vec![0.0; rows]; + assert!(unsafe { try_gemv(a.as_ptr(), x.as_ptr(), y.as_mut_ptr(), rows, cols) }); + let y_ref = gemv_scalar(&a, &x, rows, cols); + assert_close(&y, &y_ref, 2.5e-5); + + let mut yt = vec![0.0; cols]; + assert!(unsafe { try_gemv_t(a.as_ptr(), xt.as_ptr(), yt.as_mut_ptr(), rows, cols) }); + let yt_ref = gemv_t_scalar(&a, &xt, rows, cols); + assert_close(&yt, &yt_ref, 2.5e-5); + } + + #[test] + fn fixed_shapes_match_scalar_reference() { + check_shape(256, 64); + check_shape(64, 64); + check_shape(16, 64); + check_shape(64, 16); + } + + #[test] + fn unsupported_shapes_fall_back() { + let a = [0.0; 11 * 37]; + let x = [0.0; 37]; + let xt = [0.0; 11]; + let mut y = [0.0; 11]; + let mut yt = [0.0; 37]; + assert!(!unsafe { try_gemv(a.as_ptr(), x.as_ptr(), y.as_mut_ptr(), 11, 37) }); + assert!(!unsafe { try_gemv_t(a.as_ptr(), xt.as_ptr(), yt.as_mut_ptr(), 11, 37) }); + } +} diff --git a/src/backends/llm_policy.rs b/crates/infotheory/src/backends/llm_policy.rs similarity index 65% rename from src/backends/llm_policy.rs rename to crates/infotheory/src/backends/llm_policy.rs index 53a80e1f..1821681a 100644 --- a/src/backends/llm_policy.rs +++ b/crates/infotheory/src/backends/llm_policy.rs @@ -1,6 +1,6 @@ use anyhow::{Context, Result, bail}; use std::collections::BTreeSet; -use std::path::PathBuf; +use std::path::{Component, Path, PathBuf}; #[derive(Clone, Debug, PartialEq)] /// Position expression used in policy schedules. @@ -293,6 +293,184 @@ impl CompiledPolicy { } PolicyAction::Infer } + + /// Returns whether any position at or after `cursor` can emit a matching train action. + pub fn has_future_train_matching_from( + &self, + cursor: u64, + mut predicate: impl FnMut(&TrainAction) -> bool, + ) -> bool { + fn active_range(rule: &CompiledScheduleRule) -> (u64, u64) { + match rule { + CompiledScheduleRule::Interval { start, end, .. } + | CompiledScheduleRule::Repeat { start, end, .. } => (*start, *end), + } + } + + fn action_matches_train( + action: &PolicyAction, + predicate: &mut impl FnMut(&TrainAction) -> bool, + ) -> bool { + match action { + PolicyAction::Infer => false, + PolicyAction::Train(train) => predicate(train), + } + } + + fn interval_uncovered_or_next( + start: u64, + end: u64, + prior_rules: &[CompiledScheduleRule], + ) -> Result<(), u64> { + if start >= end { + return Err(end); + } + let mut probe = start; + while probe < end { + let mut next_probe = probe; + for rule in prior_rules { + let (rule_start, rule_end) = active_range(rule); + if rule_start <= probe && probe < rule_end { + next_probe = next_probe.max(rule_end.min(end)); + } + } + if next_probe == probe { + return Ok(()); + } + probe = next_probe; + } + Err(probe) + } + + fn interval_has_uncovered_position( + start: u64, + end: u64, + prior_rules: &[CompiledScheduleRule], + ) -> bool { + interval_uncovered_or_next(start, end, prior_rules).is_ok() + } + + fn next_repeat_segment_interval( + start: u64, + end: u64, + period: u64, + seg_start: u64, + seg_end: u64, + from: u64, + ) -> Option<(u64, u64)> { + if period == 0 { + return None; + } + let seg_end = seg_end.min(period); + if seg_start >= seg_end { + return None; + } + let from = from.max(start); + if from >= end { + return None; + } + let rel = from - start; + let mut cycle_idx = rel / period; + loop { + let cycle_base = start.saturating_add(cycle_idx.saturating_mul(period)); + if cycle_base >= end { + return None; + } + let interval_start = cycle_base.saturating_add(seg_start).max(from); + let interval_end = cycle_base.saturating_add(seg_end).min(end); + if interval_start < interval_end { + return Some((interval_start, interval_end)); + } + cycle_idx = cycle_idx.saturating_add(1); + } + } + + fn repeat_rule_has_future_matching_train( + start: u64, + end: u64, + period: u64, + pattern: &[CompiledPatternSegment], + cursor: u64, + prior_rules: &[CompiledScheduleRule], + predicate: &mut impl FnMut(&TrainAction) -> bool, + ) -> bool { + if period == 0 { + return false; + } + let active_start = cursor.max(start); + if active_start >= end { + return false; + } + let mut seg_start = 0u64; + for seg in pattern { + let seg_end = seg.end.min(period); + if seg_start >= seg_end { + seg_start = seg.end; + continue; + } + if action_matches_train(&seg.action, predicate) { + let mut search_from = active_start; + while let Some((candidate_start, candidate_end)) = next_repeat_segment_interval( + start, + end, + period, + seg_start, + seg_end, + search_from, + ) { + match interval_uncovered_or_next( + candidate_start, + candidate_end, + prior_rules, + ) { + Ok(()) => return true, + Err(next) if next > search_from => search_from = next, + Err(_) => search_from = candidate_end, + } + } + } + seg_start = seg.end; + } + false + } + + for (idx, rule) in self.rules.iter().enumerate() { + let prior_rules = &self.rules[..idx]; + match rule { + CompiledScheduleRule::Interval { start, end, action } + if cursor < *end && cursor.max(*start) < *end => + { + let candidate_start = cursor.max(*start); + if action_matches_train(action, &mut predicate) + && interval_has_uncovered_position(candidate_start, *end, prior_rules) + { + return true; + } + } + CompiledScheduleRule::Repeat { + start, + end, + period, + pattern_total: _, + pattern, + } => { + if repeat_rule_has_future_matching_train( + *start, + *end, + *period, + pattern, + cursor, + prior_rules, + &mut predicate, + ) { + return true; + } + } + _ => {} + } + } + false + } } #[derive(Clone, Debug)] @@ -336,6 +514,13 @@ impl PolicyRuntime { self.cursor = self.cursor.saturating_add(1); action } + + #[inline] + /// Returns whether any future cursor position can emit a matching train action. + pub fn has_future_train_matching(&self, predicate: impl FnMut(&TrainAction) -> bool) -> bool { + self.compiled + .has_future_train_matching_from(self.cursor, predicate) + } } /// Split `method` into base method segment and optional `policy:...` segment. @@ -344,6 +529,23 @@ pub fn split_method_policy_segments(method: &str) -> Result<(String, Option Result<(String, Option Option<&str> { + let path = method.strip_prefix("file:")?; + let bytes = path.as_bytes(); + let mut i = 0usize; + while i < bytes.len() { + if bytes[i] != b';' { + i += 1; + continue; + } + let start = i + 1; + let mut j = start; + match bytes.get(j) { + Some(b'a'..=b'z' | b'A'..=b'Z' | b'_') => {} + _ => { + i += 1; + continue; + } + } + j += 1; + while matches!( + bytes.get(j), + Some(b'a'..=b'z' | b'A'..=b'Z' | b'0'..=b'9' | b'_' | b'-') + ) { + j += 1; + } + if matches!(bytes.get(j), Some(b':')) { + return std::str::from_utf8(&bytes[start..j]).ok(); + } + i += 1; + } + None +} + +fn push_rendered_path_piece(out: &mut String, piece: &str, normalize_separators: bool) { + for ch in piece.chars() { + match ch { + '\\' if normalize_separators => out.push('/'), + '%' => out.push_str("%25"), + ';' => out.push_str("%3B"), + _ => out.push(ch), + } + } +} + +/// Render a canonical `file:` path using slash separators and delimiter-safe escapes. +pub(crate) fn render_method_file_path(path: &Path) -> String { + let mut out = String::new(); + for component in path.components() { + match component { + Component::Prefix(prefix) => { + push_rendered_path_piece( + &mut out, + prefix.as_os_str().to_string_lossy().as_ref(), + true, + ); + } + Component::RootDir => out.push('/'), + Component::CurDir => push_rendered_path_piece(&mut out, ".", false), + Component::ParentDir => push_rendered_path_piece(&mut out, "..", false), + Component::Normal(piece) => { + if !out.is_empty() && !out.ends_with('/') { + out.push('/'); + } + push_rendered_path_piece(&mut out, piece.to_string_lossy().as_ref(), false); + } + } + } + out +} + +/// Decode percent-encoded delimiter characters inside a method `file:` path. +pub(crate) fn parse_method_file_path(path: &str) -> PathBuf { + let bytes = path.as_bytes(); + let mut out = Vec::with_capacity(bytes.len()); + let mut i = 0usize; + while i < bytes.len() { + if bytes[i] == b'%' && i + 2 < bytes.len() { + match (bytes[i + 1], bytes[i + 2]) { + (b'2', b'5') => { + out.push(b'%'); + i += 3; + continue; + } + (b'3', b'B' | b'b') => { + out.push(b';'); + i += 3; + continue; + } + _ => {} + } + } + out.push(bytes[i]); + i += 1; + } + PathBuf::from(String::from_utf8_lossy(&out).into_owned()) +} + +/// Render a `file:[;policy:...]` method string using the shared delimiter-safe encoding. +pub fn canonical_file_method_string(path: &Path, policy: Option<&LlmPolicy>) -> Result { + let mut method = format!("file:{}", render_method_file_path(path)); + if let Some(policy) = policy { + method.push_str(";policy:"); + method.push_str(&policy.canonical()); + } + Ok(method) +} + /// Parse a `policy:...` string into [`LlmPolicy`]. pub fn parse_policy_segment(policy_segment: &str, allowed_scopes: &[&str]) -> Result { let body = policy_segment @@ -872,6 +1181,61 @@ mod tests { assert!(matches!(c.action_at(10), PolicyAction::Train(_))); } + #[test] + fn policy_runtime_future_train_matching_interval_respects_cursor() { + let p = parse_policy_segment( + "schedule=0..5:infer|5..10:train(scope=attn,opt=adam,lr=0.1,stride=1,bptt=2,clip=0,momentum=0.9)|10..20:infer", + RWKV_SCOPES, + ) + .expect("policy"); + let compiled = p.compile(Some(20)).expect("compile"); + let mut runtime = PolicyRuntime::new(compiled); + assert!(runtime.has_future_train_matching(|train| train.scope.contains("attn"))); + runtime.set_cursor(10); + assert!(!runtime.has_future_train_matching(|train| train.scope.contains("attn"))); + } + + #[test] + fn policy_runtime_future_train_matching_repeat_respects_remaining_cycles() { + let p = parse_policy_segment( + "schedule=repeat(0..30,period=10,pattern=2:train(scope=attn,opt=adam,lr=0.1,stride=1,bptt=2,clip=0,momentum=0.9)+8:infer)", + RWKV_SCOPES, + ) + .expect("repeat policy"); + let compiled = p.compile(Some(30)).expect("compile"); + let mut runtime = PolicyRuntime::new(compiled); + runtime.set_cursor(9); + assert!(runtime.has_future_train_matching(|train| train.scope.contains("attn"))); + runtime.set_cursor(29); + assert!(!runtime.has_future_train_matching(|train| train.scope.contains("attn"))); + } + + #[test] + fn policy_runtime_future_train_matching_respects_shadowed_interval_precedence() { + let p = parse_policy_segment( + "schedule=0..100:infer|10..20:train(scope=attn,opt=adam,lr=0.1,stride=1,bptt=2,clip=0,momentum=0.9)", + RWKV_SCOPES, + ) + .expect("policy"); + let compiled = p.compile(Some(100)).expect("compile"); + let runtime = PolicyRuntime::new(compiled); + assert!(matches!(runtime.peek_action(), PolicyAction::Infer)); + assert!(!runtime.has_future_train_matching(|train| train.scope.contains("attn"))); + } + + #[test] + fn policy_runtime_future_train_matching_respects_shadowed_repeat_precedence() { + let p = parse_policy_segment( + "schedule=repeat(0..100,period=10,pattern=10:infer)|10..20:train(scope=attn,opt=adam,lr=0.1,stride=1,bptt=2,clip=0,momentum=0.9)", + RWKV_SCOPES, + ) + .expect("policy"); + let compiled = p.compile(Some(100)).expect("compile"); + let runtime = PolicyRuntime::new(compiled); + assert!(matches!(runtime.peek_action(), PolicyAction::Infer)); + assert!(!runtime.has_future_train_matching(|train| train.scope.contains("attn"))); + } + #[test] fn split_method_policy() { let (base, pol) = @@ -880,4 +1244,43 @@ mod tests { assert_eq!(base, "cfg:hidden=64"); assert_eq!(pol.as_deref(), Some("schedule=0..100:infer")); } + + #[test] + fn split_method_policy_preserves_file_path_semicolons() { + let (base, pol) = + split_method_policy_segments("file:/tmp/model;v1.safetensors").expect("split"); + assert_eq!(base, "file:/tmp/model;v1.safetensors"); + assert_eq!(pol, None); + } + + #[test] + fn method_file_path_roundtrips_reserved_delimiters() { + let path = Path::new("/tmp/model;policy:v1%done.safetensors"); + let rendered = render_method_file_path(path); + assert_eq!(rendered, "/tmp/model%3Bpolicy:v1%25done.safetensors"); + assert_eq!(parse_method_file_path(&rendered), path); + } + + #[test] + fn method_file_path_preserves_unowned_percent_sequences() { + let path = "/tmp/model%2Fv1.safetensors"; + assert_eq!(parse_method_file_path(path), Path::new(path)); + } + + #[cfg(not(windows))] + #[test] + fn method_file_path_preserves_literal_backslashes_in_unix_components() { + let path = Path::new(r"weights\model;snap%done.safetensors"); + let rendered = render_method_file_path(path); + assert_eq!(rendered, r"weights\model%3Bsnap%25done.safetensors"); + } + + #[test] + fn split_method_policy_rejects_ambiguous_file_suffixes() { + let err = split_method_policy_segments("file:/tmp/model;polciy:train").unwrap_err(); + assert!( + err.to_string() + .contains("ambiguous file method segment ';polciy:'") + ); + } } diff --git a/src/backends/mambazip/mamba1/kernel.rs b/crates/infotheory/src/backends/mambazip/mamba1/kernel.rs similarity index 100% rename from src/backends/mambazip/mamba1/kernel.rs rename to crates/infotheory/src/backends/mambazip/mamba1/kernel.rs diff --git a/src/backends/mambazip/mamba1/mod.rs b/crates/infotheory/src/backends/mambazip/mamba1/mod.rs similarity index 100% rename from src/backends/mambazip/mamba1/mod.rs rename to crates/infotheory/src/backends/mambazip/mamba1/mod.rs diff --git a/src/backends/mambazip/mamba1/model.rs b/crates/infotheory/src/backends/mambazip/mamba1/model.rs similarity index 99% rename from src/backends/mambazip/mamba1/model.rs rename to crates/infotheory/src/backends/mambazip/mamba1/model.rs index c9334d71..23ce0a49 100644 --- a/src/backends/mambazip/mamba1/model.rs +++ b/crates/infotheory/src/backends/mambazip/mamba1/model.rs @@ -1339,8 +1339,11 @@ impl Model { cfg, ); let mut grad_log = vec![0.0f32; grad.a.len().min(layer.a.len())]; - for idx in 0..grad_log.len() { - grad_log[idx] = grad.a[idx] * layer.a[idx]; + for (slot, (grad_a, layer_a)) in grad_log + .iter_mut() + .zip(grad.a.as_slice().iter().zip(layer.a.as_slice().iter())) + { + *slot = *grad_a * *layer_a; } apply_adam_vec_update_and_sync_neg_exp( layer.a_log.as_mut_slice(), diff --git a/src/backends/mambazip/mamba1/tensor.rs b/crates/infotheory/src/backends/mambazip/mamba1/tensor.rs similarity index 100% rename from src/backends/mambazip/mamba1/tensor.rs rename to crates/infotheory/src/backends/mambazip/mamba1/tensor.rs diff --git a/src/backends/mambazip/mamba1/weights.rs b/crates/infotheory/src/backends/mambazip/mamba1/weights.rs similarity index 100% rename from src/backends/mambazip/mamba1/weights.rs rename to crates/infotheory/src/backends/mambazip/mamba1/weights.rs diff --git a/src/backends/mambazip/mod.rs b/crates/infotheory/src/backends/mambazip/mod.rs similarity index 88% rename from src/backends/mambazip/mod.rs rename to crates/infotheory/src/backends/mambazip/mod.rs index c7709a05..4d17042e 100644 --- a/src/backends/mambazip/mod.rs +++ b/crates/infotheory/src/backends/mambazip/mod.rs @@ -269,6 +269,10 @@ impl OnlineRuntime { } fn prepare_policy_stream(&mut self, total_symbols: Option) -> Result<()> { + let policy_runtime = match &self.policy { + Some(p) => Some(PolicyRuntime::new(p.compile(total_symbols)?)), + None => None, + }; self.policy_stream_total = total_symbols; self.policy_train_steps = 0; if let Some(tbptt) = self.full_tbptt.as_mut() { @@ -277,10 +281,7 @@ impl OnlineRuntime { tbptt.steps.clear(); tbptt.settings = None; } - self.policy_runtime = match &self.policy { - Some(p) => Some(PolicyRuntime::new(p.compile(total_symbols)?)), - None => None, - }; + self.policy_runtime = policy_runtime; Ok(()) } @@ -495,10 +496,13 @@ fn parse_cfg_positional(csv: &str) -> Result { /// /// Supported formats: /// - `file:/path/to/model.safetensors` +/// - `file:/path/to/model%3Bv1.safetensors` /// - `file:/path/to/model.safetensors;policy:...` /// - `cfg:key=value,...[;policy:...]` /// - positional `cfg` CSV /// - existing model path +/// +/// `file:` methods percent-encode reserved delimiters inside the path segment. pub fn parse_method_spec(method: &str) -> Result { let (base, policy_segment) = split_method_policy_segments(method)?; let parse_policy = |s: &str| llm_policy::parse_policy_segment(s, MAMBA_TRAIN_SCOPES); @@ -509,10 +513,11 @@ pub fn parse_method_spec(method: &str) -> Result { .context("failed to parse mamba policy segment")?; if let Some(path) = base.strip_prefix("file:") { - let p = PathBuf::from(path.trim()); + let p = llm_policy::parse_method_file_path(path.trim()); if p.as_os_str().is_empty() { bail!("empty file path in mamba method"); } + llm_policy::canonical_file_method_string(&p, policy.as_ref())?; if policy.as_ref().and_then(|p| p.load_from.as_ref()).is_some() { bail!("mamba method cannot use policy load_from together with file:"); } @@ -577,6 +582,23 @@ pub fn parse_method_spec(method: &str) -> Result { ); } +/// Convert a parsed method specification back into canonical method syntax. +pub fn canonical_method_string(spec: &MethodSpec) -> Result { + match spec { + MethodSpec::File { path, policy } => { + llm_policy::canonical_file_method_string(path, policy.as_ref()) + } + MethodSpec::Online { cfg, policy } => { + let mut method = cfg_to_method_string(cfg); + if let Some(policy) = policy { + method.push_str(";policy:"); + method.push_str(&policy.canonical()); + } + Ok(method) + } + } +} + /// Framing header for mambazip streams. #[derive(Debug, Clone)] pub struct Header { @@ -729,6 +751,8 @@ pub struct Compressor { #[cfg(test)] mod tests { use super::*; + use serde_json::json; + use std::sync::Arc; fn temp_path(prefix: &str, ext: &str) -> PathBuf { let now = std::time::SystemTime::now() @@ -738,6 +762,119 @@ mod tests { std::env::temp_dir().join(format!("{prefix}_{}_{}.{}", std::process::id(), now, ext)) } + #[test] + fn online_config_to_mamba_config_clamps_invalid_minima() { + let cfg = OnlineConfig { + hidden: 0, + layers: 0, + intermediate: 0, + state: 0, + conv: 0, + dt_rank: 0, + seed: 7, + train_mode: OnlineTrainMode::None, + lr: 0.25, + stride: 0, + }; + let mcfg = cfg.to_mamba_config().expect("validated config"); + assert_eq!(mcfg.vocab_size, VOCAB_SIZE); + assert_eq!(mcfg.hidden_size, 16); + assert_eq!(mcfg.num_layers, 1); + assert_eq!(mcfg.inner_size, 16); + assert_eq!(mcfg.state_size, 1); + assert_eq!(mcfg.conv_kernel, 1); + assert_eq!(mcfg.dt_rank, 1); + } + + #[test] + fn helper_parsers_and_method_rendering_are_canonical() { + assert_eq!( + optimizer_sidecar_path(Path::new("/tmp/model.safetensors")), + PathBuf::from("/tmp/model.opt.safetensors") + ); + assert!(matches!( + parse_train_mode_token("off").expect("off"), + OnlineTrainMode::None + )); + assert!(matches!( + parse_train_mode_token("1").expect("sgd"), + OnlineTrainMode::Sgd + )); + assert!(matches!( + parse_train_mode_token("adam").expect("adam"), + OnlineTrainMode::Adam + )); + assert!( + parse_train_mode_token("mystery") + .expect_err("invalid mode") + .to_string() + .contains("unknown train mode") + ); + + let rendered = cfg_to_method_string(&OnlineConfig { + hidden: 64, + layers: 2, + intermediate: 96, + state: 8, + conv: 3, + dt_rank: 4, + seed: 9, + train_mode: OnlineTrainMode::Sgd, + lr: 0.125, + stride: 0, + }); + assert_eq!( + rendered, + "cfg:hidden=64,layers=2,intermediate=96,state=8,conv=3,dt_rank=4,seed=9,train=sgd,lr=0.125,stride=1" + ); + } + + #[test] + fn policy_helpers_detect_adam_and_full_trace_requirements() { + let infer = llm_policy::parse_policy_segment("schedule=0..10:infer", MAMBA_TRAIN_SCOPES) + .expect("infer policy"); + assert!(!policy_uses_adam(&infer)); + assert!(!policy_needs_full_trace(&infer)); + + let head_adam = llm_policy::parse_policy_segment( + "schedule=0..10:train(scope=head+bias,opt=adam,lr=0.002,stride=1,bptt=1,clip=0,momentum=0.9)", + MAMBA_TRAIN_SCOPES, + ) + .expect("head adam policy"); + assert!(policy_uses_adam(&head_adam)); + assert!(!policy_needs_full_trace(&head_adam)); + + let mixer_proj = llm_policy::parse_policy_segment( + "schedule=0..10:train(scope=mixer_proj,opt=sgd,lr=0.002,stride=1,bptt=1,clip=0,momentum=0.9)", + MAMBA_TRAIN_SCOPES, + ) + .expect("mixer policy"); + assert!(!policy_uses_adam(&mixer_proj)); + assert!(policy_needs_full_trace(&mixer_proj)); + } + + #[test] + fn parse_method_spec_rejects_invalid_cfg_and_file_load_from_combinations() { + assert!( + parse_method_spec("cfg:hidden=64,unknown=1") + .expect_err("unknown key") + .to_string() + .contains("unknown mamba cfg key") + ); + assert!( + parse_method_spec("cfg:64,96,2,sgd,123") + .expect_err("short positional cfg") + .to_string() + .contains("expects 6 or 7 values") + ); + assert!( + parse_method_spec("file:/tmp/model.safetensors;policy:load_from=/tmp/base.safetensors,schedule=0..10:infer") + .expect_err("file + load_from") + .to_string() + .contains("cannot use policy load_from together with file:") + ); + } + #[test] fn parse_method_spec_accepts_cfg_and_positional() { let named = parse_method_spec( @@ -788,6 +925,38 @@ mod tests { } } + #[test] + fn parse_method_spec_accepts_raw_semicolons_in_file_paths() { + match parse_method_spec("file:/tmp/mamba;v1.safetensors").expect("file path parse") { + MethodSpec::File { path, policy } => { + assert_eq!(path, PathBuf::from("/tmp/mamba;v1.safetensors")); + assert!(policy.is_none()); + } + _ => panic!("expected file method"), + } + } + + #[test] + fn canonical_method_string_escapes_delimiter_bearing_file_paths() { + let method = canonical_method_string(&MethodSpec::File { + path: PathBuf::from("/tmp/mamba;policy:model%v1.safetensors"), + policy: None, + }) + .expect("delimiter-bearing file path should canonicalize"); + assert_eq!(method, "file:/tmp/mamba%3Bpolicy:model%25v1.safetensors"); + + match parse_method_spec(&method).expect("canonical method should parse") { + MethodSpec::File { path, policy } => { + assert_eq!( + path, + PathBuf::from("/tmp/mamba;policy:model%v1.safetensors") + ); + assert!(policy.is_none()); + } + _ => panic!("expected file method"), + } + } + #[test] fn canonical_method_omits_policy_when_absent() { let c = Compressor::new_from_method("cfg:hidden=64,layers=1,intermediate=96") @@ -800,6 +969,45 @@ mod tests { ); } + #[test] + fn export_without_online_state_cleans_stale_optimizer_sidecar() { + let cfg = Config { + vocab_size: 256, + hidden_size: 32, + num_layers: 1, + inner_size: 48, + state_size: 8, + conv_kernel: 3, + dt_rank: 4, + layer_norm_eps: 1e-5, + }; + let model = Arc::new(Model::new_random(cfg, 1337).expect("random model")); + let compressor = Compressor::new_from_model(model); + let model_path = temp_path("mamba_plain_export", "safetensors"); + let opt_path = optimizer_sidecar_path(&model_path); + std::fs::write(&opt_path, b"stale optimizer").expect("seed stale optimizer"); + + compressor.export_online(&model_path).expect("export"); + assert!( + !opt_path.exists(), + "plain export should remove stale optimizer sidecar" + ); + + let sidecar: serde_json::Value = serde_json::from_slice( + &std::fs::read(model_path.with_extension("json")).expect("read sidecar"), + ) + .expect("parse sidecar"); + assert_eq!(sidecar["training_mode"], json!("none")); + assert!( + sidecar["method"] + .as_str() + .is_some_and(|method| method.starts_with("file:")) + ); + + std::fs::remove_file(&model_path).ok(); + std::fs::remove_file(model_path.with_extension("json")).ok(); + } + #[test] fn export_reload_roundtrip_reproducible() { let cfg = Config { @@ -834,6 +1042,52 @@ mod tests { let _ = std::fs::remove_file(base.with_extension("json")); } + #[test] + fn loading_sidecar_requires_optimizer_when_exact_resume_is_requested() { + let cfg = Config { + vocab_size: 256, + hidden_size: 32, + num_layers: 1, + inner_size: 48, + state_size: 8, + conv_kernel: 3, + dt_rank: 4, + layer_norm_eps: 1e-5, + }; + let model_path = temp_path("mamba_missing_opt", "safetensors"); + Model::new_random(cfg, 4242) + .expect("random model") + .save_safetensors(&model_path) + .expect("save model"); + let method = canonical_method_string(&MethodSpec::File { + path: model_path.clone(), + policy: None, + }) + .expect("canonical file method"); + let sidecar = json!({ + "version": 1, + "method": method, + "training_mode": "adam", + "tokens_processed": 3, + "has_full_adam": true, + "output_bias": [0.0, 1.0, 2.0], + }); + std::fs::write( + model_path.with_extension("json"), + serde_json::to_vec_pretty(&sidecar).expect("encode sidecar"), + ) + .expect("write sidecar"); + + let err = match Compressor::new(&model_path) { + Ok(_) => panic!("missing optimizer sidecar should fail"), + Err(err) => err, + }; + assert!(err.to_string().contains("missing optimizer sidecar")); + + std::fs::remove_file(&model_path).ok(); + std::fs::remove_file(model_path.with_extension("json")).ok(); + } + #[test] fn online_training_updates_lm_head_weights() { let method = "cfg:hidden=64,layers=2,intermediate=96,state=8,conv=3,dt_rank=4,seed=11,train=sgd,lr=0.01,stride=1;policy:schedule=0..100:train(scope=head+bias,opt=sgd,lr=0.01,stride=1,bptt=1,clip=0,momentum=0.9)"; @@ -1085,12 +1339,17 @@ impl Compressor { /// Create compressor from method string. pub fn new_from_method(method: &str) -> Result { - match parse_method_spec(method)? { + let spec = parse_method_spec(method)?; + Self::new_from_method_spec(&spec) + } + + /// Create compressor from a parsed method specification. + pub fn new_from_method_spec(spec: &MethodSpec) -> Result { + match spec { MethodSpec::File { path, policy } => { - let mut c = Self::new(&path)?; - if let Some(policy) = policy { - let canonical_method = - format!("file:{};policy:{}", path.display(), policy.canonical()); + let mut c = Self::new(path)?; + if let Some(policy) = policy.as_ref() { + let canonical_method = canonical_method_string(spec)?; let hidden = c.model.config().hidden_size; let mut online = c.online.take().unwrap_or_else(|| { OnlineRuntime::new( @@ -1102,7 +1361,7 @@ impl Compressor { ) }); online.canonical_method = canonical_method; - online.policy = Some(policy); + online.policy = Some(policy.clone()); online.needs_full_trace = online .policy .as_ref() @@ -1146,15 +1405,11 @@ impl Compressor { Arc::new(Model::new_random(mcfg, cfg.seed)?) }; let mut c = Self::new_from_model(model); - let mut canonical_method = cfg_to_method_string(&cfg); - if let Some(policy) = policy.as_ref() { - canonical_method.push_str(";policy:"); - canonical_method.push_str(&policy.canonical()); - } + let canonical_method = canonical_method_string(spec)?; c.online = Some(OnlineRuntime::new( - cfg, + cfg.clone(), canonical_method, - policy, + policy.clone(), VOCAB_SIZE, c.model.config().hidden_size, )); @@ -1754,9 +2009,13 @@ impl Compressor { if opt_sidecar.exists() { let _ = fs::remove_file(&opt_sidecar); } + let canonical_method = canonical_method_string(&MethodSpec::File { + path: model_path.to_path_buf(), + policy: None, + })?; json!({ "version": 1, - "method": format!("file:{}", model_path.display()), + "method": canonical_method, "training_mode": "none", "tokens_processed": 0, }) @@ -1793,11 +2052,15 @@ impl Compressor { .collect::>() }); + let default_method = canonical_method_string(&MethodSpec::File { + path: model_path.to_path_buf(), + policy: None, + })?; let method = v .get("method") .and_then(|m| m.as_str()) .map(|s| s.to_string()) - .unwrap_or_else(|| format!("file:{}", model_path.display())); + .unwrap_or(default_method); let has_full_adam = v .get("has_full_adam") .and_then(|x| x.as_bool()) diff --git a/src/backends/match_model.rs b/crates/infotheory/src/backends/match_model.rs similarity index 90% rename from src/backends/match_model.rs rename to crates/infotheory/src/backends/match_model.rs index a2b32075..3a361f95 100644 --- a/src/backends/match_model.rs +++ b/crates/infotheory/src/backends/match_model.rs @@ -21,6 +21,18 @@ pub struct MatchModel { match_len: usize, } +#[derive(Clone, Debug)] +pub(crate) struct MatchModelLifecycleSnapshot { + history: Vec, + frozen_anchor: usize, + pdf: [f64; 256], + cdf: [f64; 257], + valid: bool, + cdf_valid: bool, + predicted: Option, + match_len: usize, +} + impl MatchModel { /// Create a match model with an inclusive stride range `[gap_min+1, gap_max+1]`. pub fn new( @@ -138,6 +150,30 @@ impl MatchModel { self.cdf = uniform_cdf(); } + pub(crate) fn lifecycle_snapshot(&self) -> MatchModelLifecycleSnapshot { + MatchModelLifecycleSnapshot { + history: self.history.clone(), + frozen_anchor: self.frozen_anchor, + pdf: self.pdf, + cdf: self.cdf, + valid: self.valid, + cdf_valid: self.cdf_valid, + predicted: self.predicted, + match_len: self.match_len, + } + } + + pub(crate) fn restore_lifecycle_snapshot(&mut self, snapshot: MatchModelLifecycleSnapshot) { + self.history = snapshot.history; + self.frozen_anchor = snapshot.frozen_anchor; + self.pdf = snapshot.pdf; + self.cdf = snapshot.cdf; + self.valid = snapshot.valid; + self.cdf_valid = snapshot.cdf_valid; + self.predicted = snapshot.predicted; + self.match_len = snapshot.match_len; + } + /// Advance conditioning history without updating learned match tables. pub fn update_history_only(&mut self, symbol: u8) { if self.frozen_anchor == 0 { diff --git a/crates/infotheory/src/backends/mod.rs b/crates/infotheory/src/backends/mod.rs new file mode 100644 index 00000000..95762f7f --- /dev/null +++ b/crates/infotheory/src/backends/mod.rs @@ -0,0 +1,542 @@ +//! Backend discovery helpers and canonical backend naming. +//! +//! This module provides: +//! - canonical backend name resolution for CLI/Python inputs, +//! - feature-aware availability reporting, +//! - exported lists of enabled backend families. + +/// Online probability calibration wrapper for rate predictors. +#[cfg(feature = "backend-calibrated")] +pub mod calibration; +#[cfg(feature = "backend-ctw")] +pub mod ctw; +/// Shared internal fixed-shape GEMV specializations for neural backends. +#[cfg(feature = "backend-rwkv")] +pub(crate) mod fixed_gemv; +/// Shared policy parser/compiler for online LLM backends. +#[cfg(any(feature = "backend-rwkv", feature = "backend-mamba"))] +pub mod llm_policy; +/// Mamba-1 based rate/compression backend. +#[cfg(feature = "backend-mamba")] +pub mod mambazip; +/// Contiguous/sparse local match predictor primitives. +#[cfg(feature = "backend-match")] +pub mod match_model; +/// Particle-latent rate backend. +#[cfg(feature = "backend-particle")] +pub mod particle; +/// Bounded-memory PPMD-style byte model. +#[cfg(feature = "backend-ppmd")] +pub mod ppmd; +#[cfg(feature = "backend-rosa")] +pub mod rosaplus; +/// RWKV7-based rate/compression backend. +#[cfg(feature = "backend-rwkv")] +pub mod rwkvzip; +/// Exact online Sequitur grammar backend with byte-level predictive readout. +#[cfg(feature = "backend-sequitur")] +pub mod sequitur; +/// Sparse/gapped match predictor that wraps [`match_model`]. +#[cfg(feature = "backend-match")] +pub mod sparse_match; +/// Text/repeat context feature extraction for adaptive backends. +pub mod text_context; +#[cfg(feature = "backend-zpaq")] +pub mod zpaq_rate; +use crate::coders::CoderType; +use std::sync::Arc; + +/// Outcome of resolving a backend alias to a canonical backend name. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum BackendAvailability { + /// Backend is compiled and available. + Enabled(&'static str), + /// Backend alias is recognized, but the required Cargo feature is disabled. + Disabled { + /// Canonical backend name. + canonical: &'static str, + /// Cargo feature needed to enable this backend. + feature: &'static str, + }, +} + +/// Method-backed neural backend family shared by CLI-facing integration helpers. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum MethodBackendFamily { + /// Mamba-family method strings and compiled plans. + Mamba, + /// RWKV7-family method strings and compiled plans. + Rwkv7, +} + +/// Canonical names for enabled rate backends in this build. +pub fn available_rate_backends() -> Vec<&'static str> { + available_backend_names(crate::runtime::RATE_BACKEND_REGISTRY) +} + +/// Canonical names for enabled compression backends in this build. +pub fn available_compression_backends() -> Vec<&'static str> { + available_backend_names(crate::runtime::COMPRESSION_BACKEND_REGISTRY) +} + +fn available_backend_names( + registry: &'static [crate::runtime::BackendDescriptor], +) -> Vec<&'static str> { + registry + .iter() + .filter(|descriptor| descriptor.enabled) + .map(|descriptor| descriptor.canonical) + .collect() +} + +fn resolve_backend_name_from_registry( + registry: &'static [crate::runtime::BackendDescriptor], + input: &str, +) -> Option { + const INTERNAL_MISMATCH_FEATURE: &str = "__internal-registry-mismatch__"; + + let descriptor = crate::runtime::find_backend_descriptor_in_registry(registry, input)?; + Some(if descriptor.enabled || descriptor.feature.is_none() { + BackendAvailability::Enabled(descriptor.canonical) + } else { + BackendAvailability::Disabled { + canonical: descriptor.canonical, + feature: descriptor.feature.unwrap_or(INTERNAL_MISMATCH_FEATURE), + } + }) +} + +/// Resolve a user-provided rate backend alias to a canonical backend name. +/// +/// Returns `None` when the alias is unknown, and `BackendAvailability::Disabled` +/// when known but not enabled in the current feature set. +pub fn resolve_rate_backend_name(input: &str) -> Option { + resolve_backend_name_from_registry(crate::runtime::RATE_BACKEND_REGISTRY, input) +} + +/// Resolve a user-provided compression backend alias to a canonical backend name. +/// +/// Returns `None` when the alias is unknown, and `BackendAvailability::Disabled` +/// when known but not enabled in the current feature set. +pub fn resolve_compression_backend_name(input: &str) -> Option { + resolve_backend_name_from_registry(crate::runtime::COMPRESSION_BACKEND_REGISTRY, input) +} + +/// Normalize a compression backend for file roundtrip helpers. +/// +/// File-oriented encode/decode APIs always use framed rate-coded payloads so the +/// backend choice roundtrips predictably across CLI and Python entrypoints. +pub fn normalize_file_roundtrip_backend( + backend: &crate::api::CompressionBackend, +) -> crate::api::CompressionBackend { + match backend { + crate::api::CompressionBackend::Rate { + rate_backend, + coder, + .. + } => crate::api::CompressionBackend::Rate { + rate_backend: rate_backend.clone(), + coder: *coder, + framing: crate::compression::FramingMode::Framed, + }, + _ => backend.clone(), + } +} + +/// Normalize a compiled compression backend for file roundtrip helpers. +pub fn normalize_file_roundtrip_compiled_backend( + backend: &crate::spec::CompiledCompressionBackend, +) -> crate::spec::CompiledCompressionBackend { + match backend.plan() { + crate::spec::core::CompressionBackendPlan::Rate { + rate_backend, + coder, + .. + } => crate::spec::core::compiled_compression_backend_from_plan(Arc::new( + crate::spec::core::CompressionBackendPlan::Rate { + rate_backend: rate_backend.clone(), + coder: *coder, + framing: crate::compression::FramingMode::Framed, + }, + )) + .expect( + "compiled rate-coded backend should remain valid when normalized for file roundtrip", + ), + _ => backend.clone(), + } +} + +fn rate_backend_plan_method_string( + plan: &crate::spec::core::RateBackendPlan, + family: MethodBackendFamily, +) -> Option<&str> { + match (family, plan) { + #[cfg(feature = "backend-rwkv")] + (MethodBackendFamily::Rwkv7, crate::spec::core::RateBackendPlan::Rwkv7 { method, .. }) => { + Some(method.as_str()) + } + #[cfg(feature = "backend-mamba")] + (MethodBackendFamily::Mamba, crate::spec::core::RateBackendPlan::Mamba { method, .. }) => { + Some(method.as_str()) + } + _ => None, + } +} + +/// Extract a method string from a rate backend when it belongs to a method-backed family. +pub fn rate_backend_method_string( + backend: &crate::api::RateBackend, + family: MethodBackendFamily, +) -> Option { + match (family, backend) { + #[cfg(feature = "backend-rwkv")] + (MethodBackendFamily::Rwkv7, crate::api::RateBackend::Rwkv7Method { method }) => { + crate::rwkvzip::canonical_method_string(method).ok() + } + #[cfg(feature = "backend-mamba")] + (MethodBackendFamily::Mamba, crate::api::RateBackend::MambaMethod { method }) => { + crate::mambazip::canonical_method_string(method).ok() + } + _ => None, + } +} + +/// Extract a method string from a compiled rate backend when it belongs to a method-backed family. +pub fn rate_backend_method_string_compiled( + backend: &crate::spec::CompiledRateBackend, + family: MethodBackendFamily, +) -> Option<&str> { + rate_backend_plan_method_string(backend.plan(), family) +} + +/// Extract a method string from a compression backend or its wrapped rate backend. +pub fn compression_backend_method_string( + backend: &crate::api::CompressionBackend, + family: MethodBackendFamily, +) -> Option { + match (family, backend) { + #[cfg(feature = "backend-rwkv")] + (MethodBackendFamily::Rwkv7, crate::api::CompressionBackend::Rwkv7 { method, .. }) => { + crate::rwkvzip::canonical_method_string(method).ok() + } + (_, crate::api::CompressionBackend::Rate { rate_backend, .. }) => { + rate_backend_method_string(rate_backend, family) + } + _ => None, + } +} + +/// Extract a method string from a compiled compression backend or its wrapped rate backend. +pub fn compression_backend_method_string_compiled( + backend: &crate::spec::CompiledCompressionBackend, + family: MethodBackendFamily, +) -> Option<&str> { + match (family, backend.plan()) { + #[cfg(feature = "backend-rwkv")] + ( + MethodBackendFamily::Rwkv7, + crate::spec::core::CompressionBackendPlan::Rwkv7 { method, .. }, + ) => Some(method.as_str()), + (_, crate::spec::core::CompressionBackendPlan::Rate { rate_backend, .. }) => { + rate_backend_plan_method_string(rate_backend.as_ref(), family) + } + _ => None, + } +} + +/// Parse a generic entropy coder alias (`"ac"`/`"rans"`). +pub fn parse_rate_coder(v: &str) -> Option { + match v { + "ac" | "AC" => Some(CoderType::AC), + "rans" | "RANS" | "rANS" => Some(CoderType::RANS), + _ => None, + } +} + +/// Parse an RWKV entropy coder alias (`"ac"`/`"rans"`). +#[cfg(feature = "backend-rwkv")] +pub fn parse_rwkv7_coder(v: &str) -> Option { + parse_rate_coder(v) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn resolve_rate_backend_name_enforces_canonical_names() { + let rosa = if cfg!(feature = "backend-rosa") { + BackendAvailability::Enabled("rosaplus") + } else { + BackendAvailability::Disabled { + canonical: "rosaplus", + feature: "backend-rosa", + } + }; + let fac_ctw = if cfg!(feature = "backend-ctw") { + BackendAvailability::Enabled("fac-ctw") + } else { + BackendAvailability::Disabled { + canonical: "fac-ctw", + feature: "backend-ctw", + } + }; + let mixture = if cfg!(feature = "backend-mixture") { + BackendAvailability::Enabled("mixture") + } else { + BackendAvailability::Disabled { + canonical: "mixture", + feature: "backend-mixture", + } + }; + let sequitur = if cfg!(feature = "backend-sequitur") { + BackendAvailability::Enabled("sequitur") + } else { + BackendAvailability::Disabled { + canonical: "sequitur", + feature: "backend-sequitur", + } + }; + + assert_eq!(resolve_rate_backend_name(" Rosa "), Some(rosa)); + assert_eq!(resolve_rate_backend_name("fac-ctw"), Some(fac_ctw)); + assert_eq!(resolve_rate_backend_name("mixture"), Some(mixture)); + assert_eq!(resolve_rate_backend_name("sequitur"), Some(sequitur)); + + assert_eq!(resolve_rate_backend_name("facctw"), None); + assert_eq!(resolve_rate_backend_name("mix"), None); + assert_eq!(resolve_rate_backend_name("sparsematch"), None); + assert_eq!(resolve_rate_backend_name("ppm"), None); + assert_eq!(resolve_rate_backend_name("cal"), None); + assert_eq!(resolve_rate_backend_name("mamba1"), None); + assert_eq!(resolve_rate_backend_name("rwkv"), None); + assert_eq!(resolve_rate_backend_name("unknown"), None); + } + + #[test] + fn resolve_rate_backend_name_reports_feature_disabled() { + if cfg!(feature = "backend-zpaq") { + assert_eq!( + resolve_rate_backend_name("zpaq"), + Some(BackendAvailability::Enabled("zpaq")) + ); + } else { + assert_eq!( + resolve_rate_backend_name("zpaq"), + Some(BackendAvailability::Disabled { + canonical: "zpaq", + feature: "backend-zpaq", + }) + ); + } + + if cfg!(feature = "backend-rwkv") { + assert_eq!( + resolve_rate_backend_name("rwkv7"), + Some(BackendAvailability::Enabled("rwkv7")) + ); + } else { + assert_eq!( + resolve_rate_backend_name("rwkv7"), + Some(BackendAvailability::Disabled { + canonical: "rwkv7", + feature: "backend-rwkv", + }) + ); + } + + if cfg!(feature = "backend-mamba") { + assert_eq!( + resolve_rate_backend_name("mamba"), + Some(BackendAvailability::Enabled("mamba")) + ); + } else { + assert_eq!( + resolve_rate_backend_name("mamba"), + Some(BackendAvailability::Disabled { + canonical: "mamba", + feature: "backend-mamba", + }) + ); + } + } + + #[test] + fn resolve_compression_backend_name_enforces_canonical_names() { + assert_eq!(resolve_compression_backend_name("unknown"), None); + + if cfg!(feature = "backend-zpaq") { + assert_eq!( + resolve_compression_backend_name("zpaq"), + Some(BackendAvailability::Enabled("zpaq")) + ); + } else { + assert_eq!( + resolve_compression_backend_name("zpaq"), + Some(BackendAvailability::Disabled { + canonical: "zpaq", + feature: "backend-zpaq", + }) + ); + } + + assert_eq!( + resolve_compression_backend_name("rate-ac"), + Some(BackendAvailability::Enabled("rate-ac")) + ); + assert_eq!( + resolve_compression_backend_name("rate-rans"), + Some(BackendAvailability::Enabled("rate-rans")) + ); + + assert_eq!(resolve_compression_backend_name("rate_ac"), None); + assert_eq!(resolve_compression_backend_name("raterans"), None); + assert_eq!(resolve_compression_backend_name("rate_rans"), None); + + if cfg!(feature = "backend-rwkv") { + assert_eq!( + resolve_compression_backend_name("rwkv7"), + Some(BackendAvailability::Enabled("rwkv7")) + ); + } + + assert_eq!(resolve_compression_backend_name("rwkv"), None); + } + + #[test] + fn available_backend_lists_track_runtime_registry() { + assert_eq!( + available_rate_backends(), + crate::runtime::RATE_BACKEND_REGISTRY + .iter() + .filter(|descriptor| descriptor.enabled) + .map(|descriptor| descriptor.canonical) + .collect::>() + ); + assert_eq!( + available_compression_backends(), + crate::runtime::COMPRESSION_BACKEND_REGISTRY + .iter() + .filter(|descriptor| descriptor.enabled) + .map(|descriptor| descriptor.canonical) + .collect::>() + ); + } + + #[test] + fn normalize_file_roundtrip_backend_forces_framed_rate_payloads() { + let rate = crate::api::CompressionBackend::Rate { + rate_backend: crate::api::RateBackend::Ctw { depth: 8 }, + coder: CoderType::RANS, + framing: crate::compression::FramingMode::Raw, + }; + let normalized = normalize_file_roundtrip_backend(&rate); + match normalized { + crate::api::CompressionBackend::Rate { + rate_backend: crate::api::RateBackend::Ctw { depth }, + coder, + framing, + } => { + assert_eq!(depth, 8); + assert_eq!(coder, CoderType::RANS); + assert_eq!(framing, crate::compression::FramingMode::Framed); + } + _ => panic!("expected normalized rate backend"), + } + + let zpaq = crate::api::CompressionBackend::zpaq("5"); + match normalize_file_roundtrip_backend(&zpaq) { + crate::api::CompressionBackend::Zpaq { method, .. } => assert_eq!(method.value(), "5"), + _ => panic!("expected zpaq backend to remain unchanged"), + } + } + + #[test] + fn normalize_file_roundtrip_compiled_backend_forces_framed_rate_payloads() { + let Some(rate_backend) = crate::runtime::first_enabled_default_rate_backend_spec() else { + return; + }; + let rate = crate::api::CompressionBackend::Rate { + rate_backend, + coder: CoderType::RANS, + framing: crate::compression::FramingMode::Raw, + } + .compile() + .expect("compiled rate backend"); + let normalized = normalize_file_roundtrip_compiled_backend(&rate); + match normalized.canonical_spec() { + crate::api::CompressionBackend::Rate { coder, framing, .. } => { + assert_eq!(*coder, CoderType::RANS); + assert_eq!(*framing, crate::compression::FramingMode::Framed); + } + _ => panic!("expected normalized compiled rate backend"), + } + + #[cfg(feature = "backend-zpaq")] + { + let zpaq = crate::api::CompressionBackend::zpaq("5") + .compile() + .expect("compiled zpaq"); + match normalize_file_roundtrip_compiled_backend(&zpaq).canonical_spec() { + crate::api::CompressionBackend::Zpaq { method, .. } => { + assert_eq!(method.value(), "5") + } + _ => panic!("expected compiled zpaq backend to remain unchanged"), + } + } + } + + #[cfg(feature = "backend-rwkv")] + #[test] + fn compiled_method_string_helpers_reuse_compiled_specs() { + let method = "cfg:hidden=64,layers=1,intermediate=64,decay_rank=8,a_rank=8,v_rank=8,g_rank=8,seed=11,train=none,lr=0.0,stride=1;policy:schedule=0..100:infer"; + let rate = crate::api::RateBackend::Rwkv7Method { + method: crate::rwkvzip::parse_method_spec(method).expect("rwkv method spec"), + } + .compile() + .expect("compiled rwkv rate backend"); + let crate::api::RateBackend::Rwkv7Method { + method: canonical_rate, + } = rate.canonical_spec() + else { + panic!("expected canonical rwkv7 rate backend"); + }; + let canonical_rate_string = + crate::rwkvzip::canonical_method_string(canonical_rate).expect("canonical rwkv method"); + assert_eq!( + rate_backend_method_string_compiled(&rate, MethodBackendFamily::Rwkv7), + Some(canonical_rate_string.as_str()) + ); + + let compression = crate::api::CompressionBackend::Rwkv7 { + method: crate::rwkvzip::parse_method_spec(method).expect("rwkv method spec"), + coder: CoderType::AC, + } + .compile() + .expect("compiled rwkv compression backend"); + let crate::api::CompressionBackend::Rwkv7 { + method: canonical_compression, + .. + } = compression.canonical_spec() + else { + panic!("expected canonical rwkv7 compression backend"); + }; + let canonical_compression_string = + crate::rwkvzip::canonical_method_string(canonical_compression) + .expect("canonical rwkv method"); + assert_eq!( + compression_backend_method_string_compiled(&compression, MethodBackendFamily::Rwkv7), + Some(canonical_compression_string.as_str()) + ); + } + + #[cfg(feature = "backend-rwkv")] + #[test] + fn parse_rwkv7_coder_accepts_common_aliases() { + assert_eq!(parse_rwkv7_coder("ac"), Some(CoderType::AC)); + assert_eq!(parse_rwkv7_coder("AC"), Some(CoderType::AC)); + assert_eq!(parse_rwkv7_coder("rans"), Some(CoderType::RANS)); + assert_eq!(parse_rwkv7_coder("RANS"), Some(CoderType::RANS)); + assert_eq!(parse_rwkv7_coder("nope"), None); + } +} diff --git a/src/backends/particle.rs b/crates/infotheory/src/backends/particle.rs similarity index 99% rename from src/backends/particle.rs rename to crates/infotheory/src/backends/particle.rs index cdf5f990..6cc9f701 100644 --- a/src/backends/particle.rs +++ b/crates/infotheory/src/backends/particle.rs @@ -4,7 +4,7 @@ //! cells, selector/rule dynamics, online SGD, Bayesian particle weighting, //! and resample+mutation. -use crate::ParticleSpec; +use crate::api::ParticleSpec; use crate::simd_math::{axpy_wide, dot_wide, logsumexp_wide, max_wide}; use std::collections::VecDeque; diff --git a/crates/infotheory/src/backends/ppmd.rs b/crates/infotheory/src/backends/ppmd.rs new file mode 100644 index 00000000..6a2682a0 --- /dev/null +++ b/crates/infotheory/src/backends/ppmd.rs @@ -0,0 +1,812 @@ +use ahash::AHashMap; +use std::collections::{VecDeque, hash_map::Entry}; + +const PDF_MIN: f64 = crate::mixture::DEFAULT_MIN_PROB; +const FNV_OFFSET: u64 = 0xCBF2_9CE4_8422_2325; +const FNV_PRIME: u64 = 0x1000_0000_01B3; +const INLINE_CONTEXT_COUNTS: usize = 4; + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +struct CountEntry { + symbol: u8, + count: u16, +} + +#[derive(Clone, Debug, Default)] +struct ContextStats { + inline_counts: [CountEntry; INLINE_CONTEXT_COUNTS], + inline_len: u8, + spill_counts: Option>, + total: u32, +} + +impl ContextStats { + fn distinct_len(&self) -> usize { + self.inline_len as usize + + self + .spill_counts + .as_ref() + .map(|entries| entries.len()) + .unwrap_or(0) + } + + #[cfg(any(test, feature = "research-tooling"))] + fn spill_bytes(&self) -> usize { + self.spill_counts + .as_ref() + .map(|entries| { + entries + .len() + .saturating_mul(std::mem::size_of::()) + }) + .unwrap_or(0) + } + + fn for_each(&self, mut f: impl FnMut(CountEntry)) { + for idx in 0..(self.inline_len as usize) { + f(self.inline_counts[idx]); + } + if let Some(spill_counts) = &self.spill_counts { + for &entry in spill_counts.iter() { + f(entry); + } + } + } + + fn find_mut(&mut self, symbol: u8) -> Option<&mut u16> { + if let Some(entry) = self.inline_counts[..(self.inline_len as usize)] + .iter_mut() + .find(|entry| entry.symbol == symbol) + { + return Some(&mut entry.count); + } + if let Some(spill_counts) = self.spill_counts.as_mut() + && let Some(entry) = spill_counts.iter_mut().find(|entry| entry.symbol == symbol) + { + return Some(&mut entry.count); + } + None + } + + fn push_new(&mut self, symbol: u8) { + let entry = CountEntry { symbol, count: 1 }; + if (self.inline_len as usize) < INLINE_CONTEXT_COUNTS { + self.inline_counts[self.inline_len as usize] = entry; + self.inline_len += 1; + return; + } + + let mut spill = Vec::with_capacity( + self.spill_counts + .as_ref() + .map(|entries| entries.len()) + .unwrap_or(0) + .saturating_add(1), + ); + if let Some(existing) = self.spill_counts.take() { + spill.extend_from_slice(existing.as_ref()); + } + spill.push(entry); + self.spill_counts = Some(spill.into_boxed_slice()); + } + + fn observe(&mut self, symbol: u8) { + if let Some(count) = self.find_mut(symbol) { + *count = count.saturating_add(1); + } else { + self.push_new(symbol); + } + self.total = self.total.saturating_add(1); + if self.total > 4096 { + self.rescale(); + } + } + + fn rescale(&mut self) { + self.total = 0; + for idx in 0..(self.inline_len as usize) { + let count = &mut self.inline_counts[idx].count; + *count = (*count).div_ceil(2).max(1); + self.total += *count as u32; + } + if let Some(spill_counts) = self.spill_counts.as_mut() { + for entry in spill_counts.iter_mut() { + entry.count = entry.count.div_ceil(2).max(1); + self.total += entry.count as u32; + } + } + } +} + +#[cfg(any(test, feature = "research-tooling"))] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct PpmdMemoryUsage { + pub context_table_bytes: usize, + pub context_spill_bytes: usize, + pub queue_bytes: usize, + pub history_bytes: usize, + pub suffix_key_bytes: usize, + pub pdf_cache_bytes: usize, +} + +#[cfg(any(test, feature = "research-tooling"))] +impl PpmdMemoryUsage { + pub(crate) fn total_bytes(self) -> usize { + self.context_table_bytes + + self.context_spill_bytes + + self.queue_bytes + + self.history_bytes + + self.suffix_key_bytes + + self.pdf_cache_bytes + } +} + +#[derive(Clone, Debug)] +/// Bounded-memory PPMD-inspired byte model with interpolation across orders. +pub struct PpmdModel { + order: usize, + max_contexts: usize, + contexts: Vec>, + queue: VecDeque<(usize, u64)>, + history: Vec, + suffix_keys: Vec, + pdf: [f64; 256], + cdf: [f64; 257], + valid: bool, + cdf_valid: bool, +} + +#[derive(Clone, Debug)] +pub(crate) struct PpmdLifecycleSnapshot { + history: Vec, + suffix_keys: Vec, + pdf: [f64; 256], + cdf: [f64; 257], + valid: bool, + cdf_valid: bool, +} + +impl PpmdModel { + /// Create a model with maximum `order` and approximate memory budget in MiB. + pub fn new(order: usize, memory_mb: usize) -> Self { + let order = order.max(1); + let max_contexts = (memory_mb.max(1) * 1024 * 1024) / 96; + Self { + order, + max_contexts: max_contexts.max(1024), + contexts: (0..=order).map(|_| AHashMap::new()).collect(), + queue: VecDeque::new(), + history: Vec::new(), + suffix_keys: vec![0; order + 1], + pdf: [1.0 / 256.0; 256], + cdf: uniform_cdf(), + valid: false, + cdf_valid: false, + } + } + + /// Fill `out` with the current normalized byte PDF. + pub fn fill_pdf(&mut self, out: &mut [f64; 256]) { + self.ensure_pdf_inner(false); + out.copy_from_slice(&self.pdf); + } + + /// Borrow the current normalized byte PDF. + pub fn pdf(&mut self) -> &[f64; 256] { + self.ensure_pdf_inner(false); + &self.pdf + } + + /// Borrow the cumulative distribution derived from the current PDF. + pub fn cdf(&mut self) -> &[f64; 257] { + self.ensure_pdf_inner(true); + &self.cdf + } + + pub(crate) fn symbol_prob(&mut self, symbol: u8) -> f64 { + self.ensure_pdf_inner(false); + self.pdf[symbol as usize] + } + + #[cfg(test)] + fn interval_mass(&mut self, lo: usize, hi: usize) -> f64 { + if lo >= hi { + return 0.0; + } + let lo = lo.min(256); + let hi = hi.min(256); + if lo >= hi { + return 0.0; + } + if self.cdf_valid { + return self.cdf[hi] - self.cdf[lo]; + } + if self.valid { + let mut acc_lo = 0.0; + let mut acc_hi = 0.0; + for i in 0..hi { + acc_hi += self.pdf[i]; + if i + 1 == lo { + acc_lo = acc_hi; + } + } + return acc_hi - acc_lo; + } + self.ensure_pdf_inner(true); + self.cdf[hi] - self.cdf[lo] + } + + /// Return `ln(max(P(symbol), min_prob))`. + pub fn log_prob(&mut self, symbol: u8, min_prob: f64) -> f64 { + self.symbol_prob(symbol).max(min_prob).ln() + } + + /// Observe one symbol and update all active contexts up to model order. + pub fn update(&mut self, symbol: u8) { + let max_order = self.order.min(self.history.len()); + for ord in 0..=max_order { + let key = self.context_key(ord); + let map = &mut self.contexts[ord]; + match map.entry(key) { + Entry::Occupied(mut entry) => { + entry.get_mut().observe(symbol); + } + Entry::Vacant(entry) => { + self.queue.push_back((ord, key)); + entry.insert(ContextStats::default()).observe(symbol); + } + } + } + self.prune(); + self.append_history_symbol(symbol); + self.valid = false; + self.cdf_valid = false; + } + + /// Reset only the conditioning history while preserving fitted contexts. + pub fn reset_history(&mut self) { + self.history.clear(); + self.suffix_keys.fill(0); + self.valid = false; + self.cdf_valid = false; + self.pdf.fill(1.0 / 256.0); + self.cdf = uniform_cdf(); + } + + pub(crate) fn lifecycle_snapshot(&self) -> PpmdLifecycleSnapshot { + PpmdLifecycleSnapshot { + history: self.history.clone(), + suffix_keys: self.suffix_keys.clone(), + pdf: self.pdf, + cdf: self.cdf, + valid: self.valid, + cdf_valid: self.cdf_valid, + } + } + + pub(crate) fn restore_lifecycle_snapshot(&mut self, snapshot: PpmdLifecycleSnapshot) { + self.history = snapshot.history; + self.suffix_keys = snapshot.suffix_keys; + self.pdf = snapshot.pdf; + self.cdf = snapshot.cdf; + self.valid = snapshot.valid; + self.cdf_valid = snapshot.cdf_valid; + } + + /// Advance conditioning history without updating fitted context counts. + pub fn update_history_only(&mut self, symbol: u8) { + self.append_history_symbol(symbol); + self.valid = false; + self.cdf_valid = false; + } + + #[cfg(any(test, feature = "research-tooling"))] + pub(crate) fn memory_usage_breakdown(&self) -> PpmdMemoryUsage { + let context_table_bytes: usize = self + .contexts + .iter() + .map(|map| { + map.capacity().saturating_mul( + std::mem::size_of::<(u64, ContextStats)>() + std::mem::size_of::(), + ) + }) + .sum(); + let context_spill_bytes: usize = self + .contexts + .iter() + .flat_map(|map| map.values()) + .map(ContextStats::spill_bytes) + .sum(); + PpmdMemoryUsage { + context_table_bytes, + context_spill_bytes, + queue_bytes: self + .queue + .capacity() + .saturating_mul(std::mem::size_of::<(usize, u64)>()), + history_bytes: self + .history + .capacity() + .saturating_mul(std::mem::size_of::()), + suffix_key_bytes: self + .suffix_keys + .capacity() + .saturating_mul(std::mem::size_of::()), + pdf_cache_bytes: std::mem::size_of::<[f64; 256]>() + std::mem::size_of::<[f64; 257]>(), + } + } + + #[cfg(any(test, feature = "research-tooling"))] + pub(crate) fn estimated_size_bytes(&self) -> usize { + self.memory_usage_breakdown().total_bytes() + } + + fn ensure_pdf_inner(&mut self, want_cdf: bool) { + if self.valid { + if want_cdf && !self.cdf_valid { + build_cdf_from_pdf(&self.pdf, &mut self.cdf); + self.cdf_valid = true; + } + return; + } + let mut lower = [1.0 / 256.0; 256]; + let max_order = self.order.min(self.history.len()); + for ord in 0..=max_order { + let key = self.context_key(ord); + if let Some(ctx) = self.contexts[ord].get(&key) { + interpolate_context_in_place(ctx, &mut lower); + } + } + self.pdf.copy_from_slice(&lower); + normalize_pdf_and_maybe_cdf( + &mut self.pdf, + if want_cdf { Some(&mut self.cdf) } else { None }, + ); + self.valid = true; + self.cdf_valid = want_cdf; + } + + fn prune(&mut self) { + let mut total_contexts: usize = self.contexts.iter().map(|m| m.len()).sum(); + while total_contexts > self.max_contexts { + let Some((ord, key)) = self.queue.pop_front() else { + break; + }; + if self.contexts[ord].remove(&key).is_some() { + total_contexts -= 1; + } + } + } + + fn context_key(&self, ord: usize) -> u64 { + if ord == 0 { + return 0; + } + debug_assert!(ord <= self.order); + debug_assert!(ord <= self.history.len()); + self.suffix_keys[ord] + } + + #[cfg(test)] + fn sparse_query_state(&self) -> SparseQueryState { + let mut state = SparseQueryState::new(); + let max_order = self.order.min(self.history.len()); + for ord in 0..=max_order { + let key = self.context_key(ord); + if let Some(ctx) = self.contexts[ord].get(&key) { + state.interpolate_context(ctx); + } + } + state + } + + #[cfg(test)] + fn flooring_diagnostics(&self) -> FlooringDiagnostics { + let state = self.sparse_query_state(); + let mut min_unfloored_probability = f64::INFINITY; + let mut floored_count = 0usize; + let mut mass_added_by_flooring = 0.0; + for symbol in 0..256usize { + let value = state.raw_value(symbol); + min_unfloored_probability = min_unfloored_probability.min(value); + if !value.is_finite() || value < PDF_MIN { + floored_count += 1; + mass_added_by_flooring += PDF_MIN - if value.is_finite() { value } else { 0.0 }; + } + } + let normalization_sum = state.normalization_sum(); + let normalization_factor_after_flooring = + if normalization_sum.is_finite() && normalization_sum > 0.0 { + 1.0 / normalization_sum + } else { + 1.0 + }; + FlooringDiagnostics { + min_unfloored_probability, + floored_count, + mass_added_by_flooring, + normalization_factor_after_flooring, + } + } + + fn append_history_symbol(&mut self, symbol: u8) { + let new_max_order = self.order.min(self.history.len() + 1); + for ord in (1..=new_max_order).rev() { + let prev_hash = if ord == 1 { + FNV_OFFSET + } else { + self.suffix_keys[ord - 1] + }; + self.suffix_keys[ord] = extend_hash(prev_hash, symbol); + } + self.suffix_keys[0] = 0; + self.history.push(symbol); + } +} + +#[cfg(test)] +struct SparseQueryState { + base: f64, + values: [f64; 256], + touched: [bool; 256], + touched_symbols: [u8; 256], + touched_len: usize, +} + +#[cfg(test)] +struct FlooringDiagnostics { + min_unfloored_probability: f64, + floored_count: usize, + mass_added_by_flooring: f64, + normalization_factor_after_flooring: f64, +} + +#[cfg(test)] +impl SparseQueryState { + fn new() -> Self { + Self { + base: 1.0 / 256.0, + values: [0.0; 256], + touched: [false; 256], + touched_symbols: [0; 256], + touched_len: 0, + } + } + + fn interpolate_context(&mut self, ctx: &ContextStats) { + let distinct = ctx.distinct_len() as f64; + let denom = (ctx.total as f64) + distinct + 1.0; + let escape = (distinct + 1.0) / denom; + self.base *= escape; + for idx in 0..self.touched_len { + let symbol = self.touched_symbols[idx] as usize; + self.values[symbol] *= escape; + } + ctx.for_each(|entry| { + let idx = entry.symbol as usize; + if !self.touched[idx] { + self.touched[idx] = true; + self.touched_symbols[self.touched_len] = entry.symbol; + self.touched_len += 1; + self.values[idx] = self.base; + } + self.values[idx] += (entry.count as f64) / denom; + }); + } + + fn raw_value(&self, symbol: usize) -> f64 { + if self.touched[symbol] { + self.values[symbol] + } else { + self.base + } + } + + fn floored_value(&self, symbol: usize) -> f64 { + let value = self.raw_value(symbol); + if value.is_finite() { + value.max(PDF_MIN) + } else { + PDF_MIN + } + } + + fn normalization_sum(&self) -> f64 { + let mut sum = 0.0; + for symbol in 0..256usize { + sum += self.floored_value(symbol); + } + sum + } +} + +fn interpolate_context_in_place(ctx: &ContextStats, lower: &mut [f64; 256]) { + let distinct = ctx.distinct_len() as f64; + let denom = (ctx.total as f64) + distinct + 1.0; + let escape = (distinct + 1.0) / denom; + for p in lower.iter_mut() { + *p *= escape; + } + ctx.for_each(|entry| { + lower[entry.symbol as usize] += (entry.count as f64) / denom; + }); +} + +fn normalize_pdf_and_maybe_cdf(pdf: &mut [f64; 256], cdf: Option<&mut [f64; 257]>) { + let mut sum = 0.0; + for p in pdf.iter_mut() { + *p = if p.is_finite() { + (*p).max(PDF_MIN) + } else { + PDF_MIN + }; + sum += *p; + } + if !(sum.is_finite()) || sum <= 0.0 { + let u = 1.0 / 256.0; + pdf.fill(u); + if let Some(cdf) = cdf { + *cdf = uniform_cdf(); + } + return; + } + let inv = 1.0 / sum; + if let Some(cdf) = cdf { + cdf[0] = 0.0; + let mut acc = 0.0; + for i in 0..256 { + pdf[i] *= inv; + acc += pdf[i]; + cdf[i + 1] = acc; + } + } else { + for p in pdf.iter_mut() { + *p *= inv; + } + } +} + +#[inline] +fn uniform_cdf() -> [f64; 257] { + let mut cdf = [0.0; 257]; + let inv = 1.0 / 256.0; + for (i, slot) in cdf.iter_mut().enumerate() { + *slot = (i as f64) * inv; + } + cdf +} + +#[inline] +fn build_cdf_from_pdf(pdf: &[f64; 256], cdf: &mut [f64; 257]) { + cdf[0] = 0.0; + let mut acc = 0.0; + for i in 0..256 { + acc += pdf[i]; + cdf[i + 1] = acc; + } +} + +#[inline] +fn extend_hash(hash: u64, byte: u8) -> u64 { + (hash ^ (byte as u64)).wrapping_mul(FNV_PRIME) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn hash_bytes(bytes: &[u8]) -> u64 { + let mut h = FNV_OFFSET; + for &b in bytes { + h = extend_hash(h, b); + } + h + } + + fn reference_context_key(history: &[u8], ord: usize) -> u64 { + if ord == 0 { + 0 + } else { + hash_bytes(&history[history.len() - ord..]) + } + } + + fn assert_suffix_keys_match_reference(model: &PpmdModel) { + let max_order = model.order.min(model.history.len()); + for ord in 0..=max_order { + assert_eq!( + model.context_key(ord), + reference_context_key(&model.history, ord) + ); + } + } + + fn reference_interpolate_context(ctx: &ContextStats, lower: &[f64; 256]) -> [f64; 256] { + let distinct = ctx.distinct_len() as f64; + let denom = (ctx.total as f64) + distinct + 1.0; + let escape = (distinct + 1.0) / denom; + let mut out = [0.0; 256]; + for i in 0..256 { + out[i] = lower[i] * escape; + } + ctx.for_each(|entry| { + out[entry.symbol as usize] += (entry.count as f64) / denom; + }); + out + } + + fn train_query_regression_model() -> PpmdModel { + let mut model = PpmdModel::new(12, 1); + let data = b"abracadabra abracadabra mississippi banana bandana ppmd query exactness"; + for &byte in data { + model.update(byte); + } + model.reset_history(); + for &byte in b"abracadabra mississippi" { + model.update_history_only(byte); + } + model + } + + #[test] + fn rolling_suffix_keys_match_recomputed_suffix_hashes() { + let mut model = PpmdModel::new(12, 1); + let bytes = [ + 0, 1, 2, 3, 255, 128, 64, 32, 16, 8, 4, 2, 1, 0, 251, 17, 99, 100, + ]; + assert_suffix_keys_match_reference(&model); + for &byte in &bytes { + model.update(byte); + assert_suffix_keys_match_reference(&model); + } + + let mut cloned = model.clone(); + assert_suffix_keys_match_reference(&cloned); + for &byte in &[7, 6, 5, 4, 3, 2, 1] { + cloned.update_history_only(byte); + assert_suffix_keys_match_reference(&cloned); + } + + cloned.reset_history(); + assert_suffix_keys_match_reference(&cloned); + assert!(cloned.suffix_keys.iter().all(|&key| key == 0)); + + for &byte in &[42, 43, 44, 45] { + cloned.update_history_only(byte); + assert_suffix_keys_match_reference(&cloned); + } + } + + #[test] + fn in_place_interpolation_matches_out_of_place_reference() { + let mut ctx = ContextStats::default(); + for &symbol in &[0, 1, 1, 2, 3, 3, 3, 128, 255, 255] { + ctx.observe(symbol); + } + + let mut lower = [0.0; 256]; + for (i, p) in lower.iter_mut().enumerate() { + *p = ((i + 1) as f64) / 32896.0; + } + + let expected = reference_interpolate_context(&ctx, &lower); + interpolate_context_in_place(&ctx, &mut lower); + for (expected, actual) in expected.iter().zip(lower.iter()) { + assert_eq!(expected.to_bits(), actual.to_bits()); + } + } + + #[test] + fn exact_symbol_queries_match_dense_pdf() { + let query_model = train_query_regression_model(); + let mut dense = query_model.clone(); + let pdf = *dense.pdf(); + + for symbol in 0..=255u8 { + let mut queried = query_model.clone(); + let got = queried.symbol_prob(symbol); + let expected = pdf[symbol as usize]; + assert_eq!( + expected.to_bits(), + got.to_bits(), + "symbol={symbol} expected={expected:?} got={got:?}" + ); + } + } + + #[test] + fn exact_interval_queries_match_dense_cdf_differences() { + let query_model = train_query_regression_model(); + let mut dense = query_model.clone(); + let cdf = *dense.cdf(); + let ranges = [ + (0usize, 1usize), + (0, 2), + (0, 128), + (0, 256), + (1, 2), + (3, 17), + (17, 42), + (42, 128), + (64, 192), + (127, 128), + (128, 256), + (255, 256), + ]; + + for &(lo, hi) in &ranges { + let mut queried = query_model.clone(); + let expected = cdf[hi] - cdf[lo]; + let got = queried.interval_mass(lo, hi); + assert_eq!( + expected.to_bits(), + got.to_bits(), + "range={lo}..{hi} expected={expected:?} got={got:?}" + ); + } + } + + #[test] + fn exact_queries_match_dense_pdf_when_probability_flooring_is_active() { + let mut query_model = PpmdModel::new(12, 1); + for _ in 0..512usize { + query_model.update(b'a'); + } + + let diagnostics = query_model.flooring_diagnostics(); + assert!(diagnostics.min_unfloored_probability < PDF_MIN); + assert!(diagnostics.floored_count > 0); + assert!(diagnostics.mass_added_by_flooring > 0.0); + assert!(diagnostics.normalization_factor_after_flooring.is_finite()); + + let mut dense = query_model.clone(); + let pdf = *dense.pdf(); + for symbol in [0u8, b'a', b'b', 127, 255] { + let mut queried = query_model.clone(); + let got = queried.symbol_prob(symbol); + let expected = pdf[symbol as usize]; + assert_eq!( + expected.to_bits(), + got.to_bits(), + "symbol={symbol} expected={expected:?} got={got:?}" + ); + } + } + + #[test] + fn context_stats_spills_after_inline_capacity_without_changing_counts() { + let mut ctx = ContextStats::default(); + for &symbol in &[1u8, 2, 3, 4, 5, 5, 4, 3, 2, 1] { + ctx.observe(symbol); + } + + assert_eq!(ctx.inline_len as usize, INLINE_CONTEXT_COUNTS); + assert!(ctx.spill_counts.is_some()); + assert_eq!(ctx.distinct_len(), 5); + + let mut seen = [0u16; 256]; + ctx.for_each(|entry| { + seen[entry.symbol as usize] = entry.count; + }); + assert_eq!(seen[1], 2); + assert_eq!(seen[2], 2); + assert_eq!(seen[3], 2); + assert_eq!(seen[4], 2); + assert_eq!(seen[5], 2); + } + + #[test] + fn ppmd_memory_usage_breakdown_sums_to_estimated_total() { + let mut model = PpmdModel::new(12, 1); + for &byte in + b"abracadabra abracadabra mississippi banana bandana ppmd memory validation payload" + { + model.update(byte); + } + + let usage = model.memory_usage_breakdown(); + assert_eq!(model.estimated_size_bytes(), usage.total_bytes()); + assert!(usage.context_table_bytes > 0); + assert!(usage.pdf_cache_bytes > 0); + } +} diff --git a/src/backends/rosaplus.rs b/crates/infotheory/src/backends/rosaplus.rs similarity index 81% rename from src/backends/rosaplus.rs rename to crates/infotheory/src/backends/rosaplus.rs index bb427e3b..0085207c 100644 --- a/src/backends/rosaplus.rs +++ b/crates/infotheory/src/backends/rosaplus.rs @@ -38,6 +38,8 @@ const LM_PACKED_CNT_MAX: u16 = u16::MAX; // This crate is used byte-wise by infotheory; for fast incremental conditional updates we // support an optional fixed 256-byte alphabet LM build/update path. const BYTE_ALPHA_N: usize = 256; +const ROSA_STREAM_HINT_CAP_SYMBOLS: usize = 1 << 16; +const ROSA_GROWTH_MIN_CHUNK_SYMBOLS: usize = 4096; #[inline(always)] fn state_ix(idx: usize) -> SamStateIx { @@ -198,6 +200,215 @@ struct SamEdge { next: SamEdgeIx, } +#[derive(Clone, Debug, PartialEq, Eq)] +enum SamText { + Byte(Vec), + Codepoint(Vec), +} + +impl Default for SamText { + fn default() -> Self { + Self::with_expected(0) + } +} + +impl SamText { + fn with_expected(expected_symbols: usize) -> Self { + let cap = if expected_symbols > 0 { + expected_symbols + 16 + } else { + 1024 + }; + Self::Byte(Vec::with_capacity(cap)) + } + + #[inline(always)] + fn len(&self) -> usize { + match self { + Self::Byte(bytes) => bytes.len(), + Self::Codepoint(cps) => cps.len(), + } + } + + #[inline(always)] + fn is_empty(&self) -> bool { + self.len() == 0 + } + + #[inline(always)] + fn capacity(&self) -> usize { + match self { + Self::Byte(bytes) => bytes.capacity(), + Self::Codepoint(cps) => cps.capacity(), + } + } + + #[inline(always)] + fn reserve_exact(&mut self, additional: usize) { + match self { + Self::Byte(bytes) => bytes.reserve_exact(additional), + Self::Codepoint(cps) => cps.reserve_exact(additional), + } + } + + #[inline(always)] + fn truncate(&mut self, new_len: usize) { + match self { + Self::Byte(bytes) => bytes.truncate(new_len), + Self::Codepoint(cps) => cps.truncate(new_len), + } + } + + #[inline(always)] + fn get_u32(&self, idx: usize) -> u32 { + match self { + Self::Byte(bytes) => bytes[idx] as u32, + Self::Codepoint(cps) => cps[idx], + } + } + + #[inline(always)] + fn push_u32(&mut self, ch: u32) { + match self { + Self::Byte(bytes) if ch < BYTE_ALPHA_N as u32 => { + bytes.push(ch as u8); + } + Self::Byte(bytes) => { + let mut promoted = + Vec::with_capacity(bytes.capacity().max(bytes.len().saturating_add(1))); + promoted.extend(bytes.iter().copied().map(u32::from)); + promoted.push(ch); + *self = Self::Codepoint(promoted); + } + Self::Codepoint(cps) => { + cps.push(ch); + } + } + } + + fn extend_from_u32_slice(&mut self, xs: &[u32]) { + for &x in xs { + self.push_u32(x); + } + } + + fn clone_as_u32_vec(&self) -> Vec { + match self { + Self::Byte(bytes) => bytes.iter().copied().map(u32::from).collect(), + Self::Codepoint(cps) => cps.clone(), + } + } + + fn allocated_bytes(&self) -> usize { + match self { + Self::Byte(bytes) => bytes.capacity().saturating_mul(std::mem::size_of::()), + Self::Codepoint(cps) => cps.capacity().saturating_mul(std::mem::size_of::()), + } + } +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +struct BoundaryBits { + words: Vec, + len: usize, +} + +impl BoundaryBits { + fn with_expected(expected_symbols: usize) -> Self { + let mut out = Self::default(); + if expected_symbols > 0 { + out.reserve_exact(expected_symbols + 16); + } else { + out.reserve_exact(1024); + } + out + } + + #[inline(always)] + fn len(&self) -> usize { + self.len + } + + #[inline(always)] + fn reserve_exact(&mut self, additional: usize) { + let needed_bits = self.len.saturating_add(additional); + let needed_words = needed_bits.div_ceil(64); + let current_words = self.words.len(); + if needed_words > current_words { + self.words.reserve_exact(needed_words - current_words); + } + } + + #[inline(always)] + fn push_clear(&mut self) { + let bit = self.len; + let word = bit / 64; + if word == self.words.len() { + self.words.push(0); + } + self.len += 1; + } + + #[inline(always)] + fn get(&self, idx: usize) -> bool { + if idx >= self.len { + return false; + } + let word = idx / 64; + let bit = idx % 64; + ((self.words[word] >> bit) & 1) != 0 + } + + #[inline(always)] + fn set(&mut self, idx: usize) { + debug_assert!(idx < self.len); + let word = idx / 64; + let bit = idx % 64; + self.words[word] |= 1u64 << bit; + } + + fn truncate(&mut self, new_len: usize) { + if new_len >= self.len { + return; + } + self.len = new_len; + let keep_words = new_len.div_ceil(64); + self.words.truncate(keep_words); + if let Some(last) = self.words.last_mut() { + let bits = new_len % 64; + if bits != 0 { + *last &= (1u64 << bits) - 1; + } + } + } + + fn to_byte_vec(&self) -> Vec { + let mut out = vec![0u8; self.len]; + for (idx, slot) in out.iter_mut().enumerate() { + *slot = u8::from(self.get(idx)); + } + out + } + + fn load_from_bytes(&mut self, xs: &[u8]) { + self.words.clear(); + self.len = 0; + self.reserve_exact(xs.len()); + for &x in xs { + self.push_clear(); + if x != 0 { + self.set(self.len - 1); + } + } + } + + fn allocated_bytes(&self) -> usize { + self.words + .capacity() + .saturating_mul(std::mem::size_of::()) + } +} + #[derive(Clone)] struct Sam { st: Vec, @@ -205,9 +416,9 @@ struct Sam { last: SamStateIx, root_to: [SamStateIx; BYTE_ALPHA_N], - text: Vec, + text: SamText, text_states: Vec, - boundary_after: Vec, + boundary_after: BoundaryBits, } impl Default for Sam { @@ -223,9 +434,9 @@ impl Sam { ed: Vec::new(), last: 0, root_to: [SAM_STATE_NONE; BYTE_ALPHA_N], - text: Vec::new(), + text: SamText::with_expected(expected_chars), text_states: Vec::new(), - boundary_after: Vec::new(), + boundary_after: BoundaryBits::with_expected(expected_chars), }; let st_cap = if expected_chars > 0 { @@ -238,16 +449,9 @@ impl Sam { } else { 2048 }; - let text_cap = if expected_chars > 0 { - expected_chars + 16 - } else { - 1024 - }; s.st.reserve(st_cap); s.ed.reserve(ed_cap); - s.text.reserve(text_cap); - s.text_states.reserve(text_cap); - s.boundary_after.reserve(text_cap); + s.text_states.reserve(s.text.capacity().max(1)); let root = SamState { link: SAM_STATE_NONE, @@ -267,11 +471,54 @@ impl Sam { if additional == 0 { return; } + self.ensure_append_capacity(additional.min(ROSA_STREAM_HINT_CAP_SYMBOLS)); + } + + #[inline(always)] + fn len(&self) -> usize { + self.text.len() + } + + #[inline(always)] + fn is_empty(&self) -> bool { + self.text.is_empty() + } + + #[inline(always)] + fn text_at(&self, idx: usize) -> u32 { + self.text.get_u32(idx) + } + + fn text_u32_vec(&self) -> Vec { + self.text.clone_as_u32_vec() + } + + fn set_text_from_u32_slice(&mut self, xs: &[u32]) { + self.text = SamText::with_expected(xs.len()); + self.text.extend_from_u32_slice(xs); + } + + fn boundary_bytes_vec(&self) -> Vec { + self.boundary_after.to_byte_vec() + } + + #[inline(always)] + fn ensure_append_capacity(&mut self, additional: usize) { + if additional == 0 { + return; + } + let current = self.text.capacity(); + let needed = self.len().saturating_add(additional); + if needed <= current { + return; + } + let missing = needed - current; + let chunk = missing.max((current / 2).max(ROSA_GROWTH_MIN_CHUNK_SYMBOLS)); self.st - .reserve_exact(additional.saturating_mul(2).saturating_add(16)); + .reserve_exact(chunk.saturating_mul(2).saturating_add(16)); self.ed - .reserve_exact(additional.saturating_mul(3).saturating_add(16)); - let text_extra = additional.saturating_add(16); + .reserve_exact(chunk.saturating_mul(3).saturating_add(16)); + let text_extra = chunk.saturating_add(16); self.text.reserve_exact(text_extra); self.text_states.reserve_exact(text_extra); self.boundary_after.reserve_exact(text_extra); @@ -391,9 +638,10 @@ impl Sam { } fn feed(&mut self, ch: u32) { - let i = self.text.len() as i32; - self.text.push(ch); - self.boundary_after.push(0); + self.ensure_append_capacity(1); + let i = self.len() as i32; + self.text.push_u32(ch); + self.boundary_after.push_clear(); let g = self.last; let r = state_ix(self.st.len()); @@ -450,9 +698,9 @@ impl Sam { } fn mark_boundary(&mut self) { - if !self.text.is_empty() { - let i = self.text.len() - 1; - self.boundary_after[i] = 1; + if !self.is_empty() { + let i = self.len() - 1; + self.boundary_after.set(i); } self.last = 0; } @@ -520,15 +768,12 @@ impl Sam { let st = unsafe { self.st.get_unchecked(state_usize(u)) }; let i = st.endpos; let j = i + 1; - if st.len > 0 && j >= 0 && (j as usize) < self.text.len() { - if i >= 0 - && (i as usize) < self.boundary_after.len() - && self.boundary_after[i as usize] != 0 - { + if st.len > 0 && j >= 0 && (j as usize) < self.len() { + if i >= 0 && self.boundary_after.get(i as usize) { u = st.link; continue; } - return Some(self.text[j as usize]); + return Some(self.text_at(j as usize)); } u = st.link; } @@ -539,7 +784,7 @@ impl Sam { fn begin_tx(&self) -> SamTx { SamTx { old_last: self.last, - old_text_len: self.text.len(), + old_text_len: self.len(), old_text_states_len: self.text_states.len(), old_boundary_len: self.boundary_after.len(), old_st_len: self.st.len(), @@ -666,9 +911,10 @@ impl Sam { } fn feed_tx(&mut self, tx: &mut SamTx, ch: u32) { - let i = self.text.len() as i32; - self.text.push(ch); - self.boundary_after.push(0); + self.ensure_append_capacity(1); + let i = self.len() as i32; + self.text.push_u32(ch); + self.boundary_after.push_clear(); let g = self.last; let r = state_ix(self.st.len()); @@ -729,10 +975,10 @@ impl Sam { } fn mark_boundary_tx(&mut self, tx: &mut SamTx) { - if !self.text.is_empty() { + if !self.is_empty() { // boundary_after is truncated on rollback, so no need to log. - let i = self.text.len() - 1; - self.boundary_after[i] = 1; + let i = self.len() - 1; + self.boundary_after.set(i); } // last is restored on rollback. self.last = 0; @@ -990,7 +1236,8 @@ impl LM { self.byte_map = [-1; 256]; let mut max_cp = 0u32; - for &v in &sam.text { + for idx in 0..sam.len() { + let v = sam.text_at(idx); if v > max_cp { max_cp = v; } @@ -998,7 +1245,8 @@ impl LM { if max_cp < 256 { let mut counts = [0u64; 256]; - for &v in &sam.text { + for idx in 0..sam.len() { + let v = sam.text_at(idx); counts[v as usize] += 1; } let mut uniq = 0usize; @@ -1038,7 +1286,7 @@ impl LM { return; } - let mut tmp = sam.text.clone(); + let mut tmp = sam.text_u32_vec(); tmp.sort_unstable(); tmp.dedup(); if tmp.is_empty() { @@ -1048,7 +1296,8 @@ impl LM { self.alpha_n = self.alphabet.len() as u32; self.unigram = vec![0u64; self.alphabet.len()]; self.total_uni = 0; - for &ch in &sam.text { + for idx in 0..sam.len() { + let ch = sam.text_at(idx); if let Ok(i) = self.alphabet.binary_search(&ch) { self.unigram[i] += 1; self.total_uni += 1; @@ -1149,6 +1398,7 @@ impl LM { if additional == 0 { return; } + let additional = additional.min(ROSA_STREAM_HINT_CAP_SYMBOLS); self.ls .reserve_exact(additional.saturating_mul(2).saturating_add(16)); self.nodes @@ -1167,19 +1417,19 @@ impl LM { self.nodes.clear(); let mut seg_start = 0usize; - while seg_start < sam.text.len() { + while seg_start < sam.len() { let mut seg_end = seg_start; - while seg_end < sam.text.len() { - let b = sam.boundary_after[seg_end]; + while seg_end < sam.len() { + let b = sam.boundary_after.get(seg_end); seg_end += 1; - if b != 0 { + if b { break; } } if seg_end - seg_start >= 2 { let mut v = 0; for i in seg_start..(seg_end - 1) { - let ch = sam.text[i]; + let ch = sam.text_at(i); v = sam.advance(v, ch); let mut ctx = v; if max_order >= 0 { @@ -1192,7 +1442,7 @@ impl LM { ctx = 0; } } - let nxt = sam.text[i + 1]; + let nxt = sam.text_at(i + 1); let si = self.find_sym(nxt); if si >= 0 { self.inc(state_usize(ctx) as u32, si as u32, 1); @@ -1466,11 +1716,28 @@ struct LmTx { old_nodes_len: usize, ls_changes: Vec<(usize, LmState)>, node_changes: Vec<(usize, CountNode)>, - // unigram delta for bytes - uni_delta: [u64; BYTE_ALPHA_N], + // Sparse unigram deltas for byte-alphabet transactions. AIQI and other + // bit-token callers usually touch only symbols 0/1, so dense 256-way + // transaction state is disproportionate in reversible hot paths. + uni_delta: Vec<(u8, u64)>, total_uni_add: u64, } +impl LmTx { + fn record_unigram_byte(&mut self, byte: u8) { + if let Some((_, delta)) = self + .uni_delta + .iter_mut() + .find(|(symbol, _)| *symbol == byte) + { + *delta += 1; + } else { + self.uni_delta.push((byte, 1)); + } + self.total_uni_add += 1; + } +} + #[derive(Clone, Default)] struct RngStream { buf: Vec, @@ -1564,18 +1831,15 @@ pub struct RosaPlus { dist: Vec, } -/// A lightweight snapshot of the append-only internal SAM buffers. +/// A full snapshot of the ROSA model state. /// -/// Restoring to a checkpoint is O(1) (via truncation) and is meant to support -/// repeated evaluation of different continuations from the same base training state. -#[derive(Clone, Copy, Debug)] +/// Generic reversible checkpoints for ROSA must restore more than append-only +/// buffers: suffix-automaton updates can rewire pre-existing states to newly +/// created clone states, so truncation-only snapshots are unsound. This +/// checkpoint therefore captures the full model state. +#[derive(Clone)] pub struct RosaCheckpoint { - sam_st_len: usize, - sam_ed_len: usize, - sam_text_len: usize, - sam_text_states_len: usize, - sam_boundary_after_len: usize, - sam_last: SamStateIx, + model: Box, } /// Transaction object used to roll back a temporary conditional update. @@ -1587,6 +1851,45 @@ pub struct RosaTx { seg_len: usize, } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[cfg(any(test, feature = "research-tooling"))] +pub(crate) struct RosaMemoryUsage { + pub sam_state_count: usize, + pub sam_edge_count: usize, + pub lm_state_count: usize, + pub lm_node_count: usize, + pub lm_sym_overflow_count: usize, + pub lm_count_overflow_count: usize, + pub sam_state_bytes: usize, + pub sam_edge_bytes: usize, + pub sam_text_bytes: usize, + pub sam_state_trace_bytes: usize, + pub sam_boundary_bytes: usize, + pub sam_root_index_bytes: usize, + pub lm_core_bytes: usize, + pub lm_node_storage_bytes: usize, + pub lm_overflow_bytes: usize, + pub scratch_bytes: usize, + pub rng_bytes: usize, +} + +#[cfg(any(test, feature = "research-tooling"))] +impl RosaMemoryUsage { + pub(crate) fn total_bytes(self) -> usize { + self.sam_state_bytes + + self.sam_edge_bytes + + self.sam_text_bytes + + self.sam_state_trace_bytes + + self.sam_boundary_bytes + + self.sam_root_index_bytes + + self.lm_core_bytes + + self.lm_node_storage_bytes + + self.lm_overflow_bytes + + self.scratch_bytes + + self.rng_bytes + } +} + impl RosaPlus { /// Create a new ROSA+ model. /// @@ -1613,9 +1916,10 @@ impl RosaPlus { return; } - if self.sam.text.is_empty() { + if self.sam.is_empty() { self.sam = Sam::new(s.len()); } + self.reserve_for_stream(s.len()); for &b in s { self.sam.feed(b as u32); @@ -1643,12 +1947,7 @@ impl RosaPlus { self.sam.finalize_endpos(); self.lm = LM::default(); self.lm.build_alphabet(&self.sam); - let mo = if self.max_order < 0 { - -1 - } else { - self.max_order - }; - self.lm.build_counts(&self.sam, mo); + self.lm.build_counts(&self.sam, self.max_order); self.lm_built = true; self.dist.resize(self.lm.alpha_n as usize, 0.0); } @@ -1662,12 +1961,7 @@ impl RosaPlus { pub fn build_lm_no_finalize_endpos(&mut self) { self.lm = LM::default(); self.lm.build_alphabet(&self.sam); - let mo = if self.max_order < 0 { - -1 - } else { - self.max_order - }; - self.lm.build_counts(&self.sam, mo); + self.lm.build_counts(&self.sam, self.max_order); self.lm_built = true; self.dist.resize(self.lm.alpha_n as usize, 0.0); } @@ -1688,7 +1982,8 @@ impl RosaPlus { // Unigram counts let mut counts = [0u64; 256]; - for &v in &self.sam.text { + for idx in 0..self.sam.len() { + let v = self.sam.text_at(idx); if v < 256 { counts[v as usize] += 1; } @@ -1703,12 +1998,7 @@ impl RosaPlus { } // Counts - let mo = if self.max_order < 0 { - -1 - } else { - self.max_order - }; - self.lm.build_counts(&self.sam, mo); + self.lm.build_counts(&self.sam, self.max_order); self.lm_built = true; self.dist.resize(BYTE_ALPHA_N, 0.0); } @@ -1721,13 +2011,13 @@ impl RosaPlus { old_nodes_len: self.lm.nodes.len(), ls_changes: Vec::new(), node_changes: Vec::new(), - uni_delta: [0u64; BYTE_ALPHA_N], + uni_delta: Vec::new(), total_uni_add: 0, }; RosaTx { sam: sam_tx, lm: lm_tx, - seg_start: self.sam.text.len(), + seg_start: self.sam.len(), seg_len: 0, } } @@ -1753,7 +2043,7 @@ impl RosaPlus { return; } - if self.sam.text.is_empty() { + if self.sam.is_empty() { self.sam = Sam::new(s.len()); } self.reserve_for_stream(s.len()); @@ -1772,7 +2062,7 @@ impl RosaPlus { ); } - let seg_start = self.sam.text.len(); + let seg_start = self.sam.len(); for &b in s { self.sam.feed(b as u32); self.lm.unigram[b as usize] += 1; @@ -1790,7 +2080,7 @@ impl RosaPlus { ); } - let seg_end = self.sam.text.len(); + let seg_end = self.sam.len(); if seg_end.saturating_sub(seg_start) >= 1 { let mo = if self.max_order < 0 { -1 @@ -1798,15 +2088,7 @@ impl RosaPlus { self.max_order }; let mut start_i = seg_start; - if seg_start > 0 - && self - .sam - .boundary_after - .get(seg_start - 1) - .copied() - .unwrap_or(0) - == 0 - { + if seg_start > 0 && !self.sam.boundary_after.get(seg_start - 1) { start_i = seg_start - 1; } for i in start_i..(seg_end - 1) { @@ -1819,7 +2101,7 @@ impl RosaPlus { ctx = 0; } } - let nxt = self.sam.text[i + 1]; + let nxt = self.sam.text_at(i + 1); let si = self.lm.find_sym(nxt); if si >= 0 { let mut u = ctx; @@ -1837,7 +2119,7 @@ impl RosaPlus { /// Apply a single byte sequential update without rollback bookkeeping. #[inline] pub fn train_byte(&mut self, b: u8) { - if self.sam.text.is_empty() { + if self.sam.is_empty() { self.sam = Sam::new(1); } if !self.lm_built || !self.lm.has_byte_map || (self.lm.alpha_n as usize) != BYTE_ALPHA_N { @@ -1859,16 +2141,8 @@ impl RosaPlus { ); } - let seg_end = self.sam.text.len(); - if seg_end > 1 - && self - .sam - .boundary_after - .get(seg_end - 2) - .copied() - .unwrap_or(0) - == 0 - { + let seg_end = self.sam.len(); + if seg_end > 1 && !self.sam.boundary_after.get(seg_end - 2) { let mo = if self.max_order < 0 { -1 } else { @@ -1899,6 +2173,19 @@ impl RosaPlus { self.sam.last = 0; } + /// Return the current predictive cursor state. + pub(crate) fn conditioning_cursor(&self) -> i32 { + self.sam.last + } + + /// Restore a previously recorded predictive cursor state. + /// + /// Available whenever the ROSA backend is compiled (used by general + /// checkpoint/restore paths including public bit-session frozen rewinds). + pub(crate) fn restore_conditioning_cursor(&mut self, cursor: i32) { + self.sam.last = cursor; + } + /// Advance only the predictive cursor without mutating fitted counts. pub fn advance_conditioning_byte(&mut self, b: u8) { self.sam.last = self.sam.advance(self.sam.last, b as u32); @@ -1924,8 +2211,9 @@ impl RosaPlus { // Feed all bytes (SAM structure changes are logged). for &b in s { self.sam.feed_tx(&mut tx.sam, b as u32); - tx.lm.uni_delta[b as usize] += 1; - tx.lm.total_uni_add += 1; + tx.lm.record_unigram_byte(b); + self.lm.unigram[b as usize] += 1; + self.lm.total_uni += 1; } if mark_boundary { self.sam.mark_boundary_tx(&mut tx.sam); @@ -1944,17 +2232,9 @@ impl RosaPlus { ); } - // Update unigram counts (fixed 256 alphabet assumed). - for i in 0..256 { - if tx.lm.uni_delta[i] != 0 { - self.lm.unigram[i] += tx.lm.uni_delta[i]; - } - } - self.lm.total_uni += tx.lm.total_uni_add; - // Update conditional counts for the new segment only. let seg_start = tx.seg_start; - let seg_end = self.sam.text.len(); + let seg_end = self.sam.len(); tx.seg_len = seg_end - seg_start; if tx.seg_len >= 1 { let mo = if self.max_order < 0 { @@ -1966,16 +2246,7 @@ impl RosaPlus { // previous symbol into the first new symbol. For segmented examples, // respect boundary markers and skip that transition. let mut start_i = seg_start; - if !mark_boundary - && seg_start > 0 - && self - .sam - .boundary_after - .get(seg_start - 1) - .copied() - .unwrap_or(0) - == 0 - { + if !mark_boundary && seg_start > 0 && !self.sam.boundary_after.get(seg_start - 1) { start_i = seg_start - 1; } for i in start_i..(seg_end - 1) { @@ -1989,7 +2260,7 @@ impl RosaPlus { ctx = 0; } } - let nxt = self.sam.text[i + 1]; + let nxt = self.sam.text_at(i + 1); let si = self.lm.find_sym(nxt); if si >= 0 { let mut u = ctx; @@ -2010,11 +2281,9 @@ impl RosaPlus { // Restore LM changes // Unigram rollback if self.lm.unigram.len() >= BYTE_ALPHA_N { - for i in 0..BYTE_ALPHA_N { - let d = tx.lm.uni_delta[i]; - if d != 0 { - self.lm.unigram[i] = self.lm.unigram[i].saturating_sub(d); - } + for (symbol, delta) in tx.lm.uni_delta { + let idx = usize::from(symbol); + self.lm.unigram[idx] = self.lm.unigram[idx].saturating_sub(delta); } self.lm.total_uni = self.lm.total_uni.saturating_sub(tx.lm.total_uni_add); } @@ -2098,6 +2367,82 @@ impl RosaPlus { } } + #[cfg(any(test, feature = "research-tooling"))] + pub(crate) fn memory_usage_breakdown(&self) -> RosaMemoryUsage { + use std::mem::size_of; + + RosaMemoryUsage { + sam_state_count: self.sam.st.len(), + sam_edge_count: self.sam.ed.len(), + lm_state_count: self.lm.ls.len(), + lm_node_count: self.lm.nodes.len(), + lm_sym_overflow_count: self.lm.nodes.sym_overflow.len(), + lm_count_overflow_count: self.lm.nodes.cnt_overflow.len(), + sam_state_bytes: self.sam.st.len().saturating_mul(size_of::()), + sam_edge_bytes: self.sam.ed.len().saturating_mul(size_of::()), + sam_text_bytes: self.sam.text.allocated_bytes(), + sam_state_trace_bytes: self + .sam + .text_states + .capacity() + .saturating_mul(size_of::()), + sam_boundary_bytes: self.sam.boundary_after.allocated_bytes(), + sam_root_index_bytes: size_of::<[SamStateIx; BYTE_ALPHA_N]>(), + lm_core_bytes: self.lm.alphabet.capacity().saturating_mul(size_of::()) + + self.lm.unigram.capacity().saturating_mul(size_of::()) + + self.lm.ls.capacity().saturating_mul(size_of::()), + lm_node_storage_bytes: self + .lm + .nodes + .sym_lo + .capacity() + .saturating_mul(size_of::()) + + self + .lm + .nodes + .cnt_lo + .capacity() + .saturating_mul(size_of::()) + + self + .lm + .nodes + .next + .capacity() + .saturating_mul(size_of::()) + + self + .lm + .nodes + .cnt_overflow_mask + .capacity() + .saturating_mul(size_of::()), + lm_overflow_bytes: self + .lm + .nodes + .sym_overflow + .capacity() + .saturating_mul(size_of::() + size_of::()) + + self + .lm + .nodes + .cnt_overflow + .capacity() + .saturating_mul(size_of::() + size_of::()), + scratch_bytes: self.dist.capacity().saturating_mul(size_of::()) + + self.scratch.idx.capacity().saturating_mul(size_of::()) + + self + .scratch + .logits + .capacity() + .saturating_mul(size_of::()) + + self + .scratch + .exps + .capacity() + .saturating_mul(size_of::()), + rng_bytes: self.rng.buf.capacity().saturating_mul(size_of::()), + } + } + /// Approximate in-memory footprint of major model buffers. pub fn estimated_size_bytes(&self) -> usize { use std::mem::size_of; @@ -2106,60 +2451,77 @@ impl RosaPlus { n = n.saturating_add(self.sam.st.len().saturating_mul(size_of::())); n = n.saturating_add(self.sam.ed.len().saturating_mul(size_of::())); - n = n.saturating_add(self.sam.text.len().saturating_mul(size_of::())); + n = n.saturating_add(self.sam.text.allocated_bytes()); n = n.saturating_add( self.sam .text_states - .len() + .capacity() .saturating_mul(size_of::()), ); n = n.saturating_add(size_of::<[SamStateIx; BYTE_ALPHA_N]>()); + n = n.saturating_add(self.sam.boundary_after.allocated_bytes()); + + n = n.saturating_add(self.lm.alphabet.capacity().saturating_mul(size_of::())); + n = n.saturating_add(self.lm.unigram.capacity().saturating_mul(size_of::())); + n = n.saturating_add(self.lm.ls.capacity().saturating_mul(size_of::())); n = n.saturating_add( - self.sam - .boundary_after - .len() - .saturating_mul(size_of::()), + self.lm + .nodes + .sym_lo + .capacity() + .saturating_mul(size_of::()), + ); + n = n.saturating_add( + self.lm + .nodes + .cnt_lo + .capacity() + .saturating_mul(size_of::()), ); - - n = n.saturating_add(self.lm.alphabet.len().saturating_mul(size_of::())); - n = n.saturating_add(self.lm.unigram.len().saturating_mul(size_of::())); - n = n.saturating_add(self.lm.ls.len().saturating_mul(size_of::())); - n = n.saturating_add(self.lm.nodes.sym_lo.len().saturating_mul(size_of::())); - n = n.saturating_add(self.lm.nodes.cnt_lo.len().saturating_mul(size_of::())); n = n.saturating_add( self.lm .nodes .next - .len() + .capacity() .saturating_mul(size_of::()), ); n = n.saturating_add( self.lm .nodes .cnt_overflow_mask - .len() + .capacity() .saturating_mul(size_of::()), ); n = n.saturating_add( self.lm .nodes .sym_overflow - .len() + .capacity() .saturating_mul(size_of::() + size_of::()), ); n = n.saturating_add( self.lm .nodes .cnt_overflow - .len() + .capacity() .saturating_mul(size_of::() + size_of::()), ); - n = n.saturating_add(self.dist.len().saturating_mul(size_of::())); - n = n.saturating_add(self.scratch.idx.len().saturating_mul(size_of::())); - n = n.saturating_add(self.scratch.logits.len().saturating_mul(size_of::())); - n = n.saturating_add(self.scratch.exps.len().saturating_mul(size_of::())); - n = n.saturating_add(self.rng.buf.len().saturating_mul(size_of::())); + n = n.saturating_add(self.dist.capacity().saturating_mul(size_of::())); + n = n.saturating_add(self.scratch.idx.capacity().saturating_mul(size_of::())); + n = n.saturating_add( + self.scratch + .logits + .capacity() + .saturating_mul(size_of::()), + ); + n = n.saturating_add( + self.scratch + .exps + .capacity() + .saturating_mul(size_of::()), + ); + n = n.saturating_add(self.rng.buf.capacity().saturating_mul(size_of::())); n } @@ -2195,33 +2557,19 @@ impl RosaPlus { } } - /// A checkpoint that allows restoring the ROSA model back to a previous trained state - /// by truncating append-only internal buffers. + /// Capture a checkpoint that can restore the exact trained and predictive state. /// - /// Intended for workflows that repeatedly evaluate different continuations from the same base - /// training text (e.g. universal-prior conditioned scoring). + /// This clones the full model. Prefer predictor/runtime checkpoints in hot + /// paths, which can use lighter-weight backend-specific rollback strategies. pub fn checkpoint(&self) -> RosaCheckpoint { RosaCheckpoint { - sam_st_len: self.sam.st.len(), - sam_ed_len: self.sam.ed.len(), - sam_text_len: self.sam.text.len(), - sam_text_states_len: self.sam.text_states.len(), - sam_boundary_after_len: self.sam.boundary_after.len(), - sam_last: self.sam.last, + model: Box::new(self.clone()), } } - /// Restore the model to a previously captured checkpoint. - /// - /// This invalidates the LM; callers should rebuild it before scoring. + /// Restore the model to a previously captured checkpoint exactly. pub fn restore(&mut self, ck: &RosaCheckpoint) { - self.sam.st.truncate(ck.sam_st_len); - self.sam.ed.truncate(ck.sam_ed_len); - self.sam.text.truncate(ck.sam_text_len); - self.sam.text_states.truncate(ck.sam_text_states_len); - self.sam.boundary_after.truncate(ck.sam_boundary_after_len); - self.sam.last = ck.sam_last; - self.lm_built = false; + *self = (*ck.model).clone(); } #[inline(always)] @@ -2529,9 +2877,9 @@ impl RosaPlus { -total_log_prob / (data.len() as f64) } - /// Returns the marginal (unigram) distribution over the training data. + /// Returns the unigram distribution over the training data. /// Output: Vec of (codepoint, probability) pairs, sorted by codepoint. - pub fn marginal_distribution(&self) -> Vec<(u32, f64)> { + pub fn unigram_distribution(&self) -> Vec<(u32, f64)> { if self.lm.total_uni == 0 { return Vec::new(); } @@ -2548,9 +2896,9 @@ impl RosaPlus { result } - /// Compute the marginal entropy H(X) from the unigram distribution. + /// Compute the unigram entropy H(X) from the observed symbol frequencies. /// Returns bits per symbol. - pub fn marginal_entropy(&self) -> f64 { + pub fn unigram_entropy(&self) -> f64 { if self.lm.total_uni == 0 { return 0.0; } @@ -2574,7 +2922,7 @@ impl RosaPlus { // Transactional conditional updates require a valid prefix-state trace. // If this invariant is violated, the loaded model would be unusable. - if self.sam.text_states.len() != self.sam.text.len() + 1 { + if self.sam.text_states.len() != self.sam.len() + 1 { return Err(std::io::Error::other( "SAM text_states mismatch (expected text.len()+1)", )); @@ -2589,7 +2937,7 @@ impl RosaPlus { // SAM write_len64(&mut f, self.sam.st.len())?; write_len64(&mut f, self.sam.ed.len())?; - write_len64(&mut f, self.sam.text.len())?; + write_len64(&mut f, self.sam.len())?; for st in &self.sam.st { f.write_all(&st.link.to_le_bytes())?; f.write_all(&st.len.to_le_bytes())?; @@ -2606,8 +2954,10 @@ impl RosaPlus { f.write_all(&e.to.to_le_bytes())?; f.write_all(&e.next.to_le_bytes())?; } - write_u32_slice_le(&mut f, &self.sam.text)?; - f.write_all(&self.sam.boundary_after)?; + let text_u32 = self.sam.text_u32_vec(); + let boundary_bytes = self.sam.boundary_bytes_vec(); + write_u32_slice_le(&mut f, &text_u32)?; + f.write_all(&boundary_bytes)?; // Persist SAM cursor + prefix trace. f.write_all(&self.sam.last.to_le_bytes())?; @@ -2670,8 +3020,8 @@ impl RosaPlus { m.sam = Sam::new(text_n); m.sam.st.resize(st_n, SamState::default()); m.sam.ed.resize(ed_n, SamEdge::default()); - m.sam.text.resize(text_n, 0u32); - m.sam.boundary_after.resize(text_n, 0u8); + let mut text = vec![0u32; text_n]; + let mut boundary = vec![0u8; text_n]; for i in 0..st_n { f.read_exact(&mut b4)?; @@ -2706,8 +3056,10 @@ impl RosaPlus { f.read_exact(&mut b4)?; m.sam.ed[i].next = u32::from_le_bytes(b4); } - read_u32_slice_le(&mut f, &mut m.sam.text)?; - f.read_exact(&mut m.sam.boundary_after)?; + read_u32_slice_le(&mut f, &mut text)?; + f.read_exact(&mut boundary)?; + m.sam.set_text_from_u32_slice(&text); + m.sam.boundary_after.load_from_bytes(&boundary); // SAM cursor + prefix trace. f.read_exact(&mut b4)?; @@ -2893,8 +3245,10 @@ impl RosaPlus { } else { self.max_order }; + self.dist.resize(self.lm.alpha_n as usize, 0.0); - self.lm.probs_for_state(&self.sam, mo, v, &mut self.dist); + self.lm + .probs_for_state_raw(&self.sam, mo, v, &mut self.dist); if self.lm.has_byte_map && (self.lm.alpha_n as usize) == BYTE_ALPHA_N @@ -3053,8 +3407,9 @@ mod tests { while u != SAM_STATE_NONE { if !(max_order >= 0 && (sam.st[state_usize(u)].len as i64) > max_order) { - let n = lm.ls[state_usize(u)].total_n; - let t = lm.ls[state_usize(u)].types_t; + let ls = &lm.ls[state_usize(u)]; + let n = ls.total_n; + let t = ls.types_t; if n > 0 { let lam = if t > 0 { (n as f64) / ((n + (t as u64)) as f64) @@ -3063,7 +3418,6 @@ mod tests { }; let scale = residual * lam; let mut count_for_sym = 0u64; - let ls = &lm.ls[state_usize(u)]; if LM::ls_is_implicit_single(ls) { if ls.last_sym == sym_idx { count_for_sym = n; @@ -3104,8 +3458,9 @@ mod tests { let mut u = v; while u != SAM_STATE_NONE { if !(max_order >= 0 && (sam.st[state_usize(u)].len as i64) > max_order) { - let n = lm.ls[state_usize(u)].total_n; - let t = lm.ls[state_usize(u)].types_t; + let ls = &lm.ls[state_usize(u)]; + let n = ls.total_n; + let t = ls.types_t; if n > 0 { let lam = if t > 0 { (n as f64) / ((n + (t as u64)) as f64) @@ -3114,7 +3469,6 @@ mod tests { }; let scale = residual * lam; let inv_n = 1.0 / (n as f64); - let ls = &lm.ls[state_usize(u)]; if LM::ls_is_implicit_single(ls) { out[ls.last_sym as usize] += scale; } else { @@ -3168,21 +3522,58 @@ mod tests { m.train_example(b"hello"); m.build_lm_full_bytes_no_finalize_endpos(); - let base_text = m.sam.text.clone(); - let base_text_len = m.sam.text.len(); + let base_text = m.sam.text_u32_vec(); + let base_text_len = m.sam.len(); let base_total_uni = m.lm.total_uni; assert!(base_text_len > 0); let mut tx = m.begin_tx(); m.train_example_tx(&mut tx, b"abc"); assert_eq!(m.lm.total_uni, base_total_uni + 3); - assert_eq!(m.sam.text.len(), base_text_len + 3); + assert_eq!(m.sam.len(), base_text_len + 3); m.rollback_tx(tx); - assert_eq!(m.sam.text, base_text); + assert_eq!(m.sam.text_u32_vec(), base_text); assert_eq!(m.lm.total_uni, base_total_uni); } + #[test] + fn byte_stream_storage_stays_compact_for_byte_only_training() { + let mut m = RosaPlus::new(4, false, 0, 123); + m.train_example(b"abracadabra mississippi"); + m.build_lm_full_bytes_no_finalize_endpos(); + + assert!(matches!(m.sam.text, SamText::Byte(_))); + assert_eq!(m.sam.text_u32_vec().len(), m.sam.len()); + } + + #[test] + fn reserve_for_stream_caps_initial_roaming_capacity() { + let requested = ROSA_STREAM_HINT_CAP_SYMBOLS.saturating_mul(8); + let mut m = RosaPlus::new(4, false, 0, 123); + m.reserve_for_stream(requested); + + let half_requested = requested / 2; + assert!(m.sam.text.capacity() < half_requested); + assert!(m.sam.text_states.capacity() < half_requested); + assert!(m.lm.ls.capacity() < half_requested); + assert!(m.lm.nodes.sym_lo.capacity() < half_requested); + assert!(m.lm.nodes.next.capacity() < half_requested); + } + + #[test] + fn cps_training_promotes_internal_text_storage_exactly() { + let cps = [0u32, 7, 300, 42, 511, 42]; + let mut m = RosaPlus::new(-1, false, 0, 7); + for &cp in &cps { + m.sam.feed(cp); + } + m.build_lm_no_finalize_endpos(); + + assert!(matches!(m.sam.text, SamText::Codepoint(_))); + assert_eq!(m.sam.text_u32_vec(), cps); + } + #[test] fn train_sequence_matches_transactional_sequence_update() { let mut direct = RosaPlus::new(4, false, 0, 123); @@ -3199,7 +3590,7 @@ mod tests { let mut tx = tx_model.begin_tx(); tx_model.train_sequence_tx(&mut tx, b" mississippi"); - assert_eq!(direct.sam.text, tx_model.sam.text); + assert_eq!(direct.sam.text_u32_vec(), tx_model.sam.text_u32_vec()); assert_eq!(direct.sam.text_states, tx_model.sam.text_states); assert_eq!(direct.sam.boundary_after, tx_model.sam.boundary_after); assert_eq!(direct.sam.last, tx_model.sam.last); @@ -3234,7 +3625,7 @@ mod tests { tx_model.train_sequence_tx(&mut tx, &[b]); } - assert_eq!(direct.sam.text, tx_model.sam.text); + assert_eq!(direct.sam.text_u32_vec(), tx_model.sam.text_u32_vec()); assert_eq!(direct.sam.text_states, tx_model.sam.text_states); assert_eq!(direct.sam.boundary_after, tx_model.sam.boundary_after); assert_eq!(direct.sam.last, tx_model.sam.last); @@ -3244,6 +3635,22 @@ mod tests { assert_eq!(direct.lm.ls, tx_model.lm.ls); } + #[test] + fn repeated_updates_in_one_transaction_count_unigrams_once_per_byte() { + let mut direct = RosaPlus::new(4, false, 0, 123); + direct.build_lm_full_bytes_no_finalize_endpos(); + direct.train_sequence(b"abracadabra"); + + let mut tx_model = RosaPlus::new(4, false, 0, 123); + tx_model.build_lm_full_bytes_no_finalize_endpos(); + let mut tx = tx_model.begin_tx(); + tx_model.train_sequence_tx(&mut tx, b"abra"); + tx_model.train_sequence_tx(&mut tx, b"cadabra"); + + assert_eq!(direct.lm.total_uni, tx_model.lm.total_uni); + assert_eq!(direct.lm.unigram, tx_model.lm.unigram); + } + #[test] fn max_order_capping_keeps_probability_semantics() { let mut m = RosaPlus::new(4, false, 0, 321); @@ -3275,25 +3682,32 @@ mod tests { } #[test] - fn checkpoint_restore_reverts_append_only_buffers() { + fn checkpoint_restore_reverts_exact_state() { let mut m = RosaPlus::new(3, true, b'\n', 7); m.train_example(b"aaaa"); + m.build_lm_full_bytes_no_finalize_endpos(); + let before_prob = m.prob_for_last(b'a' as u32); let ck = m.checkpoint(); - let base_text = m.sam.text.clone(); + let base_text = m.sam.text_u32_vec(); let base_states = m.sam.text_states.clone(); let base_boundary = m.sam.boundary_after.clone(); let base_last = m.sam.last; m.train_example(b"bbbb"); - assert_ne!(m.sam.text, base_text); + assert_ne!(m.sam.text_u32_vec(), base_text); m.restore(&ck); - assert_eq!(m.sam.text, base_text); + assert_eq!(m.sam.text_u32_vec(), base_text); assert_eq!(m.sam.text_states, base_states); assert_eq!(m.sam.boundary_after, base_boundary); assert_eq!(m.sam.last, base_last); - assert!(!m.lm_built); + assert!(m.lm_built); + let after_prob = m.prob_for_last(b'a' as u32); + assert!( + (after_prob - before_prob).abs() <= 1e-12, + "checkpoint restore should recover exact predictive state: before={before_prob} after={after_prob}" + ); } #[test] @@ -3347,8 +3761,7 @@ mod tests { m.train_example(b"abracadabra"); m.build_lm(); let before_prob = m.prob_for_last(b'a' as u32); - let before_size = m.estimated_size_bytes(); - let before_text = m.sam.text.clone(); + let before_text = m.sam.text_u32_vec(); let before_states = m.sam.text_states.clone(); let before_last = m.sam.last; let before_nodes = m.lm.nodes.len(); @@ -3362,11 +3775,46 @@ mod tests { assert_eq!(loaded.use_eot, m.use_eot); assert_eq!(loaded.eot, m.eot); assert_eq!(loaded.seed, m.seed); - assert_eq!(loaded.sam.text, before_text); + assert_eq!(loaded.sam.text_u32_vec(), before_text); assert_eq!(loaded.sam.text_states, before_states); assert_eq!(loaded.sam.last, before_last); assert_eq!(loaded.lm.nodes.len(), before_nodes); - assert_eq!(loaded.estimated_size_bytes(), before_size); + assert!(loaded.estimated_size_bytes() > 0); assert!((loaded.prob_for_last(b'a' as u32) - before_prob).abs() < 1e-12); } + + #[test] + fn save_load_roundtrip_preserves_promoted_codepoint_storage() { + let path = temp_model_path("roundtrip_cps"); + let mut m = RosaPlus::new(8, false, 0, 1234); + for &cp in &[0u32, 7, 300, 42, 511, 42, 300] { + m.sam.feed(cp); + } + m.build_lm_no_finalize_endpos(); + let before_text = m.sam.text_u32_vec(); + let before_prob = m.prob_for_last(300); + let path_str = path.to_string_lossy().into_owned(); + + m.save(&path_str).expect("save failed"); + let mut loaded = RosaPlus::load(&path_str).expect("load failed"); + fs::remove_file(&path).expect("cleanup failed"); + + assert_eq!(loaded.sam.text_u32_vec(), before_text); + assert!(matches!(loaded.sam.text, SamText::Codepoint(_))); + assert!((loaded.prob_for_last(300) - before_prob).abs() < 1e-12); + } + + #[test] + fn rosa_memory_usage_breakdown_sums_to_estimated_total() { + let mut model = RosaPlus::new(8, true, b'\n', 1234); + model.train_example( + b"abracadabra mississippi banana bandana rosa memory validation payload", + ); + model.build_lm(); + + let usage = model.memory_usage_breakdown(); + assert_eq!(usage.total_bytes(), model.estimated_size_bytes()); + assert!(usage.sam_state_bytes > 0); + assert!(usage.lm_core_bytes > 0); + } } diff --git a/src/backends/rwkvzip/mod.rs b/crates/infotheory/src/backends/rwkvzip/mod.rs similarity index 80% rename from src/backends/rwkvzip/mod.rs rename to crates/infotheory/src/backends/rwkvzip/mod.rs index 9f6f0881..3538d135 100644 --- a/src/backends/rwkvzip/mod.rs +++ b/crates/infotheory/src/backends/rwkvzip/mod.rs @@ -253,13 +253,67 @@ impl FullTrainSettings { } } -#[derive(Clone)] struct FullTbpttRuntime { pending_input_token: Option, - pending_input_pre_state: Option, - segment_start_state: Option, + pending_input_pre_state: State, + pending_input_pre_state_valid: bool, + segment_start_state: State, + segment_start_state_valid: bool, steps: Vec<(u32, u8)>, settings: Option, + replay_workspace: Option, +} + +impl Clone for FullTbpttRuntime { + fn clone(&self) -> Self { + Self { + pending_input_token: self.pending_input_token, + pending_input_pre_state: self.pending_input_pre_state.clone(), + pending_input_pre_state_valid: self.pending_input_pre_state_valid, + segment_start_state: self.segment_start_state.clone(), + segment_start_state_valid: self.segment_start_state_valid, + steps: self.steps.clone(), + settings: self.settings, + replay_workspace: None, + } + } +} + +impl FullTbpttRuntime { + fn new(model_cfg: &rwkv7::Config) -> Self { + Self { + pending_input_token: None, + pending_input_pre_state: State::new(model_cfg), + pending_input_pre_state_valid: false, + segment_start_state: State::new(model_cfg), + segment_start_state_valid: false, + steps: Vec::new(), + settings: None, + replay_workspace: None, + } + } + + fn clear_segment_buffers(&mut self) { + self.pending_input_token = None; + self.pending_input_pre_state_valid = false; + self.segment_start_state_valid = false; + self.steps.clear(); + self.settings = None; + } + + fn is_idle(&self) -> bool { + self.pending_input_token.is_none() + && !self.pending_input_pre_state_valid + && !self.segment_start_state_valid + && self.steps.is_empty() + && self.settings.is_none() + } +} + +#[derive(Clone, Copy)] +enum PdfSource<'a> { + External(&'a [f64]), + CurrentBuffer, } #[derive(Clone)] @@ -277,9 +331,10 @@ impl OnlineRuntime { cfg: OnlineConfig, canonical_method: String, policy: Option, - vocab_size: usize, - hidden_size: usize, + model_cfg: &rwkv7::Config, ) -> Self { + let vocab_size = model_cfg.vocab_size; + let hidden_size = model_cfg.hidden_size; let mut use_adam = matches!(cfg.train_mode, OnlineTrainMode::Adam); if let Some(pol) = &policy { use_adam = policy_uses_adam(pol) || use_adam; @@ -304,43 +359,88 @@ impl OnlineRuntime { lm_head_adam_m: use_adam.then(|| vec![0.0; vocab_size * hidden_size]), lm_head_adam_v: use_adam.then(|| vec![0.0; vocab_size * hidden_size]), adam_t: 0, - full_tbptt: needs_full_trace.then(|| FullTbpttRuntime { - pending_input_token: None, - pending_input_pre_state: None, - segment_start_state: None, - steps: Vec::new(), - settings: None, - }), + full_tbptt: needs_full_trace.then(|| FullTbpttRuntime::new(model_cfg)), } } - fn prepare_policy_stream(&mut self, total_symbols: Option) -> Result<()> { + fn ensure_full_tbptt_runtime(&mut self, model_cfg: &rwkv7::Config) { + if self.needs_full_trace && self.full_tbptt.is_none() { + self.full_tbptt = Some(FullTbpttRuntime::new(model_cfg)); + } + } + + fn has_future_non_head_train_in_current_stream(&self) -> bool { + let Some(runtime) = self.policy_runtime.as_ref() else { + return self.needs_full_trace; + }; + runtime.has_future_train_matching(|train| { + train.hyper.lr > 0.0 && scope_needs_full_trace(&train.scope) + }) + } + + fn maybe_release_dead_full_tbptt(&mut self) { + let should_release = self.full_tbptt.as_ref().is_some_and(|tbptt| { + tbptt.is_idle() && !self.has_future_non_head_train_in_current_stream() + }); + if should_release { + self.full_tbptt = None; + } + } + + fn prepare_policy_stream( + &mut self, + model_cfg: &rwkv7::Config, + total_symbols: Option, + ) -> Result<()> { + let policy_runtime = match &self.policy { + Some(p) => Some(PolicyRuntime::new(p.compile(total_symbols)?)), + None => None, + }; self.policy_stream_total = total_symbols; self.policy_train_steps = 0; + self.ensure_full_tbptt_runtime(model_cfg); if let Some(tbptt) = self.full_tbptt.as_mut() { // Preserve the current predictive edge so the first symbol of the // new stream still trains against the already-primed distribution. - tbptt.segment_start_state = None; + tbptt.segment_start_state_valid = false; tbptt.steps.clear(); tbptt.settings = None; } - self.policy_runtime = match &self.policy { - Some(p) => Some(PolicyRuntime::new(p.compile(total_symbols)?)), - None => None, - }; + self.policy_runtime = policy_runtime; Ok(()) } #[inline] - fn next_policy_action(&mut self) -> Result> { + fn next_policy_action(&mut self, model_cfg: &rwkv7::Config) -> Result> { if self.policy.is_none() { return Ok(None); } if self.policy_runtime.is_none() { - self.prepare_policy_stream(None)?; + self.prepare_policy_stream(model_cfg, None)?; } Ok(self.policy_runtime.as_mut().map(PolicyRuntime::next_action)) } + + #[inline] + fn should_capture_full_trace_for_next_step(&self) -> bool { + if self.full_tbptt.is_none() { + return false; + } + let Some(runtime) = self.policy_runtime.as_ref() else { + // Preserve legacy behavior outside an explicitly prepared stream. + return true; + }; + let PolicyAction::Train(train) = runtime.peek_action() else { + return false; + }; + let scope = scope_from_train_action(&train); + if !scope.trains_non_head_params() || train.hyper.lr <= 0.0 { + return false; + } + let stride = train.hyper.stride.max(1) as u64; + let next_train_step = self.policy_train_steps.saturating_add(1); + stride <= 1 || next_train_step.is_multiple_of(stride) + } } #[allow(clippy::needless_range_loop, clippy::too_many_arguments)] @@ -663,10 +763,13 @@ fn parse_cfg_positional(csv: &str) -> Result { /// /// Supported formats: /// - `file:/path/to/model.safetensors` +/// - `file:/path/to/model%3Bv1.safetensors` /// - `file:/path/to/model.safetensors;policy:...` /// - `cfg:key=value,...[;policy:...]` /// - positional `cfg` CSV /// - existing model path +/// +/// `file:` methods percent-encode reserved delimiters inside the path segment. pub fn parse_method_spec(method: &str) -> Result { let (base, policy_segment) = split_method_policy_segments(method)?; let parse_policy = |s: &str| llm_policy::parse_policy_segment(s, RWKV_TRAIN_SCOPES); @@ -677,10 +780,11 @@ pub fn parse_method_spec(method: &str) -> Result { .context("failed to parse rwkv policy segment")?; if let Some(path) = base.strip_prefix("file:") { - let p = PathBuf::from(path.trim()); + let p = llm_policy::parse_method_file_path(path.trim()); if p.as_os_str().is_empty() { bail!("empty file path in rwkv method"); } + llm_policy::canonical_file_method_string(&p, policy.as_ref())?; if policy.as_ref().and_then(|p| p.load_from.as_ref()).is_some() { bail!("rwkv method cannot use policy load_from together with file:"); } @@ -746,6 +850,23 @@ pub fn parse_method_spec(method: &str) -> Result { ); } +/// Convert a parsed method specification back into canonical method syntax. +pub fn canonical_method_string(spec: &MethodSpec) -> Result { + match spec { + MethodSpec::File { path, policy } => { + llm_policy::canonical_file_method_string(path, policy.as_ref()) + } + MethodSpec::Online { cfg, policy } => { + let mut method = cfg_to_method_string(cfg); + if let Some(policy) = policy { + method.push_str(";policy:"); + method.push_str(&policy.canonical()); + } + Ok(method) + } + } +} + // ============================================================================= // File Header // ============================================================================= @@ -952,24 +1073,27 @@ impl Compressor { /// Create a compressor from a user method string. pub fn new_from_method(method: &str) -> Result { - match parse_method_spec(method)? { + let spec = parse_method_spec(method)?; + Self::new_from_method_spec(&spec) + } + + /// Create a compressor from a parsed method specification. + pub fn new_from_method_spec(spec: &MethodSpec) -> Result { + match spec { MethodSpec::File { path, policy } => { - let mut c = Self::new(&path)?; - if let Some(policy) = policy { - let canonical_method = - format!("file:{};policy:{}", path.display(), policy.canonical()); - let hidden = c.model.config().hidden_size; + let mut c = Self::new(path)?; + if let Some(policy) = policy.as_ref() { + let canonical_method = canonical_method_string(spec)?; let mut online = c.online.take().unwrap_or_else(|| { OnlineRuntime::new( OnlineConfig::default(), canonical_method.clone(), Some(policy.clone()), - VOCAB_SIZE, - hidden, + c.model.config(), ) }); online.canonical_method = canonical_method; - online.policy = Some(policy); + online.policy = Some(policy.clone()); online.needs_full_trace = online .policy .as_ref() @@ -977,7 +1101,9 @@ impl Compressor { .unwrap_or(false); c.online = Some(online); c.scratch.set_capture_train_trace( - c.online.as_ref().is_some_and(|o| o.needs_full_trace), + c.online + .as_ref() + .is_some_and(OnlineRuntime::should_capture_full_trace_for_next_step), ); } Ok(c) @@ -1009,20 +1135,18 @@ impl Compressor { Arc::new(Model::new_random(rwcfg, cfg.seed)?) }; let mut c = Self::new_from_model(model); - let mut canonical_method = cfg_to_method_string(&cfg); - if let Some(policy) = policy.as_ref() { - canonical_method.push_str(";policy:"); - canonical_method.push_str(&policy.canonical()); - } + let canonical_method = canonical_method_string(spec)?; c.online = Some(OnlineRuntime::new( - cfg, + cfg.clone(), canonical_method, - policy, - VOCAB_SIZE, - c.model.config().hidden_size, + policy.clone(), + c.model.config(), )); - c.scratch - .set_capture_train_trace(c.online.as_ref().is_some_and(|o| o.needs_full_trace)); + c.scratch.set_capture_train_trace( + c.online + .as_ref() + .is_some_and(OnlineRuntime::should_capture_full_trace_for_next_step), + ); Ok(c) } } @@ -1039,7 +1163,7 @@ impl Compressor { fn prepare_policy_stream(&mut self, total_symbols: Option) -> Result<()> { if let Some(online) = self.online.as_mut() { - online.prepare_policy_stream(total_symbols)?; + online.prepare_policy_stream(self.model.config(), total_symbols)?; } Ok(()) } @@ -1057,53 +1181,57 @@ impl Compressor { if let Some(online) = self.online.as_mut() && let Some(tbptt) = online.full_tbptt.as_mut() { - tbptt.pending_input_token = None; - tbptt.pending_input_pre_state = None; - tbptt.segment_start_state = None; - tbptt.steps.clear(); - tbptt.settings = None; + tbptt.clear_segment_buffers(); } } fn forward_with_online_record(&mut self, token: u32) { - if let Some(online) = self.online.as_mut() - && let Some(tbptt) = online.full_tbptt.as_mut() - { - tbptt.pending_input_token = Some(token); - tbptt.pending_input_pre_state = Some(self.state.clone()); + let mut capture_full_trace = false; + if let Some(online) = self.online.as_mut() { + capture_full_trace = online.should_capture_full_trace_for_next_step(); + if capture_full_trace && let Some(tbptt) = online.full_tbptt.as_mut() { + tbptt.pending_input_token = Some(token); + tbptt.pending_input_pre_state.copy_from(&self.state); + tbptt.pending_input_pre_state_valid = true; + } } + self.scratch.set_capture_train_trace(capture_full_trace); let _ = self .model .forward(&mut self.scratch, token, &mut self.state); } fn flush_full_tbptt_segment(&mut self) -> Result<()> { - let extracted = { - match self.online.as_mut() { - Some(online) => match online.full_tbptt.as_mut() { - Some(tbptt) if !tbptt.steps.is_empty() => { - let settings = tbptt.settings.ok_or_else(|| { - anyhow::anyhow!("rwkv full tbptt settings are missing") - })?; - let start_state = tbptt.segment_start_state.clone().ok_or_else(|| { - anyhow::anyhow!("rwkv full tbptt segment start is missing") - })?; - let steps = tbptt.steps.clone(); - tbptt.steps.clear(); - tbptt.segment_start_state = None; - tbptt.settings = None; - let need_full_adam = matches!(settings.optimizer, OptimizerKind::Adam) - && settings.scope.trains_non_head_params() - && online.full_adam.is_none(); - Some((settings, start_state, steps, need_full_adam)) - } - _ => None, - }, - None => None, + { + let Some(online) = self.online.as_mut() else { + return Ok(()); + }; + let has_steps = online + .full_tbptt + .as_ref() + .is_some_and(|tbptt| !tbptt.steps.is_empty()); + if !has_steps { + online.maybe_release_dead_full_tbptt(); + return Ok(()); } - }; - let Some((settings, start_state, steps, need_full_adam)) = extracted else { - return Ok(()); + } + + let need_full_adam = { + let Some(online) = self.online.as_ref() else { + return Ok(()); + }; + let Some(tbptt) = online.full_tbptt.as_ref() else { + return Ok(()); + }; + let settings = tbptt + .settings + .ok_or_else(|| anyhow::anyhow!("rwkv full tbptt settings are missing"))?; + if !tbptt.segment_start_state_valid { + bail!("rwkv full tbptt segment start is missing"); + } + matches!(settings.optimizer, OptimizerKind::Adam) + && settings.scope.trains_non_head_params() + && online.full_adam.is_none() }; if need_full_adam { @@ -1117,36 +1245,63 @@ impl Compressor { let Some(online) = self.online.as_mut() else { return Ok(()); }; + let Some(tbptt) = online.full_tbptt.as_mut() else { + return Ok(()); + }; + let settings = tbptt + .settings + .take() + .ok_or_else(|| anyhow::anyhow!("rwkv full tbptt settings are missing"))?; + if !tbptt.segment_start_state_valid { + bail!("rwkv full tbptt segment start is missing"); + } + let OnlineRuntime { + adam_t, + full_adam, + out_bias, + adam_m, + adam_v, + .. + } = online; + let replay_workspace = tbptt + .replay_workspace + .get_or_insert_with(|| rwkv7::TbpttReplayWorkspace::new(model)); model.online_train_segment_tbptt( &mut self.scratch, - &start_state, - &steps, + replay_workspace, + &tbptt.segment_start_state, + &tbptt.steps, settings.scope, settings.optimizer, settings.lr, settings.clip, TBPTT_REPLAY_CHUNK, - &mut online.adam_t, - online.full_adam.as_mut(), + adam_t, + full_adam.as_mut(), if settings.scope.bias { - Some(online.out_bias.as_mut_slice()) + Some(out_bias.as_mut_slice()) } else { None }, if settings.scope.bias { - online.adam_m.as_deref_mut() + adam_m.as_deref_mut() } else { None }, if settings.scope.bias { - online.adam_v.as_deref_mut() + adam_v.as_deref_mut() } else { None }, &mut self.state, )?; + tbptt.steps.clear(); + tbptt.segment_start_state_valid = false; let bias = self.online.as_ref().map(|o| o.out_bias.as_slice()); Self::logits_to_pdf(self.scratch.logits(), bias, &mut self.pdf_buffer); + if let Some(online) = self.online.as_mut() { + online.maybe_release_dead_full_tbptt(); + } Ok(()) } @@ -1186,13 +1341,16 @@ impl Compressor { let Some(input_token) = tbptt.pending_input_token.take() else { return Ok(()); }; - let input_pre_state = tbptt - .pending_input_pre_state - .take() - .ok_or_else(|| anyhow::anyhow!("rwkv full tbptt pending pre-state is missing"))?; + if !tbptt.pending_input_pre_state_valid { + bail!("rwkv full tbptt pending pre-state is missing"); + } if tbptt.steps.is_empty() { - tbptt.segment_start_state = Some(input_pre_state); + tbptt + .segment_start_state + .copy_from(&tbptt.pending_input_pre_state); + tbptt.segment_start_state_valid = true; } + tbptt.pending_input_pre_state_valid = false; tbptt.settings = Some(settings); tbptt.steps.push((input_token, target_symbol)); tbptt.steps.len() >= settings.bptt.max(1) @@ -1402,6 +1560,7 @@ impl Compressor { } fn resolve_online_train_action( + model_cfg: &rwkv7::Config, online: &mut OnlineRuntime, ) -> Result<(OptimizerKind, f32, u64, rwkv7::TrainScopeMask, usize, f32)> { let mut optimizer = match online.cfg.train_mode { @@ -1418,7 +1577,7 @@ impl Compressor { let mut bptt = 1usize; let mut clip = 0.0f32; - if let Some(action) = online.next_policy_action()? { + if let Some(action) = online.next_policy_action(model_cfg)? { match action { PolicyAction::Infer => { scope = rwkv7::TrainScopeMask::default(); @@ -1430,6 +1589,9 @@ impl Compressor { bptt = train.hyper.bptt.max(1); clip = train.hyper.clip.max(0.0); scope = scope_from_train_action(&train); + if scope.trains_non_head_params() { + online.ensure_full_tbptt_runtime(model_cfg); + } } } } @@ -1439,25 +1601,25 @@ impl Compressor { /// Apply one online update using externally supplied predictive PDF. pub fn online_update_from_pdf(&mut self, symbol: u8, pdf: &[f64]) -> Result<()> { - self.online_update_with_pdf(symbol, pdf) + self.online_update_with_pdf(symbol, PdfSource::External(pdf)) } #[inline] /// Update online state from `pdf`, then advance model state with `symbol`. pub fn observe_symbol_from_pdf(&mut self, symbol: u8, pdf: &[f64]) -> Result<()> { - self.online_update_with_pdf(symbol, pdf)?; + self.online_update_with_pdf(symbol, PdfSource::External(pdf))?; self.refresh_current_pdf(symbol as u32); Ok(()) } - fn online_update_with_pdf(&mut self, symbol: u8, pdf: &[f64]) -> Result<()> { + fn online_update_with_pdf(&mut self, symbol: u8, pdf_source: PdfSource<'_>) -> Result<()> { let (optimizer, lr, stride_hit, scope, bptt, clip) = { let Some(online) = self.online.as_mut() else { return Ok(()); }; online.tokens_processed = online.tokens_processed.saturating_add(1); let (optimizer, lr, stride, scope, bptt, clip) = - Self::resolve_online_train_action(online)?; + Self::resolve_online_train_action(self.model.config(), online)?; let mut stride_hit = false; if scope.trains_any_params() { online.policy_train_steps = online.policy_train_steps.saturating_add(1); @@ -1472,7 +1634,10 @@ impl Compressor { && let Some(tbptt) = online.full_tbptt.as_mut() { tbptt.pending_input_token = None; - tbptt.pending_input_pre_state = None; + tbptt.pending_input_pre_state_valid = false; + } + if let Some(online) = self.online.as_mut() { + online.maybe_release_dead_full_tbptt(); } return Ok(()); } @@ -1488,14 +1653,17 @@ impl Compressor { if !scope.trains_non_head_params() { let hidden = self.scratch.lm_head_input().to_vec(); - let pdf_snapshot = pdf.to_vec(); + let pdf_snapshot = match pdf_source { + PdfSource::External(pdf) => pdf.to_vec(), + PdfSource::CurrentBuffer => self.pdf_buffer.clone(), + }; self.flush_full_tbptt_segment()?; let Some(online) = self.online.as_mut() else { return Ok(()); }; if let Some(tbptt) = online.full_tbptt.as_mut() { tbptt.pending_input_token = None; - tbptt.pending_input_pre_state = None; + tbptt.pending_input_pre_state_valid = false; } let model = Arc::make_mut(&mut self.model); apply_online_lm_head_update( @@ -1510,6 +1678,9 @@ impl Compressor { scope.bias, clip, ); + if let Some(online) = self.online.as_mut() { + online.maybe_release_dead_full_tbptt(); + } return Ok(()); } @@ -1524,8 +1695,7 @@ impl Compressor { } fn online_update_from_current_pdf(&mut self, symbol: u8) -> Result<()> { - let pdf_snapshot = self.pdf_buffer.clone(); - self.online_update_with_pdf(symbol, &pdf_snapshot) + self.online_update_with_pdf(symbol, PdfSource::CurrentBuffer) } #[inline] @@ -1588,9 +1758,13 @@ impl Compressor { if opt_sidecar.exists() { let _ = fs::remove_file(&opt_sidecar); } + let canonical_method = canonical_method_string(&MethodSpec::File { + path: model_path.to_path_buf(), + policy: None, + })?; json!({ "version": 1, - "method": format!("file:{}", model_path.display()), + "method": canonical_method, "training_mode": "none", "tokens_processed": 0, }) @@ -1625,11 +1799,15 @@ impl Compressor { .map(|x| x.as_f64().unwrap_or(0.0) as f32) .collect::>() }); + let default_method = canonical_method_string(&MethodSpec::File { + path: model_path.to_path_buf(), + policy: None, + })?; let method = v .get("method") .and_then(|m| m.as_str()) .map(|s| s.to_string()) - .unwrap_or_else(|| format!("file:{}", model_path.display())); + .unwrap_or(default_method); let has_full_adam = v .get("has_full_adam") .and_then(|x| x.as_bool()) @@ -1705,13 +1883,7 @@ impl Compressor { lm_head_adam_m: parse_vec_f32("lm_head_adam_m"), lm_head_adam_v: parse_vec_f32("lm_head_adam_v"), adam_t: v.get("adam_t").and_then(|x| x.as_u64()).unwrap_or(0) as usize, - full_tbptt: needs_full_trace.then(|| FullTbpttRuntime { - pending_input_token: None, - pending_input_pre_state: None, - segment_start_state: None, - steps: Vec::new(), - settings: None, - }), + full_tbptt: needs_full_trace.then(|| FullTbpttRuntime::new(self.model.config())), }); let opt_sidecar = optimizer_sidecar_path(model_path); if opt_sidecar.exists() { @@ -1729,14 +1901,17 @@ impl Compressor { && online.policy.is_some() { let train_steps = online.policy_train_steps; - online.prepare_policy_stream(online.policy_stream_total)?; + online.prepare_policy_stream(self.model.config(), online.policy_stream_total)?; online.policy_train_steps = train_steps; if let Some(rt) = online.policy_runtime.as_mut() { rt.set_cursor(cursor); } } - self.scratch - .set_capture_train_trace(self.online.as_ref().is_some_and(|o| o.needs_full_trace)); + self.scratch.set_capture_train_trace( + self.online + .as_ref() + .is_some_and(OnlineRuntime::should_capture_full_trace_for_next_step), + ); } Ok(()) } @@ -2305,6 +2480,35 @@ mod tests { std::fs::remove_file(&p).ok(); } + #[test] + fn parse_method_spec_accepts_raw_semicolons_in_file_paths() { + match parse_method_spec("file:/tmp/rwkv;v1.safetensors").expect("file path parse") { + MethodSpec::File { path, policy } => { + assert_eq!(path, PathBuf::from("/tmp/rwkv;v1.safetensors")); + assert!(policy.is_none()); + } + _ => panic!("expected file method"), + } + } + + #[test] + fn canonical_method_string_escapes_delimiter_bearing_file_paths() { + let method = canonical_method_string(&MethodSpec::File { + path: PathBuf::from("/tmp/rwkv;policy:model%v1.safetensors"), + policy: None, + }) + .expect("delimiter-bearing file path should canonicalize"); + assert_eq!(method, "file:/tmp/rwkv%3Bpolicy:model%25v1.safetensors"); + + match parse_method_spec(&method).expect("canonical method should parse") { + MethodSpec::File { path, policy } => { + assert_eq!(path, PathBuf::from("/tmp/rwkv;policy:model%v1.safetensors")); + assert!(policy.is_none()); + } + _ => panic!("expected file method"), + } + } + #[test] fn test_parse_method_spec_rejects_unknown_cfg_key() { let err = @@ -2549,6 +2753,63 @@ mod tests { std::fs::remove_file(&after_path).ok(); } + #[test] + fn test_online_infer_tail_releases_full_tbptt_runtime() { + let method = "cfg:hidden=64,layers=1,intermediate=64,decay_rank=8,a_rank=8,v_rank=8,g_rank=8,seed=37,train=adam,lr=0.0008,stride=1;policy:schedule=0..2:train(scope=all,opt=adam,lr=0.0008,stride=1,bptt=2,clip=0,momentum=0.9)|2..100:infer"; + let mut c = Compressor::new_from_method(method).unwrap(); + c.reset_and_prime(); + let score = c.cross_entropy_from_current(b"abcdef").unwrap(); + assert!(score.is_finite()); + assert!( + c.online + .as_ref() + .and_then(|online| online.full_tbptt.as_ref()) + .is_none(), + "expected full tbptt runtime to be released once the stream tail is pure inference" + ); + } + + #[test] + fn test_online_policy_restart_recreates_full_tbptt_after_release() { + let method = "cfg:hidden=64,layers=1,intermediate=64,decay_rank=8,a_rank=8,v_rank=8,g_rank=8,seed=39,train=adam,lr=0.0008,stride=1;policy:schedule=0..2:train(scope=all,opt=adam,lr=0.0008,stride=1,bptt=2,clip=0,momentum=0.9)|2..100:infer"; + let mut c = Compressor::new_from_method(method).unwrap(); + c.reset_and_prime(); + let score = c.cross_entropy_from_current(b"abcdef").unwrap(); + assert!(score.is_finite()); + assert!( + c.online + .as_ref() + .and_then(|online| online.full_tbptt.as_ref()) + .is_none() + ); + + c.restart_online_policy_stream(Some(6)).unwrap(); + + assert!( + c.online + .as_ref() + .and_then(|online| online.full_tbptt.as_ref()) + .is_some(), + "expected a fresh policy stream to recreate the full tbptt runtime" + ); + } + + #[test] + fn test_head_only_tail_releases_full_tbptt_runtime() { + let method = "cfg:hidden=64,layers=1,intermediate=64,decay_rank=8,a_rank=8,v_rank=8,g_rank=8,seed=41,train=adam,lr=0.0008,stride=1;policy:schedule=0..2:train(scope=all,opt=adam,lr=0.0008,stride=1,bptt=2,clip=0,momentum=0.9)|2..100:train(scope=head+bias,opt=adam,lr=0.0008,stride=1,bptt=1,clip=0,momentum=0.9)"; + let mut c = Compressor::new_from_method(method).unwrap(); + c.reset_and_prime(); + let score = c.cross_entropy_from_current(b"abcdef").unwrap(); + assert!(score.is_finite()); + assert!( + c.online + .as_ref() + .and_then(|online| online.full_tbptt.as_ref()) + .is_none(), + "expected full tbptt runtime to be released once only head-only updates remain" + ); + } + #[test] fn test_online_export_reload_roundtrip_preserves_full_adam_resume() { let method = "cfg:hidden=64,layers=1,intermediate=64,decay_rank=8,a_rank=8,v_rank=8,g_rank=8,seed=31,train=adam,lr=0.0008,stride=1;policy:schedule=0..100:train(scope=all,opt=adam,lr=0.0008,stride=1,bptt=1,clip=0,momentum=0.9)"; @@ -2611,4 +2872,156 @@ mod tests { .unwrap(); assert!(score.is_finite()); } + + #[test] + fn conditional_chain_matches_single_prefix_api() { + let method = "cfg:hidden=64,layers=1,intermediate=64,decay_rank=8,a_rank=8,v_rank=8,g_rank=8,seed=47,train=none,lr=0.001,stride=1;policy:schedule=0..100:infer"; + let prefix_a = b"alpha "; + let prefix_b = b"beta "; + let data = b"gamma delta"; + + let mut chained = Compressor::new_from_method(method).unwrap(); + let chain_score = chained + .cross_entropy_conditional_chain(&[prefix_a.as_slice(), prefix_b.as_slice()], data) + .unwrap(); + + let mut single = Compressor::new_from_method(method).unwrap(); + let mut merged_prefix = Vec::new(); + merged_prefix.extend_from_slice(prefix_a); + merged_prefix.extend_from_slice(prefix_b); + let single_score = single + .cross_entropy_conditional(&merged_prefix, data) + .unwrap(); + + assert!( + (chain_score - single_score).abs() < 1e-12, + "chain and single-prefix APIs should agree" + ); + } + + #[test] + fn joint_cross_entropy_aligned_min_is_symmetric_and_empty_safe() { + let method = "cfg:hidden=64,layers=1,intermediate=64,decay_rank=8,a_rank=8,v_rank=8,g_rank=8,seed=53,train=none,lr=0.001,stride=1;policy:schedule=0..100:infer"; + let x = b"abracadabra"; + let y = b"alakazam___"; + + let mut a = Compressor::new_from_method(method).unwrap(); + let xy = a.joint_cross_entropy_aligned_min(x, y).unwrap(); + assert!(xy.is_finite()); + + let mut b = Compressor::new_from_method(method).unwrap(); + let yx = b.joint_cross_entropy_aligned_min(y, x).unwrap(); + assert!((xy - yx).abs() < 1e-12, "joint score should be symmetric"); + + let mut empty = Compressor::new_from_method(method).unwrap(); + assert_eq!(empty.joint_cross_entropy_aligned_min(b"", y).unwrap(), 0.0); + } + + #[test] + fn pdf_forwarding_and_cached_pdf_views_stay_in_sync() { + let method = "cfg:hidden=64,layers=1,intermediate=64,decay_rank=8,a_rank=8,v_rank=8,g_rank=8,seed=59,train=none,lr=0.001,stride=1;policy:schedule=0..100:infer"; + let mut forward = Compressor::new_from_method(method).unwrap(); + let mut cached = Compressor::new_from_method(method).unwrap(); + + forward.reset_and_prime(); + cached.reset_and_prime(); + + let mut forwarded_pdf = vec![0.0; forward.vocab_size()]; + forward.forward_to_pdf(u32::from(b'a'), &mut forwarded_pdf); + + cached.forward_to_internal_pdf(u32::from(b'a')); + let mut cached_pdf = vec![0.0; cached.vocab_size()]; + cached.copy_current_pdf_to(&mut cached_pdf); + + assert_eq!(forwarded_pdf.len(), 256); + let sum: f64 = forwarded_pdf.iter().sum(); + assert!((sum - 1.0).abs() < 1e-9, "pdf should remain normalized"); + for (lhs, rhs) in forwarded_pdf.iter().zip(cached_pdf.iter()) { + assert!( + (lhs - rhs).abs() < 1e-12, + "cached and forwarded pdfs diverged" + ); + } + } + + #[test] + fn online_bias_accessors_and_adaptation_flags_match_policy() { + let infer_method = "cfg:hidden=64,layers=1,intermediate=64,decay_rank=8,a_rank=8,v_rank=8,g_rank=8,seed=61,train=none,lr=0.001,stride=1;policy:schedule=0..100:infer"; + let train_method = "cfg:hidden=64,layers=1,intermediate=64,decay_rank=8,a_rank=8,v_rank=8,g_rank=8,seed=67,train=adam,lr=0.0008,stride=1;policy:schedule=0..100:train(scope=head,opt=adam,lr=0.0008,stride=1,bptt=1,clip=0,momentum=0.9)"; + + let mut infer = Compressor::new_from_method(infer_method).unwrap(); + infer.reset_and_prime(); + infer.forward_to_internal_pdf(u32::from(b'z')); + let infer_bias = infer + .online_bias_snapshot() + .expect("online method should expose bias vector"); + let infer_bias_slice = infer + .online_bias_slice() + .expect("online method should expose bias slice"); + assert_eq!(infer_bias.as_slice(), infer_bias_slice); + assert!( + !infer.can_adapt_online(), + "infer-only policy should not adapt" + ); + + let train = Compressor::new_from_method(train_method).unwrap(); + assert!(train.can_adapt_online(), "train policy should adapt"); + } + + #[test] + fn compress_size_chain_matches_materialized_output_and_roundtrips() { + let method = "cfg:hidden=64,layers=1,intermediate=64,decay_rank=8,a_rank=8,v_rank=8,g_rank=8,seed=71,train=none,lr=0.001,stride=1;policy:schedule=0..100:infer"; + let parts: [&[u8]; 3] = [b"chain ", b"compression ", b"fixture"]; + + let mut sized = Compressor::new_from_method(method).unwrap(); + let predicted = sized.compress_size_chain(&parts, CoderType::AC).unwrap(); + + let mut materialized = Compressor::new_from_method(method).unwrap(); + let mut buf = Vec::new(); + materialized + .compress_chain_into(&parts, CoderType::AC, &mut buf) + .unwrap(); + assert_eq!(predicted, buf.len() as u64); + + let mut decoder = Compressor::new_from_method(method).unwrap(); + let decoded = decoder.decompress(&buf).unwrap(); + assert_eq!(decoded, b"chain compression fixture"); + } + + #[test] + fn decompress_reports_crc_and_truncation_corruption() { + let method = "cfg:hidden=64,layers=1,intermediate=64,decay_rank=8,a_rank=8,v_rank=8,g_rank=8,seed=73,train=none,lr=0.001,stride=1;policy:schedule=0..100:infer"; + let data = b"corruption-fixture"; + + let mut ac = Compressor::new_from_method(method).unwrap(); + let mut ac_bytes = ac.compress(data, CoderType::AC).unwrap(); + let crc_offset = Header::SIZE - 4; + ac_bytes[crc_offset] ^= 0x01; + + let mut ac_decoder = Compressor::new_from_method(method).unwrap(); + let ac_err = ac_decoder + .decompress(&ac_bytes) + .expect_err("corrupt AC stream must fail"); + let ac_msg = format!("{ac_err:#}"); + assert!( + ac_msg.contains("CRC32 mismatch"), + "unexpected AC corruption error: {ac_msg}" + ); + + let mut rans = Compressor::new_from_method(method).unwrap(); + let rans_bytes = rans.compress(data, CoderType::RANS).unwrap(); + let truncated = &rans_bytes[..rans_bytes.len() - 1]; + + let mut rans_decoder = Compressor::new_from_method(method).unwrap(); + let rans_err = rans_decoder + .decompress(truncated) + .expect_err("truncated rANS stream must fail"); + let rans_msg = format!("{rans_err:#}"); + assert!( + rans_msg.contains("Truncated block data") + || rans_msg.contains("rANS data too short") + || rans_msg.contains("failed to fill whole buffer"), + "unexpected rANS truncation error: {rans_msg}" + ); + } } diff --git a/src/backends/rwkvzip/rwkv7/kernel.rs b/crates/infotheory/src/backends/rwkvzip/rwkv7/kernel.rs similarity index 96% rename from src/backends/rwkvzip/rwkv7/kernel.rs rename to crates/infotheory/src/backends/rwkvzip/rwkv7/kernel.rs index 7de563b9..29778c44 100644 --- a/src/backends/rwkvzip/rwkv7/kernel.rs +++ b/crates/infotheory/src/backends/rwkvzip/rwkv7/kernel.rs @@ -6,6 +6,7 @@ #![allow(dead_code)] +use crate::backends::fixed_gemv; use wide::f32x8; const LANES: usize = 8; @@ -57,6 +58,9 @@ pub unsafe fn dot_avx(a: *const f32, b: *const f32, len: usize) -> f32 { /// Matrix-vector multiply: y = A @ x where A is (rows, cols), x is (cols,), y is (rows,). #[inline(always)] pub unsafe fn gemv_avx(a: *const f32, x: *const f32, y: *mut f32, rows: usize, cols: usize) { + if unsafe { fixed_gemv::try_gemv(a, x, y, rows, cols) } { + return; + } let mut r = 0; while r + 4 <= rows { @@ -111,6 +115,9 @@ pub unsafe fn gemv_avx(a: *const f32, x: *const f32, y: *mut f32, rows: usize, c /// Matrix-vector multiply with transposed matrix: y = A^T @ x. #[inline(always)] pub unsafe fn gemv_t_avx(a: *const f32, x: *const f32, y: *mut f32, rows: usize, cols: usize) { + if unsafe { fixed_gemv::try_gemv_t(a, x, y, rows, cols) } { + return; + } let mut c = 0; while c + LANES <= cols { store8(y.add(c), f32x8::ZERO); @@ -968,6 +975,41 @@ mod tests { let yt_ref = gemv_t_scalar(&a, &x_t, rows, cols); assert_close_slice(&yt, &yt_ref, 2.5e-5); + let rows_fixed = 64; + let cols_fixed = 64; + let mut a_fixed = vec![0.0; rows_fixed * cols_fixed]; + let mut x_fixed = vec![0.0; cols_fixed]; + let mut x_t_fixed = vec![0.0; rows_fixed]; + fill_centered(&mut a_fixed, &mut rng, 0.75); + fill_centered(&mut x_fixed, &mut rng, 0.5); + fill_centered(&mut x_t_fixed, &mut rng, 0.5); + + let mut y_fixed = vec![0.0; rows_fixed]; + unsafe { + gemv_avx( + a_fixed.as_ptr(), + x_fixed.as_ptr(), + y_fixed.as_mut_ptr(), + rows_fixed, + cols_fixed, + ) + }; + let y_fixed_ref = gemv_scalar(&a_fixed, &x_fixed, rows_fixed, cols_fixed); + assert_close_slice(&y_fixed, &y_fixed_ref, 2.5e-5); + + let mut yt_fixed = vec![0.0; cols_fixed]; + unsafe { + gemv_t_avx( + a_fixed.as_ptr(), + x_t_fixed.as_ptr(), + yt_fixed.as_mut_ptr(), + rows_fixed, + cols_fixed, + ) + }; + let yt_fixed_ref = gemv_t_scalar(&a_fixed, &x_t_fixed, rows_fixed, cols_fixed); + assert_close_slice(&yt_fixed, &yt_fixed_ref, 2.5e-5); + let ln_len = 137; let mut ln_x = vec![0.0; ln_len]; let mut ln_w = vec![0.0; ln_len]; diff --git a/src/backends/rwkvzip/rwkv7/mod.rs b/crates/infotheory/src/backends/rwkvzip/rwkv7/mod.rs similarity index 94% rename from src/backends/rwkvzip/rwkv7/mod.rs rename to crates/infotheory/src/backends/rwkvzip/rwkv7/mod.rs index 7ba203a9..703e9905 100644 --- a/src/backends/rwkvzip/rwkv7/mod.rs +++ b/crates/infotheory/src/backends/rwkvzip/rwkv7/mod.rs @@ -18,6 +18,7 @@ mod tensor; mod weights; pub use model::ScratchBuffers; +pub(crate) use model::TbpttReplayWorkspace; pub use model::{Config, FullAdamState, Model, State, TrainScopeMask}; pub use profiling::{LayerProfiler, LayerTiming, NullProfiler, ProfilerSink}; pub use tensor::{Tensor1D, Tensor2D, TensorView1D, TensorView2D}; diff --git a/src/backends/rwkvzip/rwkv7/model.rs b/crates/infotheory/src/backends/rwkvzip/rwkv7/model.rs similarity index 92% rename from src/backends/rwkvzip/rwkv7/model.rs rename to crates/infotheory/src/backends/rwkvzip/rwkv7/model.rs index cc1c4e77..c350b106 100644 --- a/src/backends/rwkvzip/rwkv7/model.rs +++ b/crates/infotheory/src/backends/rwkvzip/rwkv7/model.rs @@ -113,6 +113,12 @@ impl LayerState { ffn_x_prev: Tensor1D::zeros(cfg.hidden_size), } } + + fn copy_from(&mut self, other: &Self) { + self.att_x_prev.copy_from(&other.att_x_prev); + self.att_state.copy_from(&other.att_state); + self.ffn_x_prev.copy_from(&other.ffn_x_prev); + } } /// Full model state. @@ -146,6 +152,15 @@ impl State { layer.ffn_x_prev.zero(); } } + + pub(crate) fn copy_from(&mut self, other: &Self) { + debug_assert_eq!(self.layers.len(), other.layers.len()); + self.v_first.clone_from(&other.v_first); + self.v_first_set = other.v_first_set; + for (dst, src) in self.layers.iter_mut().zip(other.layers.iter()) { + dst.copy_from(src); + } + } } /// Weights for a single attention layer. @@ -420,6 +435,62 @@ struct FullGradState { blocks: Vec, } +impl FullGradState { + fn zero(&mut self) { + self.embeddings.zero(); + self.ln_out_w.zero(); + self.ln_out_b.zero(); + self.lm_head.zero(); + for block in &mut self.blocks { + if let Some(t) = block.pre_norm_w.as_mut() { + t.zero(); + } + if let Some(t) = block.pre_norm_b.as_mut() { + t.zero(); + } + block.attn_norm_w.zero(); + block.attn_norm_b.zero(); + block.ffn_norm_w.zero(); + block.ffn_norm_b.zero(); + + block.attn.x_r.zero(); + block.attn.x_w.zero(); + block.attn.x_k.zero(); + block.attn.x_v.zero(); + block.attn.x_a.zero(); + block.attn.x_g.zero(); + block.attn.rkv_proj.zero(); + block.attn.o_proj.zero(); + block.attn.w1.zero(); + block.attn.w2.zero(); + block.attn.w0.zero(); + block.attn.a1.zero(); + block.attn.a2.zero(); + block.attn.a0.zero(); + if let Some(t) = block.attn.v1.as_mut() { + t.zero(); + } + if let Some(t) = block.attn.v2.as_mut() { + t.zero(); + } + if let Some(t) = block.attn.v0.as_mut() { + t.zero(); + } + block.attn.g1.zero(); + block.attn.g2.zero(); + block.attn.k_k.zero(); + block.attn.k_a.zero(); + block.attn.r_k.zero(); + block.attn.g_norm_w.zero(); + block.attn.g_norm_b.zero(); + + block.ffn.x_k.zero(); + block.ffn.key_w.zero(); + block.ffn.value_w.zero(); + } + } +} + struct AdamStep { lr: f32, clip: f32, @@ -549,6 +620,14 @@ impl TokenTrainTrace { layers: scratch.train_trace_layers.clone(), } } + + fn clone_from_scratch(&mut self, scratch: &ScratchBuffers) { + self.token = scratch.train_token; + self.x.clone_from(&scratch.x); + self.x_normed.clone_from(&scratch.x_normed); + self.v_first.clone_from(&scratch.train_v_first); + self.layers.clone_from(&scratch.train_trace_layers); + } } #[derive(Clone)] @@ -639,6 +718,40 @@ pub struct ScratchBuffers { capture_train_trace: bool, } +pub(crate) struct TbpttReplayWorkspace { + grads: FullGradState, + recurrent: RecurrentGradState, + bias_grad: Vec, + checkpoint_state: State, + replay_state: State, + checkpoints: Vec, + step_states: Vec, + step_traces: Vec, + step_pdfs: Vec, +} + +impl TbpttReplayWorkspace { + pub(crate) fn new(model: &Model) -> Self { + Self { + grads: model.new_full_grad_state(), + recurrent: model.new_recurrent_grad_state(), + bias_grad: Vec::new(), + checkpoint_state: model.new_state(), + replay_state: model.new_state(), + checkpoints: Vec::new(), + step_states: Vec::new(), + step_traces: Vec::new(), + step_pdfs: Vec::new(), + } + } +} + +fn ensure_cloned_len(buf: &mut Vec, len: usize, template: &T) { + if buf.len() < len { + buf.resize_with(len, || template.clone()); + } +} + impl ScratchBuffers { /// Allocate reusable per-token scratch buffers sized for `cfg`. pub fn new(cfg: &Config) -> Self { @@ -750,7 +863,12 @@ impl Model { /// Load model from safetensors file. pub fn load>(path: P) -> Result { - let weights = Weights::load(path.as_ref()).context("Failed to load model weights")?; + let weights = Weights::load(path.as_ref()).with_context(|| { + format!( + "Failed to load model weights from {}", + path.as_ref().display() + ) + })?; // Infer config from weights let emb = weights.require("model.embeddings.weight")?; @@ -2297,7 +2415,7 @@ impl Model { Ok(()) } - #[allow(clippy::needless_range_loop)] + #[allow(clippy::needless_range_loop, clippy::too_many_arguments)] fn accumulate_token_step_gradients( &self, scratch: &mut ScratchBuffers, @@ -3137,9 +3255,10 @@ impl Model { #[allow(clippy::too_many_arguments)] /// Run one TBPTT training segment and write the resulting live state. - pub fn online_train_segment_tbptt( + pub(crate) fn online_train_segment_tbptt( &mut self, scratch: &mut ScratchBuffers, + workspace: &mut TbpttReplayWorkspace, start_state: &State, steps: &[(u32, u8)], scope: TrainScopeMask, @@ -3155,74 +3274,106 @@ impl Model { live_state_out: &mut State, ) -> Result<()> { if steps.is_empty() { - *live_state_out = start_state.clone(); + live_state_out.copy_from(start_state); return Ok(()); } let grad_scale = 1.0f32 / (steps.len() as f32); let chunk = replay_chunk.max(1).min(steps.len().max(1)); - let mut grads = self.new_full_grad_state(); - let mut recurrent = self.new_recurrent_grad_state(); + let TbpttReplayWorkspace { + grads, + recurrent, + bias_grad: workspace_bias_grad, + checkpoint_state, + replay_state, + checkpoints, + step_states, + step_traces, + step_pdfs, + } = workspace; + grads.zero(); recurrent.zero(); - let mut bias_grad = out_bias.as_deref().map(|b| vec![0.0f32; b.len()]); + let mut bias_grad = match out_bias.as_deref().map(<[f32]>::len) { + Some(len) => { + if workspace_bias_grad.len() != len { + workspace_bias_grad.resize(len, 0.0); + } else { + workspace_bias_grad.fill(0.0); + } + Some(workspace_bias_grad.as_mut_slice()) + } + None => None, + }; { - let mut checkpoints = Vec::::new(); - let mut checkpoint_state = start_state.clone(); + checkpoint_state.copy_from(start_state); + let checkpoint_count = steps.len().div_ceil(chunk); + ensure_cloned_len(checkpoints, checkpoint_count, start_state); + checkpoints.truncate(checkpoint_count); scratch.set_capture_train_trace(false); - for chunk_start in (0..steps.len()).step_by(chunk) { - checkpoints.push(checkpoint_state.clone()); + for (checkpoint_idx, chunk_start) in (0..steps.len()).step_by(chunk).enumerate() { + checkpoints[checkpoint_idx].copy_from(checkpoint_state); let chunk_end = (chunk_start + chunk).min(steps.len()); for &(input_token, _) in &steps[chunk_start..chunk_end] { - self.forward(scratch, input_token, &mut checkpoint_state); + self.forward(scratch, input_token, checkpoint_state); } } - for chunk_idx in (0..checkpoints.len()).rev() { + for chunk_idx in (0..checkpoint_count).rev() { let chunk_start = chunk_idx * chunk; let chunk_end = (chunk_start + chunk).min(steps.len()); - let mut state = checkpoints[chunk_idx].clone(); - let mut step_states = Vec::::with_capacity(chunk_end - chunk_start + 1); - let mut step_traces = - Vec::::with_capacity(chunk_end - chunk_start); - let mut step_pdfs = - Vec::>::with_capacity(chunk_end.saturating_sub(chunk_start)); - step_states.push(state.clone()); + let chunk_steps = chunk_end - chunk_start; + let checkpoint = &checkpoints[chunk_idx]; + replay_state.copy_from(checkpoint); + let state_count = chunk_steps + 1; + ensure_cloned_len(step_states, state_count, checkpoint); + step_states.truncate(state_count); + if step_traces.len() < chunk_steps { + step_traces.resize_with(chunk_steps, || TokenTrainTrace::from_scratch(scratch)); + } + step_traces.truncate(chunk_steps); + let pdf_stride = self.cfg.vocab_size; + step_pdfs.resize(chunk_steps.saturating_mul(pdf_stride), 0.0); + step_states[0].copy_from(replay_state); - for &(input_token, _) in &steps[chunk_start..chunk_end] { + for (local_idx, &(input_token, _)) in + steps[chunk_start..chunk_end].iter().enumerate() + { scratch.set_capture_train_trace(true); - let logits = self.forward(scratch, input_token, &mut state); - let mut pdf = vec![0.0f64; self.cfg.vocab_size]; + let logits = self.forward(scratch, input_token, replay_state); + let pdf_lo = local_idx * pdf_stride; + let pdf_hi = pdf_lo + pdf_stride; super::super::softmax_pdf_floor_with_bias( logits, out_bias.as_deref(), - &mut pdf, + &mut step_pdfs[pdf_lo..pdf_hi], ); - step_pdfs.push(pdf); - step_traces.push(TokenTrainTrace::from_scratch(scratch)); - step_states.push(state.clone()); + step_traces[local_idx].clone_from_scratch(scratch); + step_states[local_idx + 1].copy_from(replay_state); } - for local_idx in (0..step_traces.len()).rev() { + for local_idx in (0..chunk_steps).rev() { let (_, target_symbol) = steps[chunk_start + local_idx]; + let pdf_lo = local_idx * pdf_stride; + let pdf_hi = pdf_lo + pdf_stride; self.accumulate_token_step_gradients( scratch, &step_traces[local_idx], &step_states[local_idx + 1], target_symbol, - &step_pdfs[local_idx], + &step_pdfs[pdf_lo..pdf_hi], grad_scale, scope, - &mut grads, + grads, bias_grad.as_deref_mut(), - &mut recurrent, + recurrent, )?; } } } self.apply_full_gradients( - &grads, + grads, scope, optimizer, lr, @@ -3236,7 +3387,7 @@ impl Model { )?; scratch.set_capture_train_trace(false); - *live_state_out = start_state.clone(); + live_state_out.copy_from(start_state); for &(input_token, _) in steps { self.forward(scratch, input_token, live_state_out); } @@ -6039,6 +6190,7 @@ fn init_const(t: &mut Tensor1D, value: f32) { #[cfg(test)] mod tests { use super::*; + use std::path::PathBuf; fn test_cfg() -> Config { Config { @@ -6057,6 +6209,14 @@ mod tests { } } + fn temp_path(prefix: &str, ext: &str) -> PathBuf { + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_nanos(); + std::env::temp_dir().join(format!("{prefix}_{}_{}.{}", std::process::id(), now, ext)) + } + fn softmax_loss(logits: &[f32], target: u8) -> f64 { let max_logit = logits .iter() @@ -6196,6 +6356,201 @@ mod tests { assert_eq!(cfg.head_dim, 64); } + #[test] + fn validate_rejects_invalid_config_shapes() { + let zero_vocab = Config { + vocab_size: 0, + ..test_cfg() + }; + assert!( + zero_vocab + .validate() + .expect_err("zero vocab must fail") + .to_string() + .contains("vocab_size must be > 0") + ); + + let bad_head_dim = Config { + head_dim: 32, + hidden_size: 32, + ..test_cfg() + }; + assert!( + bad_head_dim + .validate() + .expect_err("bad head_dim must fail") + .to_string() + .contains("head_dim must be 64") + ); + + let bad_hidden = Config { + hidden_size: 128, + ..test_cfg() + }; + assert!( + bad_hidden + .validate() + .expect_err("hidden mismatch must fail") + .to_string() + .contains("hidden_size must equal num_heads * head_dim") + ); + + let zero_layers = Config { + num_layers: 0, + ..test_cfg() + }; + assert!( + zero_layers + .validate() + .expect_err("zero layers must fail") + .to_string() + .contains("num_layers must be > 0") + ); + + let zero_intermediate = Config { + intermediate_size: 0, + ..test_cfg() + }; + assert!( + zero_intermediate + .validate() + .expect_err("zero intermediate must fail") + .to_string() + .contains("intermediate_size must be > 0") + ); + } + + #[test] + fn state_reset_clears_forward_mutation() { + let cfg = test_cfg(); + cfg.validate().expect("valid cfg"); + let model = Model::new_random(cfg.clone(), 0x5151).expect("random model"); + let mut state = model.new_state(); + let mut scratch = ScratchBuffers::new(&cfg); + + let _ = model.forward(&mut scratch, 7, &mut state); + let _ = model.forward(&mut scratch, 11, &mut state); + assert!(state.v_first_set, "forward pass should initialize v_first"); + assert!( + state.layers[0] + .att_state + .as_slice() + .iter() + .any(|&v| v != 0.0), + "forward pass should mutate recurrent state" + ); + + state.reset(); + assert!(!state.v_first_set); + assert!(state.v_first.as_slice().iter().all(|&v| v == 0.0)); + for layer in &state.layers { + assert!(layer.att_x_prev.as_slice().iter().all(|&v| v == 0.0)); + assert!(layer.att_state.as_slice().iter().all(|&v| v == 0.0)); + assert!(layer.ffn_x_prev.as_slice().iter().all(|&v| v == 0.0)); + } + } + + #[test] + fn train_scope_mask_reports_expected_semantics() { + let none = TrainScopeMask::default(); + assert!(!none.trains_non_head_params()); + assert!(!none.trains_any_params()); + + let head_only = TrainScopeMask { + head: true, + ..TrainScopeMask::default() + }; + assert!(!head_only.trains_non_head_params()); + assert!(head_only.trains_any_params()); + + let all = TrainScopeMask::all(); + assert!(all.embed); + assert!(all.pre_norm); + assert!(all.attn_norm); + assert!(all.ffn_norm); + assert!(all.attn); + assert!(all.ffn); + assert!(all.head); + assert!(all.bias); + assert!(all.trains_non_head_params()); + assert!(all.trains_any_params()); + } + + #[test] + fn save_load_safetensors_roundtrip_preserves_forward_bits() { + let cfg = Config { + num_layers: 2, + intermediate_size: 128, + decay_low_rank: 16, + a_low_rank: 16, + v_low_rank: 16, + g_low_rank: 32, + ..test_cfg() + }; + cfg.validate().expect("valid cfg"); + let model = Model::new_random(cfg.clone(), 0xBEEF_CAFE).expect("random model"); + let path = temp_path("rwkv_roundtrip", "safetensors"); + model.save_safetensors(&path).expect("save model"); + let loaded = Model::load(&path).expect("load model"); + + assert_eq!(loaded.config().vocab_size, model.config().vocab_size); + assert_eq!(loaded.config().hidden_size, model.config().hidden_size); + assert_eq!(loaded.config().num_layers, model.config().num_layers); + + let mut original_state = model.new_state(); + let mut loaded_state = loaded.new_state(); + let mut original_scratch = ScratchBuffers::new(&cfg); + let mut loaded_scratch = ScratchBuffers::new(&cfg); + for &token in &[0u32, 7, 31, 99, 255] { + let original_logits = model.forward(&mut original_scratch, token, &mut original_state); + let loaded_logits = loaded.forward(&mut loaded_scratch, token, &mut loaded_state); + for (&lhs, &rhs) in original_logits.iter().zip(loaded_logits.iter()) { + assert_eq!(lhs.to_bits(), rhs.to_bits()); + } + } + + std::fs::remove_file(path).ok(); + } + + #[test] + fn save_load_full_adam_roundtrip_preserves_selected_moments() { + let cfg = test_cfg(); + cfg.validate().expect("valid cfg"); + let model = Model::new_random(cfg, 0xACED).expect("random model"); + let mut adam = model.new_full_adam_state(); + adam.embeddings.m[0] = 1.25; + adam.embeddings.v[1] = 2.5; + adam.ln_out_w.m[2] = -3.0; + adam.blocks[0].attn.x_r.m[3] = 4.5; + adam.blocks[0].ffn.key_w.v[4] = 5.75; + + let path = temp_path("rwkv_adam", "safetensors"); + model + .save_full_adam_safetensors(&adam, &path) + .expect("save adam"); + let loaded = model.load_full_adam_safetensors(&path).expect("load adam"); + + assert_eq!( + loaded.embeddings.m[0].to_bits(), + adam.embeddings.m[0].to_bits() + ); + assert_eq!( + loaded.embeddings.v[1].to_bits(), + adam.embeddings.v[1].to_bits() + ); + assert_eq!(loaded.ln_out_w.m[2].to_bits(), adam.ln_out_w.m[2].to_bits()); + assert_eq!( + loaded.blocks[0].attn.x_r.m[3].to_bits(), + adam.blocks[0].attn.x_r.m[3].to_bits() + ); + assert_eq!( + loaded.blocks[0].ffn.key_w.v[4].to_bits(), + adam.blocks[0].ffn.key_w.v[4].to_bits() + ); + + std::fs::remove_file(path).ok(); + } + #[test] fn test_forward_deterministic_snapshot() { let cfg = Config { @@ -6390,6 +6745,7 @@ mod tests { let before = segment_loss(&model, &cfg, &steps); let mut scratch = ScratchBuffers::new(&cfg); + let mut workspace = TbpttReplayWorkspace::new(&model); let start_state = model.new_state(); let mut live_state = model.new_state(); let mut adam_t = 0usize; @@ -6407,6 +6763,7 @@ mod tests { model .online_train_segment_tbptt( &mut scratch, + &mut workspace, &start_state, &steps, scope, @@ -6429,4 +6786,119 @@ mod tests { "expected SGD TBPTT step to reduce mean loss: before={before} after={after}" ); } + + #[test] + fn head_only_bptt1_update_succeeds_without_full_trace() { + let cfg = test_cfg(); + cfg.validate().expect("valid cfg"); + let mut model = Model::new_random(cfg.clone(), 0xAAAA_5555).expect("random model"); + let mut scratch = ScratchBuffers::new(&cfg); + let mut state = model.new_state(); + scratch.set_capture_train_trace(false); + + let logits = model.forward(&mut scratch, 9, &mut state).to_vec(); + let mut pdf = vec![0.0f64; cfg.vocab_size]; + super::super::super::softmax_pdf_floor_with_bias(&logits, None, &mut pdf); + let before = model.lm_head_weights()[0]; + let mut adam_t = 0usize; + let scope = TrainScopeMask { + head: true, + ..TrainScopeMask::default() + }; + + model + .online_train_step_bptt1( + &mut scratch, + &state, + 7, + &pdf, + scope, + OptimizerKind::Sgd, + 1e-3, + 0.0, + &mut adam_t, + None, + None, + None, + None, + ) + .expect("head-only update"); + + assert_ne!(model.lm_head_weights()[0].to_bits(), before.to_bits()); + } + + #[test] + fn full_training_bptt1_requires_captured_trace() { + let cfg = test_cfg(); + cfg.validate().expect("valid cfg"); + let mut model = Model::new_random(cfg.clone(), 0x1234).expect("random model"); + let mut scratch = ScratchBuffers::new(&cfg); + let mut state = model.new_state(); + scratch.set_capture_train_trace(false); + + let logits = model.forward(&mut scratch, 4, &mut state).to_vec(); + let mut pdf = vec![0.0f64; cfg.vocab_size]; + super::super::super::softmax_pdf_floor_with_bias(&logits, None, &mut pdf); + + let err = model + .online_train_step_bptt1( + &mut scratch, + &state, + 3, + &pdf, + TrainScopeMask { + attn: true, + ..TrainScopeMask::default() + }, + OptimizerKind::Sgd, + 1e-3, + 0.0, + &mut 0usize, + None, + None, + None, + None, + ) + .expect_err("non-head training should require trace"); + assert!(err.to_string().contains("full training trace is missing")); + } + + #[test] + fn adam_full_training_requires_explicit_adam_state() { + let cfg = test_cfg(); + cfg.validate().expect("valid cfg"); + let mut model = Model::new_random(cfg.clone(), 0xCAFE).expect("random model"); + let mut scratch = ScratchBuffers::new(&cfg); + let mut state = model.new_state(); + scratch.set_capture_train_trace(true); + + let logits = model.forward(&mut scratch, 5, &mut state).to_vec(); + let mut pdf = vec![0.0f64; cfg.vocab_size]; + super::super::super::softmax_pdf_floor_with_bias(&logits, None, &mut pdf); + + let err = model + .online_train_step_bptt1( + &mut scratch, + &state, + 6, + &pdf, + TrainScopeMask { + attn: true, + ..TrainScopeMask::default() + }, + OptimizerKind::Adam, + 1e-3, + 0.0, + &mut 0usize, + None, + None, + None, + None, + ) + .expect_err("adam full training should require optimizer state"); + assert!( + err.to_string() + .contains("Adam full-training state is missing") + ); + } } diff --git a/src/backends/rwkvzip/rwkv7/profiling.rs b/crates/infotheory/src/backends/rwkvzip/rwkv7/profiling.rs similarity index 100% rename from src/backends/rwkvzip/rwkv7/profiling.rs rename to crates/infotheory/src/backends/rwkvzip/rwkv7/profiling.rs diff --git a/src/backends/rwkvzip/rwkv7/tensor.rs b/crates/infotheory/src/backends/rwkvzip/rwkv7/tensor.rs similarity index 89% rename from src/backends/rwkvzip/rwkv7/tensor.rs rename to crates/infotheory/src/backends/rwkvzip/rwkv7/tensor.rs index 926af246..65783658 100644 --- a/src/backends/rwkvzip/rwkv7/tensor.rs +++ b/crates/infotheory/src/backends/rwkvzip/rwkv7/tensor.rs @@ -143,6 +143,14 @@ impl Clone for Tensor1D { new.as_mut_slice().copy_from_slice(self.as_slice()); new } + + fn clone_from(&mut self, source: &Self) { + if self.len == source.len { + self.as_mut_slice().copy_from_slice(source.as_slice()); + } else { + *self = source.clone(); + } + } } impl Drop for Tensor1D { @@ -310,6 +318,20 @@ impl Clone for Tensor2D { stride: self.stride, } } + + fn clone_from(&mut self, source: &Self) { + if self.rows == source.rows && self.cols == source.cols && self.stride == source.stride { + let total = self + .rows + .checked_mul(self.stride) + .expect("tensor allocation overflow"); + unsafe { + std::ptr::copy_nonoverlapping(source.data.as_ptr(), self.data.as_ptr(), total); + } + } else { + *self = source.clone(); + } + } } impl Drop for Tensor2D { @@ -500,4 +522,25 @@ mod tests { } t.zero(); } + + #[test] + fn tensor1d_clone_from_reuses_allocation_for_equal_shape() { + let mut dst = Tensor1D::zeros(8); + let src = Tensor1D::from_vec(vec![1.0; 8]); + let before = dst.as_ptr(); + dst.clone_from(&src); + assert_eq!(before, dst.as_ptr()); + assert_eq!(dst.as_slice(), src.as_slice()); + } + + #[test] + fn tensor2d_clone_from_reuses_allocation_for_equal_shape() { + let mut dst = Tensor2D::zeros(2, 3); + let src = Tensor2D::from_vec(vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0], 2, 3); + let before = dst.as_ptr(); + dst.clone_from(&src); + assert_eq!(before, dst.as_ptr()); + assert_eq!(dst.row(0), src.row(0)); + assert_eq!(dst.row(1), src.row(1)); + } } diff --git a/src/backends/rwkvzip/rwkv7/weights.rs b/crates/infotheory/src/backends/rwkvzip/rwkv7/weights.rs similarity index 100% rename from src/backends/rwkvzip/rwkv7/weights.rs rename to crates/infotheory/src/backends/rwkvzip/rwkv7/weights.rs diff --git a/src/backends/sequitur.rs b/crates/infotheory/src/backends/sequitur.rs similarity index 97% rename from src/backends/sequitur.rs rename to crates/infotheory/src/backends/sequitur.rs index d62ac2d6..05d7e1a0 100644 --- a/src/backends/sequitur.rs +++ b/crates/infotheory/src/backends/sequitur.rs @@ -54,6 +54,7 @@ impl ContextFollowers { } #[derive(Clone, Debug)] +#[allow(clippy::enum_variant_names)] enum UndoOp { SetPrev { node: NodeIx, @@ -150,6 +151,13 @@ pub struct SequiturModel { undo_enabled: bool, } +#[derive(Clone)] +pub(crate) struct SequiturLifecycleSnapshot { + frozen_raw_tail: Vec, + pdf: [f64; 256], + pdf_valid: bool, +} + impl SequiturModel { /// Create a new Sequitur model. /// @@ -250,12 +258,30 @@ impl SequiturModel { self.undo_enabled = false; } + pub(crate) fn checkpoints_active(&self) -> bool { + self.undo_enabled + } + /// Clear speculative frozen updates without touching committed state. pub fn reset_frozen(&mut self) { self.frozen_raw_tail.clear(); self.pdf_valid = false; } + pub(crate) fn lifecycle_snapshot(&self) -> SequiturLifecycleSnapshot { + SequiturLifecycleSnapshot { + frozen_raw_tail: self.frozen_raw_tail.clone(), + pdf: self.pdf, + pdf_valid: self.pdf_valid, + } + } + + pub(crate) fn restore_lifecycle_snapshot(&mut self, snapshot: SequiturLifecycleSnapshot) { + self.frozen_raw_tail = snapshot.frozen_raw_tail; + self.pdf = snapshot.pdf; + self.pdf_valid = snapshot.pdf_valid; + } + /// Fill `out` with the current normalized next-byte probability mass. pub fn fill_pdf(&mut self, out: &mut [f64; 256]) { self.ensure_pdf(); @@ -381,10 +407,10 @@ impl SequiturModel { let guard = self.rules[rule_id as usize].guard; let mut node = self.nodes[guard as usize].next; while node != guard { - if let NodeData::Sym(Symbol::NonTerminal(child)) = self.nodes[node as usize].data { - if self.rules[child as usize].active { - self.collect_rule_preorder(child, order, seen); - } + if let NodeData::Sym(Symbol::NonTerminal(child)) = self.nodes[node as usize].data + && self.rules[child as usize].active + { + self.collect_rule_preorder(child, order, seen); } node = self.nodes[node as usize].next; } @@ -565,8 +591,8 @@ impl SequiturModel { let total = stats.total as f64; let types = distinct as f64; let escape = types / (total + types); - for i in 0..256 { - next[i] = self.pdf[i] * escape; + for (i, slot) in next.iter_mut().enumerate() { + *slot = self.pdf[i] * escape; } for &(symbol, count) in &stats.counts { next[symbol as usize] += (count as f64) / (total + types); @@ -856,10 +882,10 @@ impl SequiturModel { }; let first = self.first_node_of_rule(rule); - if let Symbol::NonTerminal(child) = self.symbol_of(first) { - if self.rules[child as usize].ref_count == 1 { - self.expand(first, child); - } + if let Symbol::NonTerminal(child) = self.symbol_of(first) + && self.rules[child as usize].ref_count == 1 + { + self.expand(first, child); } } diff --git a/src/backends/sparse_match.rs b/crates/infotheory/src/backends/sparse_match.rs similarity index 83% rename from src/backends/sparse_match.rs rename to crates/infotheory/src/backends/sparse_match.rs index 448e54b6..3c02542d 100644 --- a/src/backends/sparse_match.rs +++ b/crates/infotheory/src/backends/sparse_match.rs @@ -1,4 +1,4 @@ -use crate::backends::match_model::MatchModel; +use crate::backends::match_model::{MatchModel, MatchModelLifecycleSnapshot}; #[derive(Clone, Debug)] /// Gapped/sparse match predictor wrapper over [`MatchModel`]. @@ -60,6 +60,14 @@ impl SparseMatchModel { self.inner.reset_history(); } + pub(crate) fn lifecycle_snapshot(&self) -> MatchModelLifecycleSnapshot { + self.inner.lifecycle_snapshot() + } + + pub(crate) fn restore_lifecycle_snapshot(&mut self, snapshot: MatchModelLifecycleSnapshot) { + self.inner.restore_lifecycle_snapshot(snapshot); + } + /// Advance history without updating learned sparse-match tables. pub fn update_history_only(&mut self, symbol: u8) { self.inner.update_history_only(symbol); diff --git a/src/backends/text_context.rs b/crates/infotheory/src/backends/text_context.rs similarity index 100% rename from src/backends/text_context.rs rename to crates/infotheory/src/backends/text_context.rs diff --git a/crates/infotheory/src/backends/zpaq_rate.rs b/crates/infotheory/src/backends/zpaq_rate.rs new file mode 100644 index 00000000..f10e76ab --- /dev/null +++ b/crates/infotheory/src/backends/zpaq_rate.rs @@ -0,0 +1,525 @@ +//! ZPAQ-backed sequential rate model. +//! +//! This backend estimates `log p(x_t | x_{, + last_bits: f64, + } + + /// Stateful ZPAQ-backed estimator of sequential symbol log-probabilities. + pub struct ZpaqRateModel { + stream: ZpaqStreaming, + history: Vec, + history_bits: f64, + pending_symbol: Option, + pending_bits: f64, + min_prob: f64, + method: String, + } + + impl ZpaqRateModel { + fn new_streaming_compressor(method: &str) -> StreamingCompressor { + StreamingCompressor::new(method).unwrap_or_else(|e| { + panic!("ZPAQ rate backend requires a streamable method; got '{method}': {e}") + }) + } + + /// Defensive global-state settlement for libzpaq C++ internals. + /// + /// Some C++ internals are process-global rather than fully represented + /// by each `Compressor` allocation. First construction through this API + /// settles those globals so subsequent fresh `ZpaqRateModel` instances + /// and temporary compressors used by `fill_log_probs` observe identical + /// initial state for empty-history first-symbol predictions. + /// + /// Architectural note: + /// This is a thin compatibility wrapper around `zpaq_rs::settle_globals`, + /// where the FFI lifecycle guarantee is implemented. + #[inline] + fn settle_zpaq_globals(_method: &str) { + // Delegate to the official API in the crate that owns the FFI. + // The method argument is kept only for source compatibility with + // existing call sites; the zpaq_rs implementation is process-global. + zpaq_rs::settle_globals(); + } + + fn replace_stream_compressor(&mut self) { + // Drop the previous compressor before constructing the replacement + // so the transition remains strictly sequential. + drop(self.stream.compressor.take()); + // Ensure settlement on rebuild paths (delegates to zpaq_rs::settle_globals). + Self::settle_zpaq_globals(&self.method); + self.stream.compressor = Some(Self::new_streaming_compressor(self.method.as_str())); + self.stream.last_bits = 0.0; + } + + fn rebuild_stream_from_history(&mut self) { + self.replace_stream_compressor(); + self.history_bits = 0.0; + let history_len = self.history.len(); + for idx in 0..history_len { + let symbol: u8 = self.history[idx]; + let (after, _) = self.encode_bits(symbol); + self.history_bits = after; + } + self.pending_symbol = None; + self.pending_bits = 0.0; + } + + /// Create a new model with the provided streamable ZPAQ `method`. + /// + /// `min_prob` clamps very small probabilities for numerical stability. + pub fn new(method: impl Into, min_prob: f64) -> Self { + let method = method.into(); + let min_prob = if min_prob.is_finite() && min_prob > 0.0 { + min_prob + } else { + DEFAULT_MIN_PROB + }; + zpaq_rs::validate_streaming_method(method.as_str()).unwrap_or_else(|e| { + panic!("ZPAQ rate backend requires a streamable method; got '{method}': {e}") + }); + + // Defensive settlement (delegates to zpaq_rs::settle_globals). + Self::settle_zpaq_globals(&method); + + Self { + stream: ZpaqStreaming { + compressor: Some(Self::new_streaming_compressor(method.as_str())), + last_bits: 0.0, + }, + history: Vec::new(), + history_bits: 0.0, + pending_symbol: None, + pending_bits: 0.0, + min_prob, + method, + } + } + + /// Begin a fresh stream lifecycle. + /// + /// Newly constructed models are already at fresh-state, so the first + /// call avoids redundant compressor reconstruction. + pub fn begin_stream(&mut self) { + if self.history.is_empty() && self.pending_symbol.is_none() && self.history_bits == 0.0 + { + return; + } + self.reset(); + } + + /// Reset model state and clear any pending prediction cache. + pub fn reset(&mut self) { + self.replace_stream_compressor(); + self.history.clear(); + self.history_bits = 0.0; + self.pending_symbol = None; + self.pending_bits = 0.0; + } + + fn log_prob_from_bits(min_prob: f64, bits: f64) -> f64 { + let logp = -(bits * LN_2); + logp.max(min_prob.ln()) + } + + fn log_prob_from_history(&self, symbol: u8) -> f64 { + let mut compressor = Self::new_streaming_compressor(self.method.as_str()); + for &b in &self.history { + compressor + .push(b) + .expect("zpaq streaming compression failed"); + } + let before = compressor.bits(); + compressor + .push(symbol) + .expect("zpaq streaming compression failed"); + let bits = (compressor.bits() - before).max(0.0); + Self::log_prob_from_bits(self.min_prob, bits) + } + + fn encode_bits(&mut self, symbol: u8) -> (f64, f64) { + let before = self.stream.last_bits; + let compressor = self + .stream + .compressor + .as_mut() + .expect("zpaq stream compressor must be initialized"); + compressor + .push(symbol) + .expect("zpaq streaming compression failed"); + let after = compressor.bits(); + self.stream.last_bits = after; + (after, (after - before).max(0.0)) + } + + /// Return `ln p(symbol | history)` under the current model state. + /// + /// This may cache the encoded-bit result for a matching immediate `update`. + pub fn log_prob(&mut self, symbol: u8) -> f64 { + if let Some(pending) = self.pending_symbol { + if pending == symbol { + return Self::log_prob_from_bits(self.min_prob, self.pending_bits); + } + // We cannot rollback `StreamingCompressor`; rebuild to committed history. + self.rebuild_stream_from_history(); + } + + let (_, bits) = self.encode_bits(symbol); + self.pending_symbol = Some(symbol); + self.pending_bits = bits; + Self::log_prob_from_bits(self.min_prob, bits) + } + + /// Fill 256-way log-probabilities for the current committed history without mutation. + pub fn fill_log_probs(&mut self, out: &mut [f64; 256]) { + for (sym, slot) in out.iter_mut().enumerate() { + *slot = self.log_prob_from_history(sym as u8); + } + } + + /// Advance model state with one observed symbol. + pub fn update(&mut self, symbol: u8) { + if let Some(pending) = self.pending_symbol + && pending == symbol + { + self.history_bits += self.pending_bits; + self.pending_symbol = None; + self.pending_bits = 0.0; + self.history.push(symbol); + return; + } + if self.pending_symbol.is_some() { + self.rebuild_stream_from_history(); + } + let (after, _) = self.encode_bits(symbol); + self.history_bits = after; + self.pending_symbol = None; + self.pending_bits = 0.0; + self.history.push(symbol); + } + + /// Score and consume an entire byte slice, returning total code length in bits. + pub fn update_and_score(&mut self, data: &[u8]) -> f64 { + if data.is_empty() { + return 0.0; + } + if self.pending_symbol.is_some() { + self.rebuild_stream_from_history(); + } + let mut bits = 0.0; + for &b in data { + let (after, delta) = self.encode_bits(b); + self.history_bits = after; + bits += delta; + self.history.push(b); + } + bits + } + } + + impl Clone for ZpaqRateModel { + fn clone(&self) -> Self { + let mut cloned = Self::new(self.method.clone(), self.min_prob); + if !self.history.is_empty() { + let _ = cloned.update_and_score(&self.history); + } + if let Some(symbol) = self.pending_symbol { + let (_, bits) = cloned.encode_bits(symbol); + cloned.pending_symbol = Some(symbol); + cloned.pending_bits = bits; + } else { + cloned.pending_symbol = None; + cloned.pending_bits = 0.0; + } + cloned + } + } + + /// Validate that `method` is streamable and accepted by the ZPAQ backend. + pub fn validate_zpaq_rate_method(method: &str) -> Result<(), String> { + zpaq_rs::validate_streaming_method(method).map_err(|e| e.to_string()) + } + + #[cfg(test)] + mod tests { + use super::*; + + #[test] + fn zpaq_log_prob_update_matches_update_and_score() { + let data = b"the quick brown fox jumps over the lazy dog"; + let mut model_a = ZpaqRateModel::new("1", 1e-9); + let mut bits_a = 0.0; + for &b in data { + let logp = model_a.log_prob(b); + bits_a += -logp / LN_2; + model_a.update(b); + } + + let mut model_b = ZpaqRateModel::new("1", 1e-9); + let bits_b = model_b.update_and_score(data); + + let diff = (bits_a - bits_b).abs(); + assert!(diff < 1e-6, "bits mismatch: {bits_a} vs {bits_b}"); + } + + #[test] + fn zpaq_update_and_score_keeps_raw_bit_deltas_when_floor_would_bind() { + let data: Vec = (0u8..=255).collect(); + let mut raw_model = ZpaqRateModel::new("1", 0.5); + let mut raw_bits = 0.0; + for &symbol in &data { + let (after, delta) = raw_model.encode_bits(symbol); + raw_model.history_bits = after; + raw_model.history.push(symbol); + raw_bits += delta; + } + assert!( + raw_bits > data.len() as f64, + "test requires raw ZPAQ cost to exceed the 1-bit floor cap" + ); + + let mut scored_model = ZpaqRateModel::new("1", 0.5); + let scored_bits = scored_model.update_and_score(&data); + + assert!( + (scored_bits - raw_bits).abs() < 1e-9, + "metric path must preserve raw ZPAQ bit growth: scored={scored_bits} raw={raw_bits}" + ); + } + + #[test] + fn zpaq_fill_log_probs_is_non_mutating() { + let history = b"zpaq fill non mutating"; + let mut model_a = ZpaqRateModel::new("1", 1e-9); + let mut model_b = ZpaqRateModel::new("1", 1e-9); + for &b in history { + model_a.update(b); + model_b.update(b); + } + + let mut row = [0.0f64; 256]; + model_b.fill_log_probs(&mut row); + + let sym = b'x'; + let lp_a = model_a.log_prob(sym); + let lp_b = model_b.log_prob(sym); + assert!((lp_a - lp_b).abs() < 1e-9, "lp_a={lp_a} lp_b={lp_b}"); + assert!((row[sym as usize] - lp_a).abs() < 1e-9); + + model_a.update(sym); + model_b.update(sym); + let next_sym = b'y'; + let lp_a2 = model_a.log_prob(next_sym); + let lp_b2 = model_b.log_prob(next_sym); + assert!((lp_a2 - lp_b2).abs() < 1e-9, "lp_a2={lp_a2} lp_b2={lp_b2}"); + } + + #[test] + fn zpaq_fill_log_probs_preserves_pending_prediction_cache() { + let history = b"zpaq fill preserves pending"; + let mut model_a = ZpaqRateModel::new("1", 1e-9); + let mut model_b = ZpaqRateModel::new("1", 1e-9); + for &b in history { + model_a.update(b); + model_b.update(b); + } + + let probe = b'x'; + let lp_before = model_a.log_prob(probe); + let mut row = [0.0f64; 256]; + model_a.fill_log_probs(&mut row); + assert!( + (row[probe as usize] - model_b.log_prob_from_history(probe)).abs() < 1e-9, + "fill must score committed history, not speculative pending state" + ); + + let lp_after = model_a.log_prob(probe); + assert!( + (lp_before - lp_after).abs() < 1e-9, + "fill must preserve the pending speculative cache: before={lp_before} after={lp_after}" + ); + + model_a.update(probe); + model_b.update(probe); + let next = b'y'; + let lp_a = model_a.log_prob(next); + let lp_b = model_b.log_prob(next); + assert!((lp_a - lp_b).abs() < 1e-9, "lp_a={lp_a} lp_b={lp_b}"); + } + + #[test] + fn zpaq_clone_preserves_pending_prediction_state() { + let mut model_a = ZpaqRateModel::new("1", 1e-9); + for &b in b"clone preserves pending state" { + model_a.update(b); + } + + let probe = b'x'; + let lp_a = model_a.log_prob(probe); + let mut model_b = model_a.clone(); + let lp_b = model_b.log_prob(probe); + assert!((lp_a - lp_b).abs() < 1e-9, "lp_a={lp_a} lp_b={lp_b}"); + + model_a.update(probe); + model_b.update(probe); + let next = b'y'; + let lp_a2 = model_a.log_prob(next); + let lp_b2 = model_b.log_prob(next); + assert!((lp_a2 - lp_b2).abs() < 1e-9, "lp_a2={lp_a2} lp_b2={lp_b2}"); + } + + #[test] + fn zpaq_interleaved_models_match_separate_baselines() { + let history_a = b"interleaved zpaq model A"; + let history_b = b"interleaved zpaq model B"; + let sequence_a = b"ABACABA"; + let sequence_b = b"XYZYZZX"; + + let mut interleaved_a = ZpaqRateModel::new("1", 1e-9); + let mut interleaved_b = ZpaqRateModel::new("1", 1e-9); + let mut baseline_a = ZpaqRateModel::new("1", 1e-9); + let mut baseline_b = ZpaqRateModel::new("1", 1e-9); + + for &b in history_a { + interleaved_a.update(b); + baseline_a.update(b); + } + for &b in history_b { + interleaved_b.update(b); + baseline_b.update(b); + } + + for (&sym_a, &sym_b) in sequence_a.iter().zip(sequence_b.iter()) { + let lp_interleaved_a = interleaved_a.log_prob(sym_a); + let lp_baseline_a = baseline_a.log_prob(sym_a); + assert!( + (lp_interleaved_a - lp_baseline_a).abs() < 1e-9, + "interleaving drifted model A: interleaved={lp_interleaved_a} baseline={lp_baseline_a}" + ); + interleaved_a.update(sym_a); + baseline_a.update(sym_a); + + let lp_interleaved_b = interleaved_b.log_prob(sym_b); + let lp_baseline_b = baseline_b.log_prob(sym_b); + assert!( + (lp_interleaved_b - lp_baseline_b).abs() < 1e-9, + "interleaving drifted model B: interleaved={lp_interleaved_b} baseline={lp_baseline_b}" + ); + interleaved_b.update(sym_b); + baseline_b.update(sym_b); + } + } + + #[test] + fn zpaq_validate_method_is_non_intrusive_with_live_model() { + let mut baseline = ZpaqRateModel::new("1", 1e-9); + let mut probe = ZpaqRateModel::new("1", 1e-9); + for &b in b"validate zpaq method while model is live" { + baseline.update(b); + probe.update(b); + } + + validate_zpaq_rate_method("1").expect("streaming method should validate"); + + let lp_baseline = baseline.log_prob(b'v'); + let lp_probe = probe.log_prob(b'v'); + assert!( + (lp_baseline - lp_probe).abs() < 1e-9, + "validation disturbed live model state: baseline={lp_baseline} probe={lp_probe}" + ); + } + + #[test] + fn zpaq_rate_restart_first_symbol_parity_after_zpaq_preceding() { + // Thin unit test: ZPAQ settlement + restart parity only. Cross-FFI preceding + // activity lives in `tests/zpaq_rate_backend.rs` with per-feature cfg blocks. + let _preceding_zpaq = ZpaqRateModel::new("1", 1e-9); + + let mut session = ZpaqRateModel::new("1", 1e-9); + session.begin_stream(); + let mut warm = [0.0f64; 256]; + session.fill_log_probs(&mut warm); + for &byte in b"zpaq history before restart" { + session.update(byte); + } + + session.begin_stream(); + let mut restarted = [0.0f64; 256]; + session.fill_log_probs(&mut restarted); + + let mut fresh = ZpaqRateModel::new("1", 1e-9); + let mut expected = [0.0f64; 256]; + fresh.fill_log_probs(&mut expected); + + for (symbol, (&actual, &expected)) in restarted.iter().zip(expected.iter()).enumerate() + { + assert!( + (actual - expected).abs() < 1e-9, + "first-symbol parity after preceding + restart failed for symbol {symbol}; diff={}", + actual - expected + ); + } + } + } +} + +#[cfg(not(feature = "backend-zpaq"))] +mod imp { + #[derive(Clone)] + pub struct ZpaqRateModel { + min_log_prob: f64, + } + + impl ZpaqRateModel { + pub fn new(_method: impl Into, min_prob: f64) -> Self { + let min_prob = if min_prob.is_finite() && min_prob > 0.0 { + min_prob + } else { + 1e-12 + }; + Self { + min_log_prob: min_prob.ln(), + } + } + + pub fn reset(&mut self) {} + + pub fn log_prob(&mut self, _symbol: u8) -> f64 { + self.min_log_prob + } + + pub fn fill_log_probs(&mut self, out: &mut [f64; 256]) { + out.fill(self.min_log_prob); + } + + pub fn update(&mut self, _symbol: u8) {} + + pub fn update_and_score(&mut self, data: &[u8]) -> f64 { + let bits_per_symbol = -self.min_log_prob / std::f64::consts::LN_2; + bits_per_symbol * (data.len() as f64) + } + } + + pub fn validate_zpaq_rate_method(_method: &str) -> Result<(), String> { + Err("zpaq backend disabled at compile time".to_string()) + } +} + +/// Stateful ZPAQ-based rate estimator. +pub use imp::ZpaqRateModel; +/// Validate that a ZPAQ method string is streamable and usable for rate modeling. +pub use imp::validate_zpaq_rate_method; diff --git a/crates/infotheory/src/cli/help.rs b/crates/infotheory/src/cli/help.rs new file mode 100644 index 00000000..f66d2204 --- /dev/null +++ b/crates/infotheory/src/cli/help.rs @@ -0,0 +1,410 @@ +fn quoted_backend_list(names: &[&str]) -> String { + names + .iter() + .enumerate() + .map(|(idx, name)| { + if idx == 0 { + format!("'{name}' (default)") + } else { + format!("'{name}'") + } + }) + .collect::>() + .join(", ") +} + +fn rate_backends() -> String { + quoted_backend_list(&infotheory::backends::available_rate_backends()) +} + +fn compression_backends() -> String { + quoted_backend_list(&infotheory::backends::available_compression_backends()) +} + +fn ctw_profile_line() -> &'static str { + if cfg!(all(feature = "backend-ctw", feature = "research-tooling")) { + " ctw-profile FAC-CTW arena telemetry JSONL\n" + } else { + "" + } +} + +/// Print the top-level CLI help. +pub(crate) fn print_global_help() { + eprintln!( + r#"InfoTheory CLI +Usage: + infotheory [args...] [options] + infotheory help [topic] + +Core commands: + h, entropy Empirical byte entropy; uses rate backend when explicitly selected + h_rate Algorithmic entropy rate via the active rate backend + mi, xe, ce Mutual information, cross entropy, conditional entropy + ncd Normalized compression distance + ned, nte Normalized entropy distance and transform effort + kl, js, tvd Empirical byte-distribution divergences/distances + compress Compress a file with the selected compression backend + decompress Decompress a framed file + generate Continue file or piped bytes with the active rate backend + search Rank code/text matches with information-theoretic scoring + batch JSONL batch API + aixi Run a canonical planner_run document + warmstart Export, convert, or merge warm-start teacher datasets + tune Run a canonical tune document + ac-log-loss Exact AC/log-loss diagnostics for mixture specs + sequitur-debug Sequitur grammar and bounded predictive trace debugging +{ctw_profile_line} +Common backend options: + --rate-backend Rate backend: {rate_backends} + --compression-backend Compression backend: {compression_backends} + --method Backend method/config, model method, or spec path + --rate-backend-json Canonical RateBackend JSON + --compression-backend-json + Canonical CompressionBackend JSON + --expert-spec Standalone expert JSON + --model-export Export updated online model and sidecar + +Topics: + metrics, backends, compression, generation, batch, aixi, warmstart, tune, + diagnostics, sequitur, search + +Examples: + infotheory h README.md + infotheory h_rate README.md --rate-backend ctw --method 32 + infotheory ncd a.bin b.bin --compression-backend zpaq --method 5 + infotheory compress in.bin out.itc --compression-backend rate-ac --rate-backend ctw + cat prompt.txt | infotheory generate --rate-backend ctw --method 32 --bytes 8 + infotheory aixi configs/aixi/paper_kuhn_poker.json + +Use `infotheory help ` or `infotheory --help` for details. +"#, + ctw_profile_line = ctw_profile_line(), + rate_backends = rate_backends(), + compression_backends = compression_backends() + ); +} + +/// Print a topic-specific help page. Unknown topics fall back to global help. +pub(crate) fn print_topic_help(topic: &str) { + match normalize_topic(topic).as_str() { + "metrics" + | "h" + | "entropy" + | "h_rate" + | "entropy_rate" + | "mi" + | "mutual_info" + | "xe" + | "cross_entropy" + | "ce" + | "conditional_entropy" + | "joint_entropy" + | "h_xy" + | "id" + | "ned" + | "nte" + | "rt" + | "resistance" + | "kl" + | "kl_divergence" + | "js" + | "js_divergence" + | "tvd" + | "nhd" => print_metrics_help(), + "backends" | "backend" | "rate-backend" | "compression-backend" => print_backends_help(), + "compression" | "compress" | "decompress" | "ncd" | "ncd_sym" | "ncd_cons" + | "ncd_sym_cons" => print_compression_help(), + "generation" | "generate" => print_generation_help(), + "batch" => print_batch_help(), + "aixi" | "planner" | "planner_run" | "planner-run" => print_aixi_help(), + "warmstart" => print_warmstart_help(), + "tune" | "tuner" => print_tune_help(), + "diagnostics" | "diagnostic" | "ac-log-loss" | "ac_log_loss" | "ctw-profile" + | "ctw_profile" => print_diagnostics_help(), + "sequitur" | "sequitur-debug" | "sequitur_debug" => print_sequitur_help(), + "search" => print_search_help(), + _ => print_global_help(), + } +} + +fn normalize_topic(topic: &str) -> String { + topic.trim().to_ascii_lowercase() +} + +fn print_metrics_help() { + eprintln!( + r#"InfoTheory metrics +Usage: + infotheory h [backend options] + infotheory h_rate [backend options] + infotheory [backend options] + +Single-file metrics: + h, entropy Empirical order-0 byte entropy unless a rate backend is selected + h_rate, entropy_rate Algorithmic entropy rate via active RateBackend + id Intrinsic dependence from empirical entropy and entropy rate + +Two-file metrics: + mi, mutual_info Mutual information + xe, cross_entropy Cross entropy + ce, conditional_entropy + joint_entropy, h_xy + ned, ned_cons Normalized entropy distance variants + nte Normalized transform effort + rt, resistance Resistance to transformation + kl, js, tvd, nhd Empirical byte-distribution divergences/distances + +Backend selection: + Add --rate-backend, --rate-backend-json, or --expert-spec to use the + algorithmic/rate-backed path where the metric supports it. + +Examples: + infotheory h README.md + infotheory h_rate README.md --rate-backend fac-ctw --method 32 --msb-first + infotheory mi a.bin b.bin --rate-backend ctw --method 16 +"# + ); +} + +fn print_backends_help() { + eprintln!( + r#"InfoTheory backend selection +Usage: + infotheory ... [backend options] + +Rate backends: + {rate_backends} + +Compression backends: + {compression_backends} + +Options: + --rate-backend Shorthand rate backend name + --compression-backend Shorthand compression backend name + --method Backend method/config or spec path + --rate-backend-json Canonical RateBackend JSON; relative asset paths resolve + against the JSON file directory + --compression-backend-json + Canonical CompressionBackend JSON, including tuner output + --expert-spec One standalone mixture expert JSON + --msb-first | --lsb-first FAC-CTW bit order; requires --rate-backend fac-ctw + --model-export Export updated online neural model plus JSON sidecar + +Method examples: + --method 5 + --method 32 + --method mixture.json + --method "file:/path/model.safetensors;policy:..." + --method "cfg:hidden=64,layers=1,intermediate=64,...;policy:..." +"#, + rate_backends = rate_backends(), + compression_backends = compression_backends() + ); +} + +fn print_compression_help() { + eprintln!( + r#"InfoTheory compression and NCD +Usage: + infotheory ncd [method] [backend options] + infotheory ncd_sym [backend options] + infotheory ncd_cons [backend options] + infotheory compress [backend options] + infotheory decompress [backend options] + +Compression backend options: + --compression-backend zpaq|rate-ac|rate-rans|rwkv7 + --compression-backend-json + --rate-backend --method For rate-ac/rate-rans wrappers + +Examples: + infotheory ncd a.bin b.bin --compression-backend zpaq --method 5 + infotheory ncd a.bin b.bin --compression-backend rate-ac --rate-backend ctw --method 16 + infotheory compress in.bin out.itc --compression-backend rate-rans --rate-backend fac-ctw --method 32 + infotheory decompress out.itc restored.bin --compression-backend rate-rans --rate-backend fac-ctw --method 32 +"# + ); +} + +fn print_generation_help() { + eprintln!( + r#"InfoTheory generation +Usage: + infotheory generate [file] [backend options] [generation options] + cat prompt.txt | infotheory generate [backend options] [generation options] + +Options: + --bytes Bytes to generate (default: 8) + --sample Use seeded sampling + --greedy Force deterministic greedy generation + --adaptive Fit on generated bytes instead of frozen continuation + --seed RNG seed; implies sampling + --temperature Sampling temperature (default: 1.0) + --top-k Sample from top-k bytes; 0 disables + --top-p

Nucleus threshold in (0, 1] + +Examples: + cat prompt.txt | infotheory generate --rate-backend ctw --method 32 --bytes 8 + infotheory generate prompt.txt --rate-backend match --bytes 16 --sample --seed 7 +"# + ); +} + +fn print_batch_help() { + eprintln!( + r#"InfoTheory JSONL batch API +Usage: + infotheory batch < input.jsonl > output.jsonl + echo '{{"op":"help"}}' | infotheory batch + +Batch operations: + help, metrics, metrics_file, ncd, ncd_files, rosa_dist, cross_entropy, + batch_metrics, ncd_matrix, rosa_matrix, spam_check + +The batch API is line-oriented: each input line is one JSON request and each +output line is one JSON response. +"# + ); +} + +fn print_aixi_help() { + eprintln!( + r#"InfoTheory AIXI/planner-run mode +Usage: + infotheory aixi + +The AIXI CLI executes canonical spec documents: + {{"schema_version": 1, "kind": "planner_run", ...}} + +Checked-in examples: + infotheory aixi configs/aixi/paper_kuhn_poker.json + infotheory aixi configs/aixi/builtin_tictactoe.json + +Legacy pre-1.2 AIXI JSON configs are intentionally rejected. From the Infotheory repository, convert them with: + ./projman.sh legacy_aixi_convert + +For VM-backed environments, build with the vm feature and provide valid Nyx-Lite +VM assets referenced by the planner_run document. +"# + ); +} + +fn print_warmstart_help() { + eprintln!( + r#"InfoTheory warm-start teacher tools +Usage: + infotheory warmstart teacher planner-run --target --teacher --out + infotheory warmstart teacher from-jsonl --target --jsonl --out + infotheory warmstart teacher merge --target --out --teacher [...] + +Subcommands: + planner-run Execute a compatible teacher planner_run and export a same-task + warm-start teacher dataset. + from-jsonl Convert normalized planner JSONL telemetry to a teacher dataset. + merge Deterministically merge same-task teacher datasets. + +The target must be an aiqi_warmstart_exact_jh planner_run. Teacher datasets are +validated against the compiled target contract before writing. +"# + ); +} + +fn print_tune_help() { + eprintln!( + r#"InfoTheory tuner +Usage: + infotheory tune [options] + +Core options: + --exec-config Executor profile JSON + --max-evaluations Optional evaluation cap + --annealer-kernel-profile reversible_elementary_metropolis or + compiled_uniform_metropolis_hastings + --cpu-affinity Comma-separated core ids + --threads Executor thread hint + --evaluator-worker-executable Explicit evaluator worker executable + --evaluator-cgroup-parent Delegated cgroup-v2 eval parent + --warmup-baseline-runs + --self-improvement-rounds + --stagnation-reset-evals + --log-path JSONL executor event log + --diagnostic-chunk-bytes + --rss-mode process_rss_peak, backend_reported, + or hybrid_strict_max + --planner-deployable-model + --warmstart-trace-refresh + +Certificate/theorem-facing options: + --timing-tier + --determinism-deadline-certificate + --deterministic-evaluator-table + --finite-planner-state-certificate + --no-hidden-state-certificate + --exact-reward-encoding-certificate + --emit-exact-reward-encoding-certificate + --exact-state-observation-certificate + --observation-adapter-spec-ref + --exact-state-encoder-spec-ref + --scalar-representation-ref + --claim-exact-finite-mdp + --claim-exact-observed-markov + --claim-planner-convergence + +Examples live under examples/tuner/. +"# + ); +} + +fn print_diagnostics_help() { + eprintln!( + r#"InfoTheory diagnostics +Usage: + infotheory ac-log-loss --mixture --out-prefix + infotheory ctw-profile [--depth N] + +ac-log-loss writes: + .trace.tsv + .nodes.tsv + .summary.tsv + +ctw-profile requires features backend-ctw and research-tooling. It emits FAC-CTW +arena telemetry as JSONL. +"# + ); +} + +fn print_sequitur_help() { + eprintln!( + r#"InfoTheory Sequitur debug +Usage: + infotheory sequitur-debug [options] + infotheory sequitur-debug --hex [--hex ...] [options] + +Options: + --hex Hex-encoded byte string; repeatable + --context-bytes Sequitur context width (default: 64) + --alphabet-prefix Prefix of predictive PDF to emit + +Example: + infotheory sequitur-debug --hex 616263616263 --alphabet-prefix 8 +"# + ); +} + +fn print_search_help() { + eprintln!( + r#"InfoTheory search +Usage: + infotheory search [options] + +Common options: + --prior Extra codebase/domain context + --level snippet|file Search granularity + --top-k Maximum results + +Example: + infotheory search "encryption" ./crates/infotheory/src --prior "codebase context" +"# + ); +} diff --git a/crates/infotheory/src/cli/mod.rs b/crates/infotheory/src/cli/mod.rs new file mode 100644 index 00000000..0b9484d8 --- /dev/null +++ b/crates/infotheory/src/cli/mod.rs @@ -0,0 +1,2291 @@ +use super::*; +use infotheory::error::InfotheoryResult; +#[cfg(all(test, feature = "vm"))] +use std::time::Duration; + +pub(crate) mod help; +pub(crate) mod planner_run; +pub(crate) mod warmstart; + +#[cfg(feature = "vm")] +#[cfg(test)] +#[allow(dead_code)] +pub(super) fn parse_shared_memory_policy(v: Option<&str>) -> SharedMemoryPolicy { + match v.unwrap_or("snapshot") { + "preserve" => SharedMemoryPolicy::Preserve, + _ => SharedMemoryPolicy::Snapshot, + } +} + +#[cfg(feature = "vm")] +#[cfg(test)] +#[allow(dead_code)] +pub(super) fn parse_nyx_environment_config( + v: &serde_json::Value, + observation_bits: usize, + reward_bits: usize, + agent_horizon: usize, + base_dir: &Path, +) -> anyhow::Result { + let vm = &v["vm_config"]; + if vm.is_null() { + return Err(anyhow::anyhow!("vm_config is required for environment=vm")); + } + + let firecracker_config = vm["firecracker_config"] + .as_str() + .or_else(|| vm["config"].as_str()) + .or_else(|| v["firecracker_config"].as_str()) + .ok_or_else(|| anyhow::anyhow!("vm_config.firecracker_config is required"))? + .to_string(); + + let instance_id = vm["instance_id"].as_str().unwrap_or("aixi-nyx").to_string(); + let shared_region_name = vm["shared_region_name"] + .as_str() + .unwrap_or("shared") + .to_string(); + let shared_region_size = vm["shared_region_size"].as_u64().unwrap_or(4096) as usize; + let shared_memory_policy = parse_shared_memory_policy( + vm["shared_memory_policy"] + .as_str() + .or_else(|| v["shared_memory_policy"].as_str()), + ); + + let step_timeout_ms = vm["step_timeout_ms"].as_u64().unwrap_or(100); + let boot_timeout_ms = vm["boot_timeout_ms"].as_u64().unwrap_or(30_000); + let episode_steps = vm["episode_steps"].as_u64().unwrap_or(agent_horizon as u64) as usize; + let step_cost = vm["step_cost"].as_i64().unwrap_or(1); + let debug_mode = vm["verbose"] + .as_bool() + .or_else(|| vm["debug"].as_bool()) + .unwrap_or(false); + + let protocol = parse_nyx_protocol_config(if !vm["protocol"].is_null() { + &vm["protocol"] + } else { + &v["vm_protocol"] + })?; + let stats_backend = parse_vm_stats_backend( + if !vm["stats_backend"].is_null() { + &vm["stats_backend"] + } else { + &v["vm_stats_backend"] + }, + v, + base_dir, + )?; + let trace = parse_nyx_trace_config(if !vm["trace"].is_null() { + &vm["trace"] + } else { + &v["vm_trace"] + })?; + let action_source = parse_nyx_actions(if !vm["actions"].is_null() { + &vm["actions"] + } else { + &v["vm_actions"] + })?; + let observation_policy = parse_nyx_observation_policy(if !vm["observation"].is_null() { + &vm["observation"] + } else { + &v["vm_observation"] + })?; + let observation_stream_len = + parse_observation_stream_len_for_vm(if !vm["observation"].is_null() { + &vm["observation"] + } else { + &v["vm_observation"] + }); + let observation_stream_mode = + parse_nyx_observation_stream_mode(if !vm["observation"].is_null() { + &vm["observation"] + } else { + &v["vm_observation"] + })?; + let observation_stream_pad_byte = + parse_nyx_observation_pad_byte(if !vm["observation"].is_null() { + &vm["observation"] + } else { + &v["vm_observation"] + }); + let reward_policy = parse_nyx_reward_policy(if !vm["reward"].is_null() { + &vm["reward"] + } else { + &v["vm_reward"] + })?; + let reward_shaping = if !vm["reward_shaping"].is_null() { + parse_nyx_reward_shaping(&vm["reward_shaping"], base_dir)? + } else if !v["vm_reward_shaping"].is_null() { + parse_nyx_reward_shaping(&v["vm_reward_shaping"], base_dir)? + } else if !vm["reward"].is_null() && !vm["reward"]["shaping"].is_null() { + parse_nyx_reward_shaping(&vm["reward"]["shaping"], base_dir)? + } else { + None + }; + let action_filter = parse_nyx_filter( + if !vm["filter"].is_null() { + &vm["filter"] + } else { + &v["vm_filter"] + }, + step_cost, + )?; + + let mut cfg = NyxVmConfig::default(); + cfg.firecracker_config = firecracker_config; + cfg.instance_id = instance_id; + cfg.shared_region_name = shared_region_name; + cfg.shared_region_size = shared_region_size; + cfg.shared_memory_policy = shared_memory_policy; + cfg.step_timeout = Duration::from_millis(step_timeout_ms); + cfg.boot_timeout = Duration::from_millis(boot_timeout_ms); + cfg.episode_steps = episode_steps; + cfg.step_cost = step_cost; + cfg.observation_policy = observation_policy; + cfg.observation_bits = observation_bits; + cfg.observation_stream_len = observation_stream_len; + cfg.observation_stream_mode = observation_stream_mode; + cfg.observation_pad_byte = observation_stream_pad_byte; + cfg.reward_bits = reward_bits; + cfg.reward_policy = reward_policy; + cfg.reward_shaping = reward_shaping; + cfg.action_source = action_source; + cfg.action_filter = action_filter; + cfg.protocol = protocol; + cfg.stats_backend = stats_backend; + cfg.trace = trace; + cfg.debug_mode = debug_mode; + cfg.crash_log = vm["crash_log"].as_str().map(|s| s.to_string()); + Ok(cfg) +} + +#[cfg(all(test, feature = "vm"))] +pub(super) fn parse_vm_stats_backend( + cfg: &serde_json::Value, + root: &serde_json::Value, + base_dir: &Path, +) -> anyhow::Result { + let spec = normalize_vm_stats_backend_spec(cfg, root)?; + infotheory::spec::parse_rate_backend_json(&spec, base_dir, MAX_MIXTURE_NESTING) + .map_err(anyhow::Error::msg) +} + +#[cfg(feature = "vm")] +#[allow(dead_code)] +#[cfg(test)] +pub(super) fn parse_nyx_trace_config( + v: &serde_json::Value, +) -> anyhow::Result> { + if v.is_null() { + return Ok(None); + } + let max_bytes = v["max_bytes"].as_u64().unwrap_or(1_000_000) as usize; + let reset_on_episode = v["reset_on_episode"].as_bool().unwrap_or(false); + let shared_region_name = v["shared_region_name"] + .as_str() + .or_else(|| { + if v["mode"].as_str() == Some("shared_memory") { + Some("trace") + } else { + None + } + }) + .map(|s| s.to_string()) + .or(Some("trace".to_string())); + + let mut trace = NyxTraceConfig::new(); + trace.shared_region_name = shared_region_name; + trace.max_bytes = max_bytes; + trace.reset_on_episode = reset_on_episode; + Ok(Some(trace)) +} + +#[cfg(feature = "vm")] +#[allow(dead_code)] +#[cfg(test)] +pub(super) fn parse_nyx_protocol_config( + v: &serde_json::Value, +) -> anyhow::Result { + let mut cfg = NyxProtocolConfig::default(); + if let Some(s) = v["action_prefix"].as_str() { + cfg.action_prefix = s.to_string(); + } + if let Some(s) = v["action_suffix"].as_str() { + cfg.action_suffix = s.to_string(); + } + if let Some(s) = v["obs_prefix"].as_str() { + cfg.obs_prefix = s.to_string(); + } + if let Some(s) = v["rew_prefix"].as_str() { + cfg.rew_prefix = s.to_string(); + } + if let Some(s) = v["done_prefix"].as_str() { + cfg.done_prefix = s.to_string(); + } + if let Some(s) = v["data_prefix"].as_str() { + cfg.data_prefix = s.to_string(); + } + if let Some(s) = v["wire_encoding"].as_str() { + cfg.wire_encoding = s + .parse::() + .map_err(|_| anyhow::anyhow!("unknown VM wire_encoding '{s}'"))?; + } + Ok(cfg) +} + +#[cfg(feature = "vm")] +#[allow(dead_code)] +#[cfg(test)] +pub(super) fn parse_nyx_actions(v: &serde_json::Value) -> anyhow::Result { + let mode = v["mode"].as_str().unwrap_or("literal"); + match mode { + "fuzz" => { + let fuzz = if v["fuzz"].is_null() { v } else { &v["fuzz"] }; + let seed_encoding_label = fuzz["seed_encoding"].as_str().unwrap_or("utf8"); + let seed_encoding = + seed_encoding_label + .parse::() + .map_err(|_| { + anyhow::anyhow!( + "unknown vm_actions.fuzz.seed_encoding '{seed_encoding_label}'" + ) + })?; + let mut seeds = Vec::new(); + if let Some(arr) = fuzz["seed_paths"].as_array() { + for item in arr { + if let Some(path) = item.as_str() { + let data = std::fs::read(path)?; + seeds.push(data); + } + } + } + if let Some(arr) = fuzz["seed_inputs"].as_array() { + for item in arr { + if let Some(text) = item.as_str() { + seeds.push(seed_encoding.decode(text)?); + } + } + } + + let mut mutators = Vec::new(); + if let Some(arr) = fuzz["mutators"].as_array() { + for item in arr { + if let Some(name) = item.as_str() { + if let Some(m) = parse_nyx_fuzz_mutator(name) { + mutators.push(m); + } + } + } + } + let min_len = fuzz["min_len"].as_u64().unwrap_or(1) as usize; + let max_len = fuzz["max_len"].as_u64().unwrap_or(4096) as usize; + let dict_encoding_label = fuzz["dict_encoding"].as_str().unwrap_or("utf8"); + let dict_encoding = + dict_encoding_label + .parse::() + .map_err(|_| { + anyhow::anyhow!( + "unknown vm_actions.fuzz.dict_encoding '{dict_encoding_label}'" + ) + })?; + let mut dictionary = Vec::new(); + if let Some(arr) = fuzz["dictionary"].as_array() { + for item in arr { + if let Some(text) = item.as_str() { + dictionary.push(dict_encoding.decode(text)?); + } + } + } + let rng_seed = fuzz["rng_seed"].as_u64().unwrap_or(0); + let mut fuzz_cfg = NyxFuzzConfig::new(seeds); + fuzz_cfg.mutators = mutators; + fuzz_cfg.min_len = min_len; + fuzz_cfg.max_len = max_len; + fuzz_cfg.dictionary = dictionary; + fuzz_cfg.rng_seed = rng_seed; + Ok(NyxActionSource::Fuzz(fuzz_cfg)) + } + _ => { + let mut actions = Vec::new(); + if let Some(arr) = v["actions"].as_array() { + for item in arr { + if let Some(text) = item.as_str() { + let payload = NyxPayloadEncoding::Utf8.decode(text)?; + actions.push(NyxActionSpec::new(payload)); + continue; + } + let payload = item["payload"].as_str().unwrap_or_default(); + let encoding_label = item["encoding"].as_str().unwrap_or("utf8"); + let encoding = encoding_label.parse::().map_err(|_| { + anyhow::anyhow!("unknown vm_actions[].encoding '{encoding_label}'") + })?; + let payload = encoding.decode(payload)?; + let mut spec = NyxActionSpec::new(payload); + spec.name = item["name"].as_str().map(|s| s.to_string()); + actions.push(spec); + } + } + Ok(NyxActionSource::Literal(actions)) + } + } +} + +#[cfg(feature = "vm")] +#[allow(dead_code)] +#[cfg(test)] +pub(super) fn parse_nyx_fuzz_mutator(name: &str) -> Option { + match name { + "flip_bit" => Some(NyxFuzzMutator::FlipBit), + "flip_byte" => Some(NyxFuzzMutator::FlipByte), + "insert_byte" => Some(NyxFuzzMutator::InsertByte), + "delete_byte" => Some(NyxFuzzMutator::DeleteByte), + "splice_seed" => Some(NyxFuzzMutator::SpliceSeed), + "reset_seed" => Some(NyxFuzzMutator::ResetSeed), + "havoc" => Some(NyxFuzzMutator::Havoc), + _ => None, + } +} + +#[cfg(feature = "vm")] +#[cfg(test)] +fn parse_nyx_observation_policy_str(mode: &str) -> anyhow::Result { + match mode { + "from_guest" => Ok(NyxObservationPolicy::FromGuest), + "raw_output" => Ok(NyxObservationPolicy::RawOutput), + "output_hash" => Ok(NyxObservationPolicy::OutputHash), + "shared_memory" => Ok(NyxObservationPolicy::SharedMemory), + other => Err(anyhow::anyhow!("unknown VM observation policy '{other}'")), + } +} + +#[cfg(feature = "vm")] +#[cfg(test)] +pub(super) fn parse_nyx_observation_policy( + v: &serde_json::Value, +) -> anyhow::Result { + parse_nyx_observation_policy_str(v["mode"].as_str().unwrap_or("from_guest")) +} + +#[cfg(feature = "vm")] +#[allow(dead_code)] +#[cfg(test)] +pub(super) fn parse_nyx_observation_stream_mode( + v: &serde_json::Value, +) -> anyhow::Result { + match v["stream_mode"].as_str().unwrap_or("pad_truncate") { + "pad" => Ok(NyxObservationStreamMode::Pad), + "truncate" => Ok(NyxObservationStreamMode::Truncate), + "pad_truncate" => Ok(NyxObservationStreamMode::PadTruncate), + other => Err(anyhow::anyhow!("unknown observation stream mode '{other}'")), + } +} + +#[cfg(feature = "vm")] +#[allow(dead_code)] +#[cfg(test)] +pub(super) fn parse_nyx_observation_pad_byte(v: &serde_json::Value) -> u8 { + v["pad_byte"].as_u64().unwrap_or(0) as u8 +} + +#[cfg(feature = "vm")] +#[allow(dead_code)] +#[cfg(test)] +pub(super) fn parse_nyx_reward_policy(v: &serde_json::Value) -> anyhow::Result { + match v["mode"].as_str().unwrap_or("guest") { + "pattern" => { + let pattern = v["pattern"] + .as_str() + .ok_or_else(|| anyhow::anyhow!("vm_reward.pattern is required"))? + .to_string(); + let base_reward = v["base_reward"].as_i64().unwrap_or(0); + let bonus_reward = v["bonus_reward"].as_i64().unwrap_or(10); + Ok(NyxRewardPolicy::Pattern { + pattern, + base_reward, + bonus_reward, + }) + } + _ => Ok(NyxRewardPolicy::FromGuest), + } +} + +#[cfg(feature = "vm")] +#[allow(dead_code)] +#[cfg(test)] +pub(super) fn parse_nyx_reward_shaping( + v: &serde_json::Value, + base_dir: &Path, +) -> anyhow::Result> { + if v.is_null() { + return Ok(None); + } + match v["mode"].as_str().unwrap_or("none") { + "entropy_reduction" => { + let baseline_path = v["baseline_path"] + .as_str() + .ok_or_else(|| anyhow::anyhow!("vm_reward_shaping.baseline_path is required"))?; + let baseline_path = infotheory::spec::resolve_spec_path(base_dir, baseline_path); + let baseline_bytes = std::fs::read(&baseline_path)?; + let scale = v["scale"].as_f64().unwrap_or(10.0); + let crash_bonus = v["crash_bonus"].as_i64(); + let timeout_bonus = v["timeout_bonus"].as_i64(); + Ok(Some(NyxRewardShaping::EntropyReduction { + baseline_bytes, + scale, + crash_bonus, + timeout_bonus, + })) + } + "trace_entropy" => { + let scale = v["scale"].as_f64().unwrap_or(1.0); + let normalize = v["normalize"].as_bool().unwrap_or(false); + Ok(Some(NyxRewardShaping::TraceEntropy { scale, normalize })) + } + "none" => Ok(None), + other => Err(anyhow::anyhow!("unknown vm_reward_shaping.mode '{other}'")), + } +} + +#[cfg(feature = "vm")] +#[allow(dead_code)] +#[cfg(test)] +pub(super) fn parse_nyx_filter( + v: &serde_json::Value, + step_cost: i64, +) -> anyhow::Result> { + if v.is_null() { + return Ok(None); + } + let novelty_prior = if let Some(path) = v["novelty_prior_path"].as_str() { + Some(std::fs::read(path)?) + } else { + None + }; + let reject_reward = v["reject_reward"].as_i64().or_else(|| Some(-step_cost)); + let mut filter = NyxActionFilter::new(); + filter.min_entropy = v["min_entropy"].as_f64(); + filter.max_entropy = v["max_entropy"].as_f64(); + filter.min_intrinsic_dependence = v["min_intrinsic_dependence"].as_f64(); + filter.min_novelty = v["min_novelty"].as_f64(); + filter.novelty_prior = novelty_prior; + filter.reject_reward = reject_reward; + Ok(Some(filter)) +} + +#[cfg(feature = "backend-rwkv")] +pub(super) fn rwkv7_model_path_from_env() -> String { + env::var("RWKV7_MODEL_PATH").unwrap_or_else(|_| { + eprintln!("Error: RWKV7_MODEL_PATH env var must be set when using rwkv7 backends"); + std::process::exit(1); + }) +} + +#[cfg(feature = "backend-mamba")] +pub(super) fn mamba_model_path_from_env() -> String { + env::var("MAMBA_MODEL_PATH").unwrap_or_else(|_| { + eprintln!("Error: MAMBA_MODEL_PATH env var must be set when using mamba backends"); + std::process::exit(1); + }) +} + +pub(super) fn parse_rate_backend(v: &str) -> Option<&'static str> { + match infotheory::backends::resolve_rate_backend_name(v) { + Some(infotheory::backends::BackendAvailability::Enabled(name)) => Some(name), + Some(infotheory::backends::BackendAvailability::Disabled { canonical, feature }) => { + eprintln!( + "Error: rate backend '{canonical}' requires infotheory built with feature '{feature}'" + ); + std::process::exit(1); + } + None => None, + } +} + +pub(super) fn parse_compression_backend(v: &str) -> Option<&'static str> { + match infotheory::backends::resolve_compression_backend_name(v) { + Some(infotheory::backends::BackendAvailability::Enabled(name)) => Some(name), + Some(infotheory::backends::BackendAvailability::Disabled { canonical, feature }) => { + eprintln!( + "Error: compression backend '{canonical}' requires infotheory built with feature '{feature}'" + ); + std::process::exit(1); + } + None => None, + } +} + +pub(super) fn load_mixture_spec(path: &str) -> anyhow::Result { + infotheory::spec::load_mixture_spec(path).map_err(anyhow::Error::msg) +} + +pub(super) fn load_expert_spec(path: &str) -> anyhow::Result { + infotheory::spec::load_expert_spec(path).map_err(anyhow::Error::msg) +} + +#[cfg(all(test, feature = "vm"))] +const VM_DEFAULT_CTW_DEPTH: usize = 32; +#[cfg(all(test, feature = "vm"))] +const VM_DEFAULT_FAC_CTW_ENCODING_BITS: usize = 8; +#[cfg(all(test, feature = "vm"))] +const VM_DEFAULT_OBSERVATION_BITS: u64 = 16; +#[cfg(all(test, feature = "vm"))] +const VM_DEFAULT_REWARD_BITS: u64 = 8; + +#[cfg(all(test, feature = "vm"))] +fn vm_default_fac_ctw_num_percept_bits(root: &serde_json::Value) -> u64 { + let observation_bits = root["observation_bits"] + .as_u64() + .unwrap_or(VM_DEFAULT_OBSERVATION_BITS); + let reward_bits = root["reward_bits"] + .as_u64() + .unwrap_or(VM_DEFAULT_REWARD_BITS); + observation_bits + reward_bits +} + +#[cfg(all(test, feature = "vm"))] +fn apply_vm_fac_ctw_defaults( + object: &mut serde_json::Map, + root: &serde_json::Value, + default_base_depth: usize, +) { + if !object.contains_key("base_depth") { + object.insert( + "base_depth".to_string(), + serde_json::json!(default_base_depth), + ); + } + if !object.contains_key("encoding_bits") { + object.insert( + "encoding_bits".to_string(), + serde_json::json!(VM_DEFAULT_FAC_CTW_ENCODING_BITS), + ); + } + if !object.contains_key("num_percept_bits") { + object.insert( + "num_percept_bits".to_string(), + serde_json::json!(vm_default_fac_ctw_num_percept_bits(root)), + ); + } +} + +#[cfg(all(test, feature = "vm"))] +pub(super) fn vm_stats_backend_spec_value( + root: &serde_json::Value, +) -> anyhow::Result { + let raw_kind = root["algorithm"].as_str().unwrap_or("ctw"); + let resolved = match infotheory::backends::resolve_rate_backend_name(raw_kind) { + Some(infotheory::backends::BackendAvailability::Enabled(name)) => name, + Some(infotheory::backends::BackendAvailability::Disabled { canonical, feature }) => { + return Err(anyhow::anyhow!( + "rate backend '{canonical}' requires infotheory feature '{feature}'" + )); + } + None => { + return Err(anyhow::anyhow!( + "unknown stats backend algorithm '{raw_kind}'" + )); + } + }; + let ct_depth = root["ct_depth"].as_u64().unwrap_or(20) as usize; + let spec = match resolved { + "ctw" => serde_json::json!({ + "kind": "ctw", + "depth": ct_depth, + }), + "fac-ctw" => { + let mut spec = serde_json::Map::new(); + spec.insert("kind".to_string(), serde_json::json!("fac-ctw")); + apply_vm_fac_ctw_defaults(&mut spec, root, ct_depth); + serde_json::Value::Object(spec) + } + "sequitur" => serde_json::json!({ + "kind": "sequitur", + "context_bytes": root["context_bytes"].as_u64().unwrap_or(64) as usize, + }), + "mamba" => { + #[cfg(feature = "backend-mamba")] + { + serde_json::json!({ + "kind": "mamba", + "model_path": root["mamba_model_path"] + .as_str() + .map(ToOwned::to_owned) + .unwrap_or_else(mamba_model_path_from_env), + }) + } + #[cfg(not(feature = "backend-mamba"))] + { + return Err(anyhow::anyhow!( + "mamba default stats backend requires 'backend-mamba' feature in infotheory" + )); + } + } + "rosaplus" => serde_json::json!({ "kind": "rosaplus" }), + "rwkv7" => { + #[cfg(feature = "backend-rwkv")] + { + serde_json::json!({ + "kind": "rwkv7", + "model_path": root["rwkv_model_path"] + .as_str() + .map(ToOwned::to_owned) + .unwrap_or_else(rwkv7_model_path_from_env), + }) + } + #[cfg(not(feature = "backend-rwkv"))] + { + return Err(anyhow::anyhow!( + "rwkv7 default stats backend requires 'backend-rwkv' feature in infotheory" + )); + } + } + "zpaq" => serde_json::json!({ + "kind": "zpaq", + "method": { + "kind": "literal", + "value": root["method"].as_str().unwrap_or("2"), + }, + }), + "mixture" => { + let spec_path = root["mixture_spec"] + .as_str() + .ok_or_else(|| anyhow::anyhow!("mixture stats backend requires mixture_spec"))?; + serde_json::json!({ + "kind": "mixture", + "spec_path": spec_path, + }) + } + other => return Err(anyhow::anyhow!("unknown stats backend algorithm '{other}'")), + }; + Ok(spec) +} + +#[cfg(all(test, feature = "vm"))] +pub(super) fn normalize_vm_stats_backend_spec( + cfg: &serde_json::Value, + root: &serde_json::Value, +) -> anyhow::Result { + if cfg.is_null() { + return vm_stats_backend_spec_value(root); + } + + let mut spec = if let Some(name) = cfg.as_str() { + serde_json::json!({ "kind": name }) + } else if let Some(object) = cfg.as_object() { + serde_json::Value::Object(object.clone()) + } else { + return Err(anyhow::anyhow!( + "vm stats backend must be a backend name string or JSON object" + )); + }; + + let raw_kind = spec["kind"].as_str().unwrap_or("rosaplus"); + + let resolved = match infotheory::backends::resolve_rate_backend_name(raw_kind) { + Some(infotheory::backends::BackendAvailability::Enabled(name)) => name, + Some(infotheory::backends::BackendAvailability::Disabled { canonical, feature }) => { + return Err(anyhow::anyhow!( + "rate backend '{canonical}' requires infotheory feature '{feature}'" + )); + } + None => return Err(anyhow::anyhow!("unknown vm stats backend '{raw_kind}'")), + }; + + let obj = spec + .as_object_mut() + .ok_or_else(|| anyhow::anyhow!("vm stats backend must be a JSON object"))?; + obj.insert( + "kind".to_string(), + serde_json::Value::String(resolved.to_string()), + ); + + match resolved { + "ctw" => { + if !obj.contains_key("depth") { + obj.insert("depth".to_string(), serde_json::json!(VM_DEFAULT_CTW_DEPTH)); + } + } + "fac-ctw" => { + apply_vm_fac_ctw_defaults(obj, root, VM_DEFAULT_CTW_DEPTH); + } + "mamba" => + { + #[cfg(feature = "backend-mamba")] + if !obj.contains_key("method") && !obj.contains_key("model_path") { + obj.insert( + "model_path".to_string(), + serde_json::Value::String( + root["mamba_model_path"] + .as_str() + .map(ToOwned::to_owned) + .unwrap_or_else(mamba_model_path_from_env), + ), + ); + } + } + "rwkv7" => + { + #[cfg(feature = "backend-rwkv")] + if !obj.contains_key("method") && !obj.contains_key("model_path") { + obj.insert( + "model_path".to_string(), + serde_json::Value::String( + root["rwkv_model_path"] + .as_str() + .map(ToOwned::to_owned) + .unwrap_or_else(rwkv7_model_path_from_env), + ), + ); + } + } + "zpaq" => { + if !obj.contains_key("method") { + obj.insert( + "method".to_string(), + serde_json::json!({ + "kind": "literal", + "value": root["method"].as_str().unwrap_or("2"), + }), + ); + } + } + "mixture" => { + if !obj.contains_key("spec") + && !obj.contains_key("spec_path") + && let Some(path) = root["mixture_spec"].as_str() + { + obj.insert( + "spec_path".to_string(), + serde_json::Value::String(path.to_string()), + ); + } + } + "particle" => { + if !obj.contains_key("spec") + && !obj.contains_key("spec_path") + && let Some(path) = root["particle_spec"].as_str() + { + obj.insert( + "spec_path".to_string(), + serde_json::Value::String(path.to_string()), + ); + } + } + "calibrated" => { + if !obj.contains_key("spec") + && !obj.contains_key("spec_path") + && let Some(path) = root["calibrated_spec"].as_str() + { + obj.insert( + "spec_path".to_string(), + serde_json::Value::String(path.to_string()), + ); + } + } + _ => {} + } + + Ok(spec) +} + +#[cfg(test)] +pub(super) fn parse_observation_stream_len(v: &serde_json::Value) -> usize { + v["observation_stream_len"].as_u64().unwrap_or(1) as usize +} + +#[cfg(test)] +pub(super) fn parse_observation_key_mode( + v: &serde_json::Value, +) -> anyhow::Result { + parse_observation_key_mode_str(v["observation_key_mode"].as_str().unwrap_or("full_stream")) +} + +#[cfg(test)] +pub(super) fn parse_observation_key_mode_str(s: &str) -> anyhow::Result { + match s { + "first" => Ok(ObservationKeyMode::First), + "full_stream" => Ok(ObservationKeyMode::FullStream), + "last" => Ok(ObservationKeyMode::Last), + "stream_hash" => Ok(ObservationKeyMode::StreamHash), + other => Err(anyhow::anyhow!("unknown observation key mode '{other}'")), + } +} + +#[cfg(test)] +pub(super) fn parse_observation_stream_len_for_env(v: &serde_json::Value, env_name: &str) -> usize { + if env_name == "vm" { + if v["vm_observation"].is_null() { + parse_observation_stream_len(v) + } else { + parse_observation_stream_len_for_vm(&v["vm_observation"]) + } + } else { + parse_observation_stream_len(v) + } +} + +#[cfg(test)] +pub(super) fn parse_observation_key_mode_for_env( + v: &serde_json::Value, + env_name: &str, +) -> anyhow::Result { + if env_name == "vm" { + if v["vm_observation"].is_null() { + parse_observation_key_mode(v) + } else { + parse_observation_key_mode_for_vm(&v["vm_observation"]) + } + } else { + parse_observation_key_mode(v) + } +} + +#[cfg(test)] +pub(super) fn parse_observation_key_mode_for_vm( + v: &serde_json::Value, +) -> anyhow::Result { + if v.is_null() { + return Ok(ObservationKeyMode::FullStream); + } + parse_observation_key_mode_str( + v["key_mode"] + .as_str() + .unwrap_or_else(|| v["observation_key_mode"].as_str().unwrap_or("full_stream")), + ) +} + +#[cfg(test)] +pub(super) fn parse_observation_stream_len_for_vm(v: &serde_json::Value) -> usize { + if v.is_null() { + return 1; + } + v["stream_len"] + .as_u64() + .or_else(|| v["observation_stream_len"].as_u64()) + .unwrap_or(1) as usize +} + +#[cfg(test)] +fn extract_observation_stream_len_raw(v: &serde_json::Value) -> Option { + v["observation_stream_len"].as_u64().map(|n| n as usize) +} + +#[cfg(test)] +fn extract_vm_observation_stream_len_raw(v: &serde_json::Value) -> Option { + if v.is_null() { + return None; + } + v["stream_len"] + .as_u64() + .or_else(|| v["observation_stream_len"].as_u64()) + .map(|n| n as usize) +} + +#[cfg(test)] +fn extract_observation_key_mode_raw(v: &serde_json::Value) -> Option { + v["observation_key_mode"] + .as_str() + .map(parse_observation_key_mode_str) + .transpose() + .ok() + .flatten() +} + +#[cfg(test)] +fn extract_vm_observation_key_mode_raw(v: &serde_json::Value) -> Option { + if v.is_null() { + return None; + } + v["key_mode"] + .as_str() + .or_else(|| v["observation_key_mode"].as_str()) + .map(parse_observation_key_mode_str) + .transpose() + .ok() + .flatten() +} + +#[cfg(test)] +pub(super) fn validate_observation_config( + env_name: &str, + v: &serde_json::Value, + observation_stream_len: usize, + observation_key_mode: ObservationKeyMode, +) -> anyhow::Result<()> { + if observation_stream_len == 0 { + return Err(anyhow::anyhow!("observation_stream_len must be > 0")); + } + if env_name == "vm" { + if let (Some(top_len), Some(vm_len)) = ( + extract_observation_stream_len_raw(v), + extract_vm_observation_stream_len_raw(&v["vm_observation"]), + ) && top_len != vm_len + { + return Err(anyhow::anyhow!( + "observation_stream_len ({}) conflicts with vm_observation.stream_len ({})", + top_len, + vm_len + )); + } + if let (Some(top_mode), Some(vm_mode)) = ( + extract_observation_key_mode_raw(v), + extract_vm_observation_key_mode_raw(&v["vm_observation"]), + ) && top_mode != vm_mode + { + return Err(anyhow::anyhow!( + "observation_key_mode ({:?}) conflicts with vm_observation.key_mode ({:?})", + top_mode, + vm_mode + )); + } + } + if observation_stream_len > 1 && matches!(observation_key_mode, ObservationKeyMode::First) { + eprintln!( + "Warning: observation_key_mode=first collapses multi-symbol observation streams; prefer \"full_stream\" for paper-accurate expectimax." + ); + } + if observation_stream_len > 1 && !matches!(observation_key_mode, ObservationKeyMode::FullStream) + { + eprintln!( + "Warning: observation_key_mode {:?} reduces multi-symbol observation streams and deviates from paper-accurate expectimax.", + observation_key_mode + ); + } + Ok(()) +} + +pub(super) struct BuiltCtx { + pub(super) ctx: InfotheoryCtx, +} + +/// Tracks whether the user supplied CLI flags that conflict with JSON backend specs. +#[derive(Clone, Copy, Default)] +pub(super) struct CliBackendSourceFlags { + pub explicit_rate_backend: bool, + pub explicit_compression_backend: bool, + pub explicit_method: bool, +} + +pub(super) struct CliBackendInvocation<'a> { + pub rate_backend: &'a str, + pub compression_backend: &'a str, + pub method: Option<&'a str>, + pub expert_spec_path: Option<&'a str>, + pub rate_backend_json_path: Option<&'a str>, + pub compression_backend_json_path: Option<&'a str>, + /// Optional FAC-CTW MSB-first override from `--msb-first` / `--lsb-first`. + pub fac_ctw_msb_first: Option, + pub flags: CliBackendSourceFlags, +} + +pub(super) fn load_backend_spec_json( + path: &str, + label: &str, +) -> Result<(serde_json::Value, std::path::PathBuf), String> { + infotheory::spec::load_json_value_from_path(std::path::Path::new("."), path, label) + .map_err(|err| err.to_string()) +} + +fn validate_fac_ctw_bit_order_flags(inv: &CliBackendInvocation<'_>) -> Result<(), String> { + if inv.fac_ctw_msb_first.is_none() { + return Ok(()); + } + if inv.rate_backend_json_path.is_some() || inv.compression_backend_json_path.is_some() { + return Err( + "--msb-first/--lsb-first cannot be combined with --rate-backend-json or --compression-backend-json" + .to_string(), + ); + } + if inv.expert_spec_path.is_some() { + return Err("--msb-first/--lsb-first cannot be combined with --expert-spec".to_string()); + } + if inv.rate_backend != "fac-ctw" { + return Err(format!( + "--msb-first/--lsb-first apply only to --rate-backend fac-ctw, got '{}'", + inv.rate_backend + )); + } + Ok(()) +} + +fn validate_cli_backend_sources(inv: &CliBackendInvocation<'_>) -> Result<(), String> { + if inv.compression_backend_json_path.is_some() { + if inv.flags.explicit_compression_backend { + return Err( + "--compression-backend-json cannot be combined with --compression-backend" + .to_string(), + ); + } + if inv.expert_spec_path.is_some() { + return Err( + "--compression-backend-json cannot be combined with --expert-spec".to_string(), + ); + } + if inv.flags.explicit_method { + return Err("--compression-backend-json cannot be combined with --method".to_string()); + } + } + if inv.rate_backend_json_path.is_some() { + if inv.flags.explicit_rate_backend { + return Err( + "--rate-backend-json cannot be combined with --rate-backend or --expert-spec" + .to_string(), + ); + } + if inv.expert_spec_path.is_some() { + return Err("--rate-backend-json cannot be combined with --expert-spec".to_string()); + } + + // `--method` is permitted alongside `--rate-backend-json` because it parameterizes the + // *compression* backend shorthand (e.g. zpaq method), while the rate backend is already + // fixed by the JSON spec. + } + Ok(()) +} + +fn assert_rate_backend_json_matches_embedded( + embedded: &infotheory::api::RateBackend, + from_file: &infotheory::api::RateBackend, + rate_json_path: &str, + compression_json_path: &str, +) -> Result<(), String> { + let embedded_compiled = embedded.compile().map_err(|err| err.to_string())?; + let file_compiled = from_file.compile().map_err(|err| err.to_string())?; + if embedded_compiled.canonical_bytes().as_slice() != file_compiled.canonical_bytes().as_slice() + { + return Err(format!( + "--rate-backend-json ({rate_json_path}) does not match the rate model embedded in --compression-backend-json ({compression_json_path})" + )); + } + Ok(()) +} + +fn rate_backend_from_cli_shorthand( + rate_backend: &str, + method: Option<&str>, + expert_spec_path: Option<&str>, + fac_ctw_msb_first: Option, +) -> infotheory::api::RateBackend { + if let Some(path) = expert_spec_path { + let spec = load_expert_spec(path).unwrap_or_else(|e| { + eprintln!("Error: failed to load expert spec '{path}': {e}"); + std::process::exit(1); + }); + return spec.backend; + } + let mut shorthand = infotheory::spec::RateBackendShorthandOptions::default(); + shorthand.base_dir = std::path::PathBuf::from("."); + shorthand.particle_default_if_missing_method = false; + shorthand.fac_ctw_msb_first = fac_ctw_msb_first; + #[cfg(any(feature = "backend-mamba", feature = "backend-rwkv"))] + let shorthand = { + let mut shorthand = shorthand; + #[cfg(feature = "backend-mamba")] + if rate_backend == "mamba" && method.is_none() { + shorthand.default_mamba_model_path = Some(mamba_model_path_from_env()); + } + #[cfg(feature = "backend-rwkv")] + if rate_backend == "rwkv7" && method.is_none() { + shorthand.default_rwkv_model_path = Some(rwkv7_model_path_from_env()); + } + shorthand + }; + infotheory::spec::parse_rate_backend_name_method(rate_backend, method, &shorthand) + .unwrap_or_else(|e| { + eprintln!("Error: {e}"); + std::process::exit(1); + }) +} + +fn compression_backend_from_cli_shorthand( + rate_backend: &infotheory::api::RateBackend, + compression_backend: &str, + method: Option<&str>, +) -> infotheory::api::CompressionBackend { + #[allow(unused_mut)] + let mut compression_opts = infotheory::spec::CompressionBackendShorthandOptions::default(); + compression_opts.default_rate_backend = Some(rate_backend.clone()); + compression_opts.default_framing = infotheory::compression::FramingMode::Raw; + #[cfg(feature = "backend-rwkv")] + if compression_backend == "rwkv7" + && (method.is_none() + || method + .map(|value| infotheory::backends::parse_rwkv7_coder(value).is_some()) + .unwrap_or(false)) + { + compression_opts.default_rwkv_model_path = Some(rwkv7_model_path_from_env()); + } + infotheory::spec::parse_compression_backend_name_method( + compression_backend, + method, + Some(rate_backend.clone()), + &compression_opts, + ) + .unwrap_or_else(|e| { + eprintln!("Error: {e}"); + std::process::exit(1); + }) +} + +pub(super) fn build_ctx_invocation(inv: CliBackendInvocation<'_>) -> BuiltCtx { + if let Err(err) = validate_cli_backend_sources(&inv) { + eprintln!("Error: {err}"); + std::process::exit(1); + } + if let Err(err) = validate_fac_ctw_bit_order_flags(&inv) { + eprintln!("Error: {err}"); + std::process::exit(1); + } + + let default_framing = infotheory::compression::FramingMode::Raw; + + if let Some(cb_path) = inv.compression_backend_json_path { + let (val, full_path) = load_backend_spec_json(cb_path, "compression backend spec") + .unwrap_or_else(|e| { + eprintln!("Error: {e}"); + std::process::exit(1); + }); + let base_dir = full_path + .parent() + .unwrap_or_else(|| std::path::Path::new(".")); + let compression_backend_ast = + infotheory::spec::parse_compression_backend_json(&val, base_dir, None, default_framing) + .unwrap_or_else(|e| { + eprintln!("Error: {e}"); + std::process::exit(1); + }); + + let rate_from_file: Option = + if let Some(rb_path) = inv.rate_backend_json_path { + let (rv, rfull) = load_backend_spec_json(rb_path, "rate backend spec") + .unwrap_or_else(|e| { + eprintln!("Error: {e}"); + std::process::exit(1); + }); + let rbase = rfull.parent().unwrap_or_else(|| std::path::Path::new(".")); + Some( + infotheory::spec::parse_rate_backend_json( + &rv, + rbase, + infotheory::api::MAX_MIXTURE_NESTING, + ) + .unwrap_or_else(|e| { + eprintln!("Error: {e}"); + std::process::exit(1); + }), + ) + } else { + None + }; + + let rate_backend_ast = match &compression_backend_ast { + infotheory::api::CompressionBackend::Rate { + rate_backend: emb, .. + } => { + if let Some(ref from_file) = rate_from_file { + let rate_json_path = inv + .rate_backend_json_path + .expect("rate backend JSON path when secondary file parsed"); + if let Err(err) = assert_rate_backend_json_matches_embedded( + emb, + from_file, + rate_json_path, + cb_path, + ) { + eprintln!("Error: {err}"); + std::process::exit(1); + } + } + emb.clone() + } + #[cfg(feature = "backend-rwkv")] + infotheory::api::CompressionBackend::Rwkv7 { method, .. } => { + let companion = infotheory::api::RateBackend::Rwkv7Method { + method: method.clone(), + }; + if let Some(ref from_file) = rate_from_file { + let rate_json_path = inv + .rate_backend_json_path + .expect("rate backend JSON path when secondary file parsed"); + if let Err(err) = assert_rate_backend_json_matches_embedded( + &companion, + from_file, + rate_json_path, + cb_path, + ) { + eprintln!("Error: {err}"); + std::process::exit(1); + } + } + companion + } + infotheory::api::CompressionBackend::Zpaq { .. } => { + if let Some(from_file) = rate_from_file { + from_file + } else { + rate_backend_from_cli_shorthand( + inv.rate_backend, + inv.method, + inv.expert_spec_path, + inv.fac_ctw_msb_first, + ) + } + } + _ => { + eprintln!( + "Error: --compression-backend-json produced a compression backend variant not supported by this CLI build" + ); + std::process::exit(1); + } + }; + + return BuiltCtx { + ctx: InfotheoryCtx::from_specs(rate_backend_ast, compression_backend_ast) + .unwrap_or_else(|e| { + eprintln!("Error: {e}"); + std::process::exit(1); + }), + }; + } + + if let Some(rb_path) = inv.rate_backend_json_path { + let (rv, rfull) = + load_backend_spec_json(rb_path, "rate backend spec").unwrap_or_else(|e| { + eprintln!("Error: {e}"); + std::process::exit(1); + }); + let rbase = rfull.parent().unwrap_or_else(|| std::path::Path::new(".")); + let rate_backend_ast = infotheory::spec::parse_rate_backend_json( + &rv, + rbase, + infotheory::api::MAX_MIXTURE_NESTING, + ) + .unwrap_or_else(|e| { + eprintln!("Error: {e}"); + std::process::exit(1); + }); + let compression_backend_ast = compression_backend_from_cli_shorthand( + &rate_backend_ast, + inv.compression_backend, + inv.method, + ); + return BuiltCtx { + ctx: InfotheoryCtx::from_specs(rate_backend_ast, compression_backend_ast) + .unwrap_or_else(|e| { + eprintln!("Error: {e}"); + std::process::exit(1); + }), + }; + } + + let rate_backend_ast = rate_backend_from_cli_shorthand( + inv.rate_backend, + inv.method, + inv.expert_spec_path, + inv.fac_ctw_msb_first, + ); + let compression_backend_ast = compression_backend_from_cli_shorthand( + &rate_backend_ast, + inv.compression_backend, + inv.method, + ); + BuiltCtx { + ctx: InfotheoryCtx::from_specs(rate_backend_ast, compression_backend_ast).unwrap_or_else( + |e| { + eprintln!("Error: {e}"); + std::process::exit(1); + }, + ), + } +} + +/// Builds a context from shorthand flags only. Prefer [`build_ctx_invocation`] when using JSON specs. +#[allow(dead_code)] +pub(super) fn build_ctx( + rate_backend: &str, + compression_backend: &str, + method: Option<&str>, + expert_spec_path: Option<&str>, +) -> BuiltCtx { + build_ctx_invocation(CliBackendInvocation { + rate_backend, + compression_backend, + method, + expert_spec_path, + rate_backend_json_path: None, + compression_backend_json_path: None, + fac_ctw_msb_first: None, + flags: CliBackendSourceFlags::default(), + }) +} + +pub(super) fn read_file(path: &str) -> Vec { + match std::fs::read(path) { + Ok(data) => data, + Err(e) => { + eprintln!("Error reading file '{}': {}", path, e); + std::process::exit(1); + } + } +} + +#[cfg_attr(not(feature = "backend-sequitur"), allow(dead_code))] +pub(super) fn parse_hex_bytes(raw: &str) -> anyhow::Result> { + fn nibble(byte: u8) -> anyhow::Result { + match byte { + b'0'..=b'9' => Ok(byte - b'0'), + b'a'..=b'f' => Ok(byte - b'a' + 10), + b'A'..=b'F' => Ok(byte - b'A' + 10), + _ => Err(anyhow::anyhow!("invalid hex digit '{}'", byte as char)), + } + } + + let cleaned: Vec = raw + .bytes() + .filter(|b| !matches!(b, b' ' | b'\n' | b'\r' | b'\t' | b'_')) + .collect(); + if !cleaned.len().is_multiple_of(2) { + return Err(anyhow::anyhow!( + "hex input must have an even number of digits" + )); + } + let mut out = Vec::with_capacity(cleaned.len() / 2); + let mut i = 0usize; + while i < cleaned.len() { + let hi = nibble(cleaned[i])?; + let lo = nibble(cleaned[i + 1])?; + out.push((hi << 4) | lo); + i += 2; + } + Ok(out) +} + +#[cfg_attr(not(feature = "backend-sequitur"), allow(dead_code))] +pub(super) fn bytes_to_hex(bytes: &[u8]) -> String { + const HEX: &[u8; 16] = b"0123456789abcdef"; + let mut out = String::with_capacity(bytes.len() * 2); + for &byte in bytes { + out.push(HEX[(byte >> 4) as usize] as char); + out.push(HEX[(byte & 0x0F) as usize] as char); + } + out +} + +pub(super) fn read_stdin_all_for_generate() -> Vec { + let stdin = io::stdin(); + if stdin.is_terminal() { + eprintln!("Error: 'generate' requires or piped stdin"); + std::process::exit(1); + } + let mut data = Vec::new(); + if let Err(e) = stdin.lock().read_to_end(&mut data) { + eprintln!("Error reading stdin: {e}"); + std::process::exit(1); + } + data +} + +#[cfg(test)] +pub(super) fn file_roundtrip_backend(backend: &CompressionBackend) -> CompressionBackend { + infotheory::backends::normalize_file_roundtrip_backend(backend) +} + +pub(super) fn file_roundtrip_compiled_backend( + backend: &infotheory::spec::CompiledCompressionBackend, +) -> infotheory::spec::CompiledCompressionBackend { + infotheory::backends::normalize_file_roundtrip_compiled_backend(backend) +} + +pub(super) fn maybe_export_online_model( + export_path: Option<&str>, + ctx: &InfotheoryCtx, + parts: &[&[u8]], +) -> anyhow::Result<()> { + let Some(path) = export_path else { + return Ok(()); + }; + + #[cfg(not(any(feature = "backend-rwkv", feature = "backend-mamba")))] + let _ = (ctx, parts, path); + + #[cfg(feature = "backend-rwkv")] + { + let rwkv_method = infotheory::backends::rate_backend_method_string_compiled( + &ctx.rate_backend, + infotheory::backends::MethodBackendFamily::Rwkv7, + ) + .or_else(|| { + infotheory::backends::compression_backend_method_string_compiled( + &ctx.compression_backend, + infotheory::backends::MethodBackendFamily::Rwkv7, + ) + }); + if let Some(method) = rwkv_method { + let mut compressor = rwkvzip::Compressor::new_from_method(method)?; + let _ = compressor.compress_size_chain(parts, infotheory::coders::CoderType::AC)?; + compressor.export_online(path)?; + return Ok(()); + } + } + + #[cfg(feature = "backend-mamba")] + { + let mamba_method = infotheory::backends::rate_backend_method_string_compiled( + &ctx.rate_backend, + infotheory::backends::MethodBackendFamily::Mamba, + ) + .or_else(|| { + infotheory::backends::compression_backend_method_string_compiled( + &ctx.compression_backend, + infotheory::backends::MethodBackendFamily::Mamba, + ) + }); + if let Some(method) = mamba_method { + let mut compressor = mambazip::Compressor::new_from_method(method)?; + let _ = compressor.compress_size_chain(parts, infotheory::coders::CoderType::AC)?; + compressor.export_online(path)?; + return Ok(()); + } + } + + eprintln!( + "Warning: --model-export was requested but the current backend does not support \ + online model export. Only RWKV7 and Mamba method-based backends support export." + ); + Ok(()) +} + +/// ROSA-based symmetric codelength distance (NCD-like but faster) +/// d_ROSA(x,y) = 0.5 * (H_y(x)/H_x(x) + H_x(y)/H_y(y)) - 1 +/// Clamped to [0, 1] +fn json_error(message: impl std::fmt::Display) -> String { + serde_json::json!({ + "error": message.to_string(), + }) + .to_string() +} + +fn try_metrics_summary(data: &[u8]) -> InfotheoryResult<(f64, f64, f64, usize)> { + let h0 = empirical_entropy_bytes(data); + let h_rate = try_entropy_rate_bytes(data)?; + let id = if h0 < 1e-9 { + 0.0 + } else { + ((h0 - h_rate) / h0).clamp(0.0, 1.0) + }; + Ok((h0, h_rate, id, data.len())) +} + +fn format_metrics_json(h0: f64, h_rate: f64, id: f64, len: usize) -> String { + format!( + r#"{{"h0":{:.6},"h_rate":{:.6},"id":{:.6},"len":{}}}"#, + h0, h_rate, id, len + ) +} + +fn parse_ncd_variant_name(variant: &str) -> NcdVariant { + match variant { + "sym" | "sym_vitanyi" => NcdVariant::SymVitanyi, + "cons" => NcdVariant::Cons, + "sym_cons" => NcdVariant::SymCons, + _ => NcdVariant::Vitanyi, + } +} + +fn compiled_compression_backend_from_json( + value: &serde_json::Value, +) -> InfotheoryResult { + let Some(backend_value) = value + .get("compression_backend") + .or_else(|| value.get("backend")) + else { + return Ok(get_default_ctx()?.compression_backend); + }; + let backend = infotheory::spec::parse_compression_backend_json( + backend_value, + Path::new("."), + None, + infotheory::compression::FramingMode::Raw, + ) + .map_err(|err| infotheory::error::InfotheoryError::invalid_backend_config(err.to_string()))?; + backend + .compile() + .map_err(|err| infotheory::error::InfotheoryError::invalid_backend_config(err.to_string())) +} + +fn rosa_distance(x: &[u8], y: &[u8]) -> InfotheoryResult { + if x.is_empty() || y.is_empty() { + return Ok(1.0); + } + + let h_x_x = try_biased_entropy_rate_bytes(x)?; + let h_y_y = try_biased_entropy_rate_bytes(y)?; + let h_y_x = try_cross_entropy_rate_bytes(x, y)?; + let h_x_y = try_cross_entropy_rate_bytes(y, x)?; + + if h_x_x < 1e-9 || h_y_y < 1e-9 { + return Ok(1.0); + } + + Ok((0.5 * (h_y_x / h_x_x + h_x_y / h_y_y) - 1.0).clamp(0.0, 1.0)) +} + +/// Process a single JSON line and return result. +pub(super) fn process_json_line(line: &str) -> String { + let line = line.trim(); + if line.is_empty() { + return r#"{"error":"empty input"}"#.to_string(); + } + + let v: serde_json::Value = match serde_json::from_str(line) { + Ok(v) => v, + Err(e) => { + return serde_json::json!({ + "error": format!("invalid json: {e}") + }) + .to_string(); + } + }; + let op = v.get("op").and_then(|x| x.as_str()).unwrap_or(""); + + match op { + "metrics" => { + let text = v + .get("text") + .and_then(|x| x.as_str()) + .unwrap_or_default() + .to_string(); + let data = text.as_bytes(); + + if data.is_empty() { + return r#"{"error":"empty text"}"#.to_string(); + } + + match try_metrics_summary(data) { + Ok((h0, h_rate, id, len)) => format_metrics_json(h0, h_rate, id, len), + Err(err) => json_error(format!("metrics failed: {err}")), + } + } + "metrics_file" => { + let path = v + .get("path") + .and_then(|x| x.as_str()) + .unwrap_or_default() + .to_string(); + + match std::fs::read(&path) { + Ok(data) => match try_metrics_summary(&data) { + Ok((h0, h_rate, id, len)) => format_metrics_json(h0, h_rate, id, len), + Err(err) => json_error(format!("metrics_file failed: {err}")), + }, + Err(err) => json_error(format!("failed to read file: {err}")), + } + } + "ncd" => { + let text1 = v + .get("text1") + .and_then(|x| x.as_str()) + .unwrap_or_default() + .to_string(); + let text2 = v + .get("text2") + .and_then(|x| x.as_str()) + .unwrap_or_default() + .to_string(); + let method = v + .get("method") + .and_then(|x| x.as_str()) + .unwrap_or("5") + .to_string(); + let variant = v + .get("variant") + .and_then(|x| x.as_str()) + .unwrap_or("vitanyi") + .to_string(); + + let x = text1.as_bytes(); + let y = text2.as_bytes(); + if x.is_empty() || y.is_empty() { + return r#"{"error":"empty text(s)"}"#.to_string(); + } + + let ncd_variant = parse_ncd_variant_name(&variant); + + let ncd_result = if v.get("compression_backend").is_some() || v.get("backend").is_some() + { + compiled_compression_backend_from_json(&v) + .and_then(|backend| try_ncd_bytes_backend(x, y, &backend, ncd_variant)) + } else if cfg!(feature = "backend-zpaq") { + CompressionBackend::zpaq(method.as_str()) + .compile() + .map_err(|err| infotheory::error::InfotheoryError::invalid_backend_config(err.to_string())) + .and_then(|backend| try_ncd_bytes_backend(x, y, &backend, ncd_variant)) + } else { + try_ncd_bytes_default(x, y, ncd_variant) + }; + + match ncd_result { + Ok(ncd) => format!(r#"{{"ncd":{:.6}}}"#, ncd), + Err(err) => json_error(format!("ncd failed: {err}")), + } + } + "ncd_files" => { + let path1 = v + .get("path1") + .and_then(|x| x.as_str()) + .unwrap_or_default() + .to_string(); + let path2 = v + .get("path2") + .and_then(|x| x.as_str()) + .unwrap_or_default() + .to_string(); + let method = v + .get("method") + .and_then(|x| x.as_str()) + .unwrap_or("5") + .to_string(); + let variant = v + .get("variant") + .and_then(|x| x.as_str()) + .unwrap_or("vitanyi") + .to_string(); + + let ncd_variant = parse_ncd_variant_name(&variant); + + let ncd_result = if v.get("compression_backend").is_some() || v.get("backend").is_some() + { + compiled_compression_backend_from_json(&v) + .and_then(|backend| try_ncd_paths_compiled_backend(&path1, &path2, &backend, ncd_variant)) + } else if cfg!(feature = "backend-zpaq") { + try_ncd_paths_backend( + &path1, + &path2, + &CompressionBackend::zpaq(method.as_str()), + ncd_variant, + ) + } else { + let (left, right) = rayon::join(|| std::fs::read(&path1), || std::fs::read(&path2)); + match (left, right) { + (Ok(left), Ok(right)) => try_ncd_bytes_default(&left, &right, ncd_variant), + (Err(err), _) | (_, Err(err)) => Err(infotheory::error::InfotheoryError::from(err)), + } + }; + + match ncd_result { + Ok(ncd) => format!(r#"{{"ncd":{:.6}}}"#, ncd), + Err(err) => json_error(format!("ncd_files failed: {err}")), + } + } + "rosa_dist" => { + let text1 = v + .get("text1") + .and_then(|x| x.as_str()) + .unwrap_or_default() + .to_string(); + let text2 = v + .get("text2") + .and_then(|x| x.as_str()) + .unwrap_or_default() + .to_string(); + + let x = text1.as_bytes(); + let y = text2.as_bytes(); + if x.is_empty() || y.is_empty() { + return r#"{"error":"empty text(s)"}"#.to_string(); + } + + match rosa_distance(x, y) { + Ok(dist) => format!(r#"{{"rosa_dist":{:.6}}}"#, dist), + Err(err) => json_error(format!("rosa_dist failed: {err}")), + } + } + "cross_entropy" => { + let text_x = v + .get("text_x") + .and_then(|x| x.as_str()) + .unwrap_or_default() + .to_string(); + let text_y = v + .get("text_y") + .and_then(|x| x.as_str()) + .unwrap_or_default() + .to_string(); + + let x = text_x.as_bytes(); + let y = text_y.as_bytes(); + if x.is_empty() || y.is_empty() { + return r#"{"error":"empty text(s)"}"#.to_string(); + } + + match try_cross_entropy_rate_bytes(x, y) { + Ok(xe) => format!(r#"{{"cross_entropy":{:.6}}}"#, xe), + Err(err) => json_error(format!("cross_entropy failed: {err}")), + } + } + "batch_metrics" => { + let texts: Vec = v + .get("texts") + .and_then(|x| x.as_array()) + .map(|arr| { + arr.iter() + .filter_map(|item| item.as_str().map(ToString::to_string)) + .collect() + }) + .unwrap_or_default(); + + let results: Vec = texts + .iter() + .map(|text| { + let data = text.as_bytes(); + if data.is_empty() { + r#"{"h0":0,"h_rate":0,"id":0,"len":0}"#.to_string() + } else { + match try_metrics_summary(data) { + Ok((h0, h_rate, id, len)) => format_metrics_json(h0, h_rate, id, len), + Err(err) => serde_json::json!({ + "error": format!("{err}"), + "len": data.len(), + }) + .to_string(), + } + } + }) + .collect(); + + format!(r#"{{"results":[{}]}}"#, results.join(",")) + } + "ncd_matrix" => { + let texts: Vec = v + .get("texts") + .and_then(|x| x.as_array()) + .map(|arr| { + arr.iter() + .filter_map(|item| item.as_str().map(ToString::to_string)) + .collect() + }) + .unwrap_or_default(); + let method = v + .get("method") + .and_then(|x| x.as_str()) + .unwrap_or("5") + .to_string(); + let variant = v + .get("variant") + .and_then(|x| x.as_str()) + .unwrap_or("vitanyi") + .to_string(); + + let ncd_variant = parse_ncd_variant_name(&variant); + + let datas: Vec> = texts.iter().map(|t| t.as_bytes().to_vec()).collect(); + let matrix_result = + if v.get("compression_backend").is_some() || v.get("backend").is_some() { + compiled_compression_backend_from_json(&v) + .and_then(|backend| try_ncd_matrix_bytes_backend(&datas, &backend, ncd_variant)) + } else if cfg!(feature = "backend-zpaq") { + CompressionBackend::zpaq(method.as_str()) + .compile() + .map_err(|err| infotheory::error::InfotheoryError::invalid_backend_config(err.to_string())) + .and_then(|backend| try_ncd_matrix_bytes_backend(&datas, &backend, ncd_variant)) + } else { + try_ncd_matrix_bytes_default(&datas, ncd_variant) + }; + let matrix = match matrix_result { + Ok(matrix) => matrix, + Err(err) => return json_error(format!("ncd_matrix failed: {err}")), + }; + let n = datas.len(); + + let rows: Vec = (0..n) + .map(|i| { + let row: Vec = (0..n) + .map(|j| format!("{:.6}", matrix[i * n + j])) + .collect(); + format!("[{}]", row.join(",")) + }) + .collect(); + + format!(r#"{{"matrix":[{}],"n":{}}}"#, rows.join(","), n) + } + "rosa_matrix" => { + let texts: Vec = v + .get("texts") + .and_then(|x| x.as_array()) + .map(|arr| { + arr.iter() + .filter_map(|item| item.as_str().map(ToString::to_string)) + .collect() + }) + .unwrap_or_default(); + + let n = texts.len(); + let datas: Vec<&[u8]> = texts.iter().map(|t| t.as_bytes()).collect(); + let mut matrix = vec![0.0f64; n * n]; + for i in 0..n { + for j in i..n { + let d = if i == j { + 0.0 + } else { + match rosa_distance(datas[i], datas[j]) { + Ok(dist) => dist, + Err(err) => { + return json_error(format!("rosa_matrix failed: {err}")); + } + } + }; + matrix[i * n + j] = d; + matrix[j * n + i] = d; + } + } + + let rows: Vec = (0..n) + .map(|i| { + let row: Vec = (0..n) + .map(|j| format!("{:.6}", matrix[i * n + j])) + .collect(); + format!("[{}]", row.join(",")) + }) + .collect(); + + format!(r#"{{"matrix":[{}],"n":{}}}"#, rows.join(","), n) + } + "spam_check" => { + let text = v + .get("text") + .and_then(|x| x.as_str()) + .unwrap_or_default() + .to_string(); + let h0_threshold = v.get("h0_min").and_then(|x| x.as_f64()).unwrap_or(1.0); + let h_rate_threshold = v.get("h_rate_min").and_then(|x| x.as_f64()).unwrap_or(0.5); + let id_threshold = v.get("id_max").and_then(|x| x.as_f64()).unwrap_or(0.95); + let min_len = v.get("min_len").and_then(|x| x.as_i64()).unwrap_or(10) as usize; + + let data = text.as_bytes(); + let len = data.len(); + if len < min_len { + return format!(r#"{{"pass":false,"reason":"too_short","len":{}}}"#, len); + } + + let h0 = empirical_entropy_bytes(data); + if h0 < h0_threshold { + return format!(r#"{{"pass":false,"reason":"low_entropy","h0":{:.4}}}"#, h0); + } + + let h_rate = match try_entropy_rate_bytes(data) { + Ok(h_rate) => h_rate, + Err(err) => return json_error(format!("spam_check failed: {err}")), + }; + if h_rate < h_rate_threshold { + return format!( + r#"{{"pass":false,"reason":"low_entropy_rate","h_rate":{:.4}}}"#, + h_rate + ); + } + + let id = if h0 < 1e-9 { + 0.0 + } else { + ((h0 - h_rate) / h0).clamp(0.0, 1.0) + }; + if id > id_threshold { + return format!( + r#"{{"pass":false,"reason":"high_redundancy","id":{:.4}}}"#, + id + ); + } + + format!( + r#"{{"pass":true,"h0":{:.4},"h_rate":{:.4},"id":{:.4},"len":{}}}"#, + h0, h_rate, id, len + ) + } + "help" => { + r#"{"ops":["metrics","metrics_file","ncd","ncd_files","rosa_dist","cross_entropy","batch_metrics","ncd_matrix","rosa_matrix","spam_check"]}"#.to_string() + } + _ => format!(r#"{{"error":"unknown op: {}"}}"#, op), + } +} + +pub(super) fn run_batch_mode() { + let stdin = io::stdin(); + for line in stdin.lock().lines() { + match line { + Ok(l) => println!("{}", process_json_line(&l)), + Err(_) => continue, + } + } +} + +#[cfg(test)] +mod non_vm_tests { + use super::*; + use serde_json::Value; + use std::fs; + use std::time::{SystemTime, UNIX_EPOCH}; + + fn unique_temp_path(label: &str, ext: &str) -> std::path::PathBuf { + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("system clock must be after unix epoch") + .as_nanos(); + std::env::temp_dir().join(format!( + "infotheory-cli-tests-{label}-{}-{nonce}.{ext}", + std::process::id() + )) + } + + fn parse_json_output(line: &str) -> Value { + serde_json::from_str(line).expect("output should be valid json") + } + + fn has_default_rate_backend() -> bool { + RateBackend::try_default().is_ok() + } + + fn has_default_compression_backend() -> bool { + CompressionBackend::try_default().is_ok() + } + + fn assert_backend_unavailable_error(output: &Value) { + let err = output["error"] + .as_str() + .expect("output should contain an error string"); + assert!( + err.contains("no default rate backend is available in this build") + || err.contains("requires infotheory feature 'backend-zpaq'") + || err.contains("CompressionBackend::Zpaq is unavailable"), + "unexpected backend-availability error: {output}" + ); + } + + #[test] + fn hex_helpers_roundtrip_and_reject_invalid_inputs() { + let parsed = parse_hex_bytes("00 ff_10\n7A").expect("hex string should parse"); + assert_eq!(parsed, vec![0x00, 0xff, 0x10, 0x7a]); + assert_eq!(bytes_to_hex(&parsed), "00ff107a"); + + let err = parse_hex_bytes("abc").expect_err("odd hex digit count must fail"); + assert!( + err.to_string() + .contains("hex input must have an even number of digits"), + "unexpected error: {err}" + ); + + let err = parse_hex_bytes("0g").expect_err("invalid digit must fail"); + assert!( + err.to_string().contains("invalid hex digit 'g'"), + "unexpected error: {err}" + ); + } + + #[test] + fn process_json_line_reports_help_and_input_errors() { + let help = parse_json_output(&process_json_line(r#"{ "op": "help" }"#)); + let ops = help["ops"].as_array().expect("help ops array"); + assert!(ops.iter().any(|value| value == "metrics")); + assert!(ops.iter().any(|value| value == "spam_check")); + + let empty = parse_json_output(&process_json_line(" ")); + assert_eq!(empty["error"], "empty input"); + + let invalid = parse_json_output(&process_json_line("{ invalid")); + assert!( + invalid["error"] + .as_str() + .expect("error string") + .contains("invalid json"), + "unexpected invalid-json output: {invalid}" + ); + + let unknown = parse_json_output(&process_json_line(r#"{ "op": "nope" }"#)); + assert_eq!(unknown["error"], "unknown op: nope"); + } + + #[test] + fn process_json_line_handles_metrics_batch_and_spam_semantics() { + let has_rate_backend = has_default_rate_backend(); + let metrics = parse_json_output(&process_json_line( + r#"{ "op": "metrics", "text": "banana bandana" }"#, + )); + if has_rate_backend { + assert_eq!(metrics["len"], 14); + assert!(metrics["h0"].as_f64().expect("h0") >= 0.0); + assert!(metrics["h_rate"].as_f64().expect("h_rate") >= 0.0); + } else { + assert_backend_unavailable_error(&metrics); + } + + let batch = parse_json_output(&process_json_line( + r#"{ "op": "batch_metrics", "texts": ["abcabcabc", ""] }"#, + )); + let results = batch["results"].as_array().expect("batch results"); + assert_eq!(results.len(), 2); + if has_rate_backend { + assert_eq!(results[0]["len"], 9); + assert!(results[0]["h_rate"].as_f64().expect("h_rate") >= 0.0); + } else { + assert_backend_unavailable_error(&results[0]); + assert_eq!(results[0]["len"], 9); + } + assert_eq!(results[1]["len"], 0); + assert_eq!(results[1]["h_rate"], 0); + + let too_short = parse_json_output(&process_json_line( + r#"{ "op": "spam_check", "text": "tiny", "min_len": 10 }"#, + )); + assert_eq!(too_short["pass"], false); + assert_eq!(too_short["reason"], "too_short"); + assert_eq!(too_short["len"], 4); + + let pass = parse_json_output(&process_json_line( + r#"{ "op": "spam_check", "text": "bananas foster waffle cartography", "min_len": 8, "h0_min": 0.0, "h_rate_min": 0.0, "id_max": 1.0 }"#, + )); + if has_rate_backend { + assert_eq!(pass["pass"], true); + assert_eq!(pass["len"], 33); + } else { + assert_backend_unavailable_error(&pass); + } + } + + #[test] + fn process_json_line_emits_structured_matrix_and_file_results() { + let has_rate_backend = has_default_rate_backend(); + let has_compression_backend = has_default_compression_backend(); + let file_path = unique_temp_path("metrics-file", "txt"); + fs::write(&file_path, b"structured metrics fixture").expect("write metrics file"); + + let metrics_file_input = serde_json::json!({ + "op": "metrics_file", + "path": file_path.display().to_string() + }); + let metrics_file = parse_json_output(&process_json_line(&metrics_file_input.to_string())); + if has_rate_backend { + assert_eq!(metrics_file["len"], 26); + assert!(metrics_file["id"].as_f64().expect("id") >= 0.0); + } else { + assert_backend_unavailable_error(&metrics_file); + } + + let ncd_matrix = parse_json_output(&process_json_line( + r#"{ "op": "ncd_matrix", "texts": ["aaaa", "aaab"] }"#, + )); + if has_compression_backend { + assert_eq!(ncd_matrix["n"], 2); + let matrix = ncd_matrix["matrix"].as_array().expect("matrix rows"); + assert_eq!(matrix.len(), 2); + assert_eq!(matrix[0][0], 0.0); + assert_eq!(matrix[1][1], 0.0); + } else { + assert_backend_unavailable_error(&ncd_matrix); + } + + let rosa_matrix = parse_json_output(&process_json_line( + r#"{ "op": "rosa_matrix", "texts": ["alpha alpha", "alpha beta"] }"#, + )); + if has_rate_backend { + assert_eq!(rosa_matrix["n"], 2); + let rosa_rows = rosa_matrix["matrix"].as_array().expect("rosa matrix rows"); + assert_eq!(rosa_rows.len(), 2); + assert_eq!(rosa_rows[0][0], 0.0); + assert_eq!(rosa_rows[1][1], 0.0); + } else { + assert_backend_unavailable_error(&rosa_matrix); + } + + let _ = fs::remove_file(file_path); + } + + #[test] + fn process_json_line_covers_pairwise_ops_and_contract_errors() { + let has_rate_backend = has_default_rate_backend(); + let has_compression_backend = has_default_compression_backend(); + let ncd = parse_json_output(&process_json_line( + r#"{ "op": "ncd", "text1": "abracadabra", "text2": "alakazam", "method": "5", "variant": "sym_cons" }"#, + )); + if has_compression_backend { + assert!(ncd["ncd"].as_f64().expect("ncd value").is_finite()); + } else { + assert_backend_unavailable_error(&ncd); + } + + let cross = parse_json_output(&process_json_line( + r#"{ "op": "cross_entropy", "text_x": "abracadabra", "text_y": "alakazam" }"#, + )); + if has_rate_backend { + assert!( + cross["cross_entropy"] + .as_f64() + .expect("cross entropy value") + .is_finite() + ); + } else { + assert_backend_unavailable_error(&cross); + } + + let rosa = parse_json_output(&process_json_line( + r#"{ "op": "rosa_dist", "text1": "alpha alpha alpha", "text2": "alpha beta alpha" }"#, + )); + if has_rate_backend { + let rosa_dist = rosa["rosa_dist"].as_f64().expect("rosa distance value"); + assert!(rosa_dist.is_finite()); + assert!((0.0..=1.0).contains(&rosa_dist)); + } else { + assert_backend_unavailable_error(&rosa); + } + + let left_path = unique_temp_path("ncd-left", "txt"); + let right_path = unique_temp_path("ncd-right", "txt"); + fs::write(&left_path, b"left fixture bytes").expect("write left fixture"); + fs::write(&right_path, b"right fixture bytes").expect("write right fixture"); + + let ncd_files_input = serde_json::json!({ + "op": "ncd_files", + "path1": left_path.display().to_string(), + "path2": right_path.display().to_string(), + "method": "5", + "variant": "cons" + }); + let ncd_files = parse_json_output(&process_json_line(&ncd_files_input.to_string())); + if has_compression_backend { + assert!( + ncd_files["ncd"] + .as_f64() + .expect("ncd file value") + .is_finite() + ); + } else { + assert_backend_unavailable_error(&ncd_files); + } + + let empty_ncd = parse_json_output(&process_json_line( + r#"{ "op": "ncd", "text1": "", "text2": "non-empty" }"#, + )); + assert_eq!(empty_ncd["error"], "empty text(s)"); + + let empty_cross = parse_json_output(&process_json_line( + r#"{ "op": "cross_entropy", "text_x": "", "text_y": "non-empty" }"#, + )); + assert_eq!(empty_cross["error"], "empty text(s)"); + + let empty_rosa = parse_json_output(&process_json_line( + r#"{ "op": "rosa_dist", "text1": "", "text2": "non-empty" }"#, + )); + assert_eq!(empty_rosa["error"], "empty text(s)"); + + let empty_metrics = + parse_json_output(&process_json_line(r#"{ "op": "metrics", "text": "" }"#)); + assert_eq!(empty_metrics["error"], "empty text"); + + let missing_metrics_file = parse_json_output(&process_json_line( + r#"{ "op": "metrics_file", "path": "/definitely/missing/file/path" }"#, + )); + assert!( + missing_metrics_file["error"] + .as_str() + .expect("metrics_file error string") + .contains("failed to read file") + ); + + let _ = fs::remove_file(left_path); + let _ = fs::remove_file(right_path); + } + + #[test] + fn process_json_line_spam_check_reasons_cover_entropy_guards() { + let has_rate_backend = has_default_rate_backend(); + let low_entropy = parse_json_output(&process_json_line( + r#"{ "op": "spam_check", "text": "aaaaaaaaaaaa", "min_len": 4, "h0_min": 3.0, "h_rate_min": 0.0, "id_max": 1.0 }"#, + )); + assert_eq!(low_entropy["pass"], false); + assert_eq!(low_entropy["reason"], "low_entropy"); + + let low_entropy_rate = parse_json_output(&process_json_line( + r#"{ "op": "spam_check", "text": "abcdefghijklmno", "min_len": 4, "h0_min": 0.0, "h_rate_min": 1000.0, "id_max": 1.0 }"#, + )); + if has_rate_backend { + assert_eq!(low_entropy_rate["pass"], false); + assert_eq!(low_entropy_rate["reason"], "low_entropy_rate"); + } else { + assert_backend_unavailable_error(&low_entropy_rate); + } + + let high_redundancy = parse_json_output(&process_json_line( + r#"{ "op": "spam_check", "text": "abababababababab", "min_len": 4, "h0_min": 0.0, "h_rate_min": 0.0, "id_max": -1.0 }"#, + )); + if has_rate_backend { + assert_eq!(high_redundancy["pass"], false); + assert_eq!(high_redundancy["reason"], "high_redundancy"); + } else { + assert_backend_unavailable_error(&high_redundancy); + } + } + + #[cfg(feature = "backend-ctw")] + #[test] + fn process_json_line_ncd_accepts_explicit_rate_coded_compression_backend() { + let ncd = parse_json_output(&process_json_line( + r#"{ + "op": "ncd", + "text1": "abracadabra", + "text2": "alakazam", + "variant": "sym", + "compression_backend": { + "kind": "rate-ac", + "rate_backend": { "kind": "ctw", "depth": 4 }, + "framing": "raw" + } + }"#, + )); + assert!(ncd["ncd"].as_f64().expect("ncd value").is_finite()); + + let matrix = parse_json_output(&process_json_line( + r#"{ + "op": "ncd_matrix", + "texts": ["aaaa", "aaab"], + "compression_backend": { + "kind": "rate-ac", + "rate_backend": { "kind": "ctw", "depth": 4 }, + "framing": "raw" + } + }"#, + )); + assert_eq!(matrix["n"], 2); + let rows = matrix["matrix"].as_array().expect("matrix rows"); + assert_eq!(rows.len(), 2); + assert_eq!(rows[0][0], 0.0); + assert_eq!(rows[1][1], 0.0); + } +} + +#[cfg(all(test, feature = "vm"))] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn parse_nyx_observation_policy_accepts_canonical_names() { + let cases = [ + ("from_guest", NyxObservationPolicy::FromGuest), + ("output_hash", NyxObservationPolicy::OutputHash), + ("raw_output", NyxObservationPolicy::RawOutput), + ("shared_memory", NyxObservationPolicy::SharedMemory), + ]; + + for (mode, expected) in cases { + let parsed = + parse_nyx_observation_policy(&json!({ "mode": mode })).expect("parse mode"); + assert!( + std::mem::discriminant(&parsed) == std::mem::discriminant(&expected), + "mode {mode} parsed as {parsed:?}" + ); + } + + let err = parse_nyx_observation_policy(&json!({ "mode": "guest" })) + .expect_err("legacy alias should be rejected"); + assert!( + err.to_string().contains("unknown VM observation policy"), + "unexpected error: {err}" + ); + } +} diff --git a/crates/infotheory/src/cli/planner_run.rs b/crates/infotheory/src/cli/planner_run.rs new file mode 100644 index 00000000..c8028da3 --- /dev/null +++ b/crates/infotheory/src/cli/planner_run.rs @@ -0,0 +1,1021 @@ +use infotheory::aixi::common::{Action, Reward, observation_repr_from_stream}; +use infotheory::aixi::planner_agent::{ + PlannerActionProvenance, PlannerAgentError, PlannerControllerAgent, PlannerCycleObserver, + PlannerEnvironment, PlannerPhase, PlannerRunSession, PlannerSchedule, + build_planner_environment, +}; +use infotheory::aixi::warmstart::{warmstart_jsonl_action_record, warmstart_jsonl_percept_record}; +#[cfg(test)] +use infotheory::spec::BuiltinEnvironmentSpec; +use infotheory::spec::{self, CompiledPlannerController, CompiledPlannerRunSpec, SpecDocument}; +use std::fs::File; +use std::io::{BufWriter, Write}; +use std::path::Path; +use std::time::Instant; + +#[cfg(test)] +pub(crate) fn is_canonical_spec_document(value: &serde_json::Value) -> bool { + is_canonical_spec_document_runtime(value) +} + +#[cfg(test)] +pub(crate) fn builtin_environment_name(spec: BuiltinEnvironmentSpec) -> &'static str { + spec.canonical_name() +} + +fn is_canonical_spec_document_runtime(value: &serde_json::Value) -> bool { + value["schema_version"].as_u64().is_some() && value["kind"].as_str().is_some() +} + +fn legacy_interface_reward_range_field(value: &serde_json::Value) -> Option<&'static str> { + let interface = value.get("interface")?.as_object()?; + ["min_reward", "max_reward", "reward_offset"] + .into_iter() + .find(|field| interface.contains_key(*field)) +} + +#[cfg(test)] +pub(crate) fn legacy_planner_config_error(path: &str) -> anyhow::Error { + legacy_planner_config_error_runtime(path) +} + +fn legacy_planner_config_error_runtime(path: &str) -> anyhow::Error { + anyhow::anyhow!( + "legacy aixi JSON configs are no longer executable; convert '{}' to a canonical planner_run document with top-level 'schema_version' and 'kind'", + path + ) +} + +fn legacy_interface_reward_range_error(path: &str, field: &str) -> anyhow::Error { + anyhow::anyhow!( + "unknown interface field '{field}': legacy aixi reward-range fields are no longer accepted in planner_run documents; remove 'min_reward', 'max_reward', and 'reward_offset' from '{}' and use the canonical reward_bits-derived interface contract", + path + ) +} + +/// Planner telemetry logger for complete observable interaction traces. +/// +/// The JSONL sink records normalized action and percept events. The `bits01` +/// sink records the same telemetry stream in the planner's bit encoding: each +/// action contributes `action_bits` bytes, and each percept contributes all +/// observations plus reward bytes. It is a trace representation, not a +/// codelength accounting surface for the predictor's actual online updates. +pub(crate) struct AixiRunLogger { + bits01: Option>, + jsonl: Option>, + flush_every: usize, + completed_steps: usize, +} + +impl AixiRunLogger { + pub(crate) fn new(v: Option<&serde_json::Value>) -> anyhow::Result> { + let bits01_path = v.and_then(|value| value["trace_bits01_path"].as_str()); + let jsonl_path = v.and_then(|value| value["trace_jsonl_path"].as_str()); + if bits01_path.is_none() && jsonl_path.is_none() { + return Ok(None); + } + + let bits01 = if let Some(p) = bits01_path { + let f = File::create(p)?; + Some(BufWriter::new(f)) + } else { + None + }; + let jsonl = if let Some(p) = jsonl_path { + let f = File::create(p)?; + Some(BufWriter::new(f)) + } else { + None + }; + let flush_every = v + .and_then(|value| value["trace_flush_every"].as_u64()) + .unwrap_or(1024) as usize; + + Ok(Some(Self { + bits01, + jsonl, + flush_every, + completed_steps: 0, + })) + } + + fn write_bits01(&mut self, bits: &[bool]) -> anyhow::Result<()> { + if let Some(w) = self.bits01.as_mut() { + for &b in bits { + w.write_all(&[if b { 1u8 } else { 0u8 }])?; + } + } + Ok(()) + } + + pub(crate) fn flush(&mut self) -> anyhow::Result<()> { + if let Some(w) = self.bits01.as_mut() { + w.flush()?; + } + if let Some(w) = self.jsonl.as_mut() { + w.flush()?; + } + Ok(()) + } + + pub(crate) fn log_percept( + &mut self, + step: usize, + observations: &[u64], + reward: i64, + observation_bits: usize, + reward_bits: usize, + reward_offset: i64, + ) -> anyhow::Result<()> { + // Same symbol encoding as the controller, applied to the telemetry event. + let mut bits = Vec::new(); + for &obs in observations { + infotheory::aixi::common::encode(&mut bits, obs, observation_bits); + } + infotheory::aixi::common::encode_reward_offset( + &mut bits, + reward, + reward_bits, + reward_offset, + ); + + self.write_bits01(&bits)?; + + if let Some(w) = self.jsonl.as_mut() { + let rec = warmstart_jsonl_percept_record(step, observations, reward); + writeln!(w, "{rec}")?; + } + Ok(()) + } + + pub(crate) fn log_action( + &mut self, + step: usize, + action: Action, + action_bits: usize, + provenance: PlannerActionProvenance, + ) -> anyhow::Result<()> { + let mut bits = Vec::new(); + infotheory::aixi::common::encode(&mut bits, action, action_bits); + self.write_bits01(&bits)?; + + if let Some(w) = self.jsonl.as_mut() { + let rec = warmstart_jsonl_action_record(step, action, provenance); + writeln!(w, "{rec}")?; + } + Ok(()) + } + + pub(crate) fn next_step(&mut self) -> anyhow::Result<()> { + self.completed_steps = self.completed_steps.saturating_add(1); + if self.flush_every > 0 && self.completed_steps.is_multiple_of(self.flush_every) { + self.flush()?; + } + Ok(()) + } +} + +fn controller_backend_label(controller: &CompiledPlannerController) -> String { + controller.backend_label() +} + +struct PlannerCliObserver { + action_bits: usize, + observation_bits: usize, + reward_bits: usize, + reward_offset: i64, + trace_logger: Option, +} + +impl PlannerCliObserver { + fn new( + compiled: &CompiledPlannerRunSpec, + cli_overlay: Option<&serde_json::Value>, + ) -> anyhow::Result { + Ok(Self { + action_bits: planner_action_bits(compiled), + observation_bits: compiled.interface().observation_bits, + reward_bits: compiled.interface().reward_bits, + reward_offset: 0, + trace_logger: AixiRunLogger::new(cli_overlay)?, + }) + } + + fn flush(&mut self) -> Result<(), PlannerAgentError> { + if let Some(logger) = self.trace_logger.as_mut() { + logger.flush().map_err(planner_observer_error)?; + } + Ok(()) + } +} + +fn planner_observer_error(err: anyhow::Error) -> PlannerAgentError { + PlannerAgentError::Observer { + reason: err.to_string(), + } +} + +impl PlannerCycleObserver for PlannerCliObserver { + fn observe_percept( + &mut self, + step: usize, + observations: &[u64], + reward: Reward, + ) -> Result<(), PlannerAgentError> { + if let Some(logger) = self.trace_logger.as_mut() { + logger + .log_percept( + step, + observations, + reward, + self.observation_bits, + self.reward_bits, + self.reward_offset, + ) + .map_err(planner_observer_error)?; + } + Ok(()) + } + + fn observe_action( + &mut self, + step: usize, + action: Action, + provenance: PlannerActionProvenance, + ) -> Result<(), PlannerAgentError> { + if let Some(logger) = self.trace_logger.as_mut() { + logger + .log_action(step, action, self.action_bits, provenance) + .map_err(planner_observer_error)?; + } + Ok(()) + } + + fn end_cycle(&mut self, _step: usize) -> Result<(), PlannerAgentError> { + if let Some(logger) = self.trace_logger.as_mut() { + logger.next_step().map_err(planner_observer_error)?; + } + Ok(()) + } +} + +fn planner_action_bits(compiled: &CompiledPlannerRunSpec) -> usize { + compiled.interface().agent_actions.action_bits() +} + +fn print_planner_cycle_log( + compiled: &CompiledPlannerRunSpec, + step: usize, + outcome: &infotheory::aixi::planner_agent::PlannerCycleOutcome, +) { + match compiled.controller() { + CompiledPlannerController::McAixi { .. } => { + let obs_repr = observation_repr_from_stream( + compiled.interface().observation_key_mode, + &outcome.pre_observations, + compiled.interface().observation_bits, + ); + println!( + "Cycle {}: Obs={:?}, Rew={}", + step, obs_repr, outcome.pre_reward + ); + println!("Cycle {}: Planned Action={}", step, outcome.action); + } + CompiledPlannerController::AiqiDiscounted { .. } + | CompiledPlannerController::AiqiWarmstartExactJh { .. } => { + println!( + "Cycle {}: Action={} Obs={:?} Rew={}", + step, outcome.action, outcome.pre_observations, outcome.pre_reward + ); + } + _ => { + println!( + "Cycle {}: Action={} Obs={:?} Rew={}", + step, outcome.action, outcome.pre_observations, outcome.pre_reward + ); + } + } +} + +pub(crate) fn run_vm_perf_only( + schedule: &PlannerSchedule, + log_every: usize, + perf: bool, + env: &mut PlannerEnvironment, +) -> anyhow::Result<()> { + let mut obs = env.observations().first().copied().unwrap_or(0); + let mut rew = env.reward(); + let start = Instant::now(); + for step in 0..schedule.learn_cycles { + if log_every > 0 && step % log_every == 0 { + println!("Cycle {}: Obs={}, Rew={}", step, obs, rew); + } + env.perform_action(0) + .map_err(|err| anyhow::anyhow!("{err}"))?; + obs = env.observations().first().copied().unwrap_or(0); + rew = env.reward(); + } + if perf && schedule.learn_cycles > 0 { + let elapsed = start.elapsed().as_secs_f64().max(1e-9); + let cps = schedule.learn_cycles as f64 / elapsed; + println!("Perf cycles/s: {:.2}", cps); + } + Ok(()) +} + +pub(crate) fn run_compiled_planner_run( + compiled: &CompiledPlannerRunSpec, + cli_overlay: Option<&serde_json::Value>, +) -> anyhow::Result<()> { + let runtime = compiled.runtime(); + let schedule = PlannerSchedule::from_runtime(runtime); + let (env, env_name) = build_planner_environment(compiled)?; + + match compiled.controller() { + CompiledPlannerController::McAixi { .. } => println!( + "Agent initialized with {} algorithm for {} environment.", + controller_backend_label(compiled.controller()), + env_name + ), + CompiledPlannerController::AiqiDiscounted { .. } => println!( + "AIQI initialized ({}) for {} environment.", + controller_backend_label(compiled.controller()), + env_name + ), + CompiledPlannerController::AiqiWarmstartExactJh { .. } => println!( + "Warm-start exact-J_H AIQI initialized ({}) for {} environment.", + controller_backend_label(compiled.controller()), + env_name + ), + other => println!( + "Planner controller '{}' initialized ({}) for {} environment.", + other.kind_str(), + controller_backend_label(compiled.controller()), + env_name + ), + } + + if runtime.vm_perf_only { + let mut planner_env = + PlannerEnvironment::new(compiled, env).map_err(|err| anyhow::anyhow!("{err}"))?; + return run_vm_perf_only(&schedule, runtime.log_every, runtime.perf, &mut planner_env); + } + + let controller = + PlannerControllerAgent::from_compiled(compiled).map_err(|err| anyhow::anyhow!("{err}"))?; + let mut session = PlannerRunSession::new(compiled, controller, env) + .map_err(|err| anyhow::anyhow!("{err}"))?; + let mut observer = PlannerCliObserver::new(compiled, cli_overlay)?; + let learn_start = Instant::now(); + let mut eval_start: Option = None; + let mut learn_perf_reported = false; + let mut learn_total_reward: i64 = 0; + let mut eval_total_reward: i64 = 0; + + while let Some(phase) = session.next_phase() { + if phase == PlannerPhase::Eval && !learn_perf_reported { + if runtime.perf && schedule.learn_cycles > 0 { + let elapsed = learn_start.elapsed().as_secs_f64().max(1e-9); + let cps = schedule.learn_cycles as f64 / elapsed; + println!("Learn cycles/s: {:.2}", cps); + } + learn_perf_reported = true; + eval_start = Some(Instant::now()); + } + + let step = session.next_step(); + let outcome = session + .run_next_cycle(&mut observer) + .map_err(|err| anyhow::anyhow!("{err}"))? + .expect("PlannerRunSession next_phase promised a next cycle"); + if runtime.log_every > 0 && step.is_multiple_of(runtime.log_every) { + print_planner_cycle_log(compiled, step, &outcome); + } + match phase { + PlannerPhase::Learn => { + learn_total_reward = learn_total_reward.saturating_add(outcome.reward); + } + PlannerPhase::Eval => { + eval_total_reward = eval_total_reward.saturating_add(outcome.reward); + } + _ => { + return Err(anyhow::anyhow!( + "unsupported planner phase returned by PlannerRunSession" + )); + } + } + } + + observer.flush().map_err(|err| anyhow::anyhow!("{err}"))?; + + if !learn_perf_reported && runtime.perf && schedule.learn_cycles > 0 { + let elapsed = learn_start.elapsed().as_secs_f64().max(1e-9); + let cps = schedule.learn_cycles as f64 / elapsed; + println!("Learn cycles/s: {:.2}", cps); + } + + if schedule.eval_cycles > 0 { + if runtime.perf { + let elapsed = eval_start + .unwrap_or_else(Instant::now) + .elapsed() + .as_secs_f64() + .max(1e-9); + let cps = schedule.eval_cycles as f64 / elapsed; + println!("Eval cycles/s: {:.2}", cps); + } + let avg = (eval_total_reward as f64) / (schedule.eval_cycles as f64); + println!("Eval Total Reward: {}", eval_total_reward); + println!("Eval Average Reward per Cycle: {:.6}", avg); + } + + println!("Total Reward: {}", learn_total_reward); + Ok(()) +} + +pub(crate) fn run_aixi_mode(config_path: &str) -> anyhow::Result<()> { + let raw = std::fs::read(config_path)?; + let json_overlay = serde_json::from_slice::(&raw).ok(); + if let Some(value) = json_overlay.as_ref() + && !is_canonical_spec_document_runtime(value) + { + return Err(legacy_planner_config_error_runtime(config_path)); + } + if let Some(value) = json_overlay.as_ref() + && let Some(field) = legacy_interface_reward_range_field(value) + { + return Err(legacy_interface_reward_range_error(config_path, field)); + } + + let config_dir = Path::new(config_path).parent().unwrap_or(Path::new(".")); + let document = infotheory::spec::load_spec_document(config_path).map_err(anyhow::Error::msg)?; + match document { + SpecDocument::PlannerRun(spec) => { + let compiled = spec + .compile_in(&spec::SpecEnvironment::new(config_dir)) + .map_err(anyhow::Error::msg)?; + run_compiled_planner_run(&compiled, json_overlay.as_ref()) + } + other => Err(anyhow::anyhow!( + "aixi expects a planner_run document, found kind '{}'", + other.kind_str() + )), + } +} + +#[cfg(test)] +mod tests { + #[cfg(all(feature = "backend-ctw", feature = "aixi-gameengine"))] + use super::run_compiled_planner_run; + use super::{ + AixiRunLogger, builtin_environment_name, is_canonical_spec_document, + legacy_planner_config_error, run_aixi_mode, + }; + use infotheory::aixi::planner_agent::PlannerActionProvenance; + use infotheory::spec::BuiltinEnvironmentSpec; + use serde_json::json; + use std::path::PathBuf; + use std::sync::atomic::{AtomicU64, Ordering}; + use std::time::{SystemTime, UNIX_EPOCH}; + + static TEMP_TEST_PATH_COUNTER: AtomicU64 = AtomicU64::new(0); + + fn unique_temp_path(prefix: &str, suffix: &str) -> PathBuf { + let counter = TEMP_TEST_PATH_COUNTER.fetch_add(1, Ordering::Relaxed); + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_nanos()) + .unwrap_or(0); + std::env::temp_dir().join(format!( + "{prefix}-{}-{nanos}-{counter}{suffix}", + std::process::id() + )) + } + + #[cfg(all(feature = "backend-ctw", feature = "aixi-gameengine"))] + fn compile_planner_run_value( + value: &serde_json::Value, + ) -> infotheory::spec::CompiledPlannerRunSpec { + let doc = + infotheory::spec::SpecDocument::parse_json_value(value, std::path::Path::new(".")) + .expect("parse planner_run JSON"); + let infotheory::spec::SpecDocument::PlannerRun(spec) = doc else { + panic!("expected planner_run document"); + }; + spec.compile().expect("compile planner_run") + } + + #[test] + fn canonical_spec_detection_and_legacy_error_messages_are_stable() { + assert!(is_canonical_spec_document(&json!({ + "schema_version": 1, + "kind": "planner_run" + }))); + assert!(!is_canonical_spec_document(&json!({ + "schema_version": 1 + }))); + assert_eq!( + builtin_environment_name(BuiltinEnvironmentSpec::CoinFlip), + "coin_flip" + ); + + let err = legacy_planner_config_error("/tmp/legacy.json"); + let msg = err.to_string(); + assert!(msg.contains("legacy aixi JSON configs are no longer executable")); + assert!(msg.contains("/tmp/legacy.json")); + assert!(msg.contains("planner_run")); + } + + #[test] + fn aixi_run_logger_handles_disabled_and_single_sink_modes() { + assert!( + AixiRunLogger::new(None) + .expect("logger creation should succeed") + .is_none() + ); + + let bits_path = unique_temp_path("aixi-trace-only-bits", ".bin"); + let jsonl_path = unique_temp_path("aixi-trace-only-jsonl", ".jsonl"); + + let bits_overlay = json!({ + "trace_bits01_path": bits_path, + }); + let mut bits_logger = AixiRunLogger::new(Some(&bits_overlay)) + .expect("bits logger") + .expect("bits logger should be enabled"); + bits_logger + .log_action(0, 1, 1, PlannerActionProvenance::Greedy) + .expect("log action to bits"); + bits_logger.next_step().expect("advance bits step"); + drop(bits_logger); + let bits = std::fs::read(&bits_path).expect("read bits trace"); + assert!(!bits.is_empty()); + + let jsonl_overlay = json!({ + "trace_jsonl_path": jsonl_path, + "trace_flush_every": 2 + }); + let mut jsonl_logger = AixiRunLogger::new(Some(&jsonl_overlay)) + .expect("jsonl logger") + .expect("jsonl logger should be enabled"); + jsonl_logger + .log_percept(0, &[3], 1, 2, 4, 1) + .expect("log percept to jsonl"); + jsonl_logger.next_step().expect("advance jsonl step"); + drop(jsonl_logger); + let jsonl = std::fs::read_to_string(&jsonl_path).expect("read jsonl trace"); + assert!(jsonl.contains("\"kind\":\"percept\"")); + + let _ = std::fs::remove_file(bits_path); + let _ = std::fs::remove_file(jsonl_path); + } + + #[test] + fn aixi_run_logger_writes_bits_and_jsonl_records() { + let bits_path = unique_temp_path("aixi-trace-bits", ".bin"); + let jsonl_path = unique_temp_path("aixi-trace-jsonl", ".jsonl"); + let overlay = json!({ + "trace_bits01_path": bits_path, + "trace_jsonl_path": jsonl_path, + "trace_flush_every": 1 + }); + + let mut logger = AixiRunLogger::new(Some(&overlay)) + .expect("logger setup") + .expect("logger should be enabled"); + logger + .log_action(0, 1, 2, PlannerActionProvenance::Exploratory) + .expect("log action"); + logger + .log_percept(0, &[2], 1, 2, 4, 0) + .expect("log percept"); + logger.next_step().expect("advance step"); + drop(logger); + + let bits = std::fs::read(&bits_path).expect("read bits trace"); + assert!(!bits.is_empty()); + assert!(bits.iter().all(|byte| *byte == 0 || *byte == 1)); + + let jsonl = std::fs::read_to_string(&jsonl_path).expect("read jsonl trace"); + assert!(jsonl.contains("\"kind\":\"action\"")); + assert!(jsonl.contains("\"provenance\":\"exploratory\"")); + assert!(jsonl.contains("\"kind\":\"percept\"")); + + let _ = std::fs::remove_file(bits_path); + let _ = std::fs::remove_file(jsonl_path); + } + + #[cfg(all(feature = "backend-ctw", feature = "aixi-gameengine"))] + #[test] + fn mc_aixi_jsonl_trace_can_be_converted_to_warmstart_teacher_trace() { + use infotheory::aixi::warmstart::{ + standalone_warmstart_teacher_contract_for_compiled_planner_run, + warmstart_teacher_trace_from_jsonl_path, + }; + + let bits_path = unique_temp_path("mc-aixi-warmstart-trace", ".bits01"); + let jsonl_path = unique_temp_path("mc-aixi-warmstart-trace", ".jsonl"); + let teacher_path = unique_temp_path("mc-aixi-warmstart-teacher", ".json"); + let mc_aixi = compile_planner_run_value(&json!({ + "schema_version": 1, + "kind": "planner_run", + "assets": [], + "environment": { + "kind": "builtin", + "name": "coin_flip" + }, + "interface": { + "observation_bits": 1, + "observation_stream_len": 1, + "observation_key_mode": "full_stream", + "reward_bits": 1, + "agent_actions": 2 + }, + "controller": { + "kind": "mc_aixi", + "predictor": { + "kind": "ctw", + "depth": 4 + }, + "bit_stream_semantics": { "kind": "binary_tokens" }, + "agent_horizon": 1, + "num_simulations": 1, + "mcts_strategy": { + "kind": "rho_uct" + }, + "exploration_exploitation_ratio": 1.0, + "discount_gamma": 1.0 + }, + "runtime": { + "random_seed": 7, + "learn_cycles": 2, + "eval_cycles": 0, + "terminate_lifetime": 2, + "log_every": 1, + "perf": false, + "vm_perf_only": false, + "explore_epsilon": 0.0, + "explore_gamma": 1.0 + } + })); + let overlay = json!({ + "trace_bits01_path": bits_path, + "trace_jsonl_path": jsonl_path, + "trace_flush_every": 1 + }); + run_compiled_planner_run(&mc_aixi, Some(&overlay)) + .expect("MC-AIXI planner run should produce JSONL"); + + let warmstart_target = compile_planner_run_value(&json!({ + "schema_version": 1, + "kind": "planner_run", + "assets": [{ + "id": "teacher", + "path": teacher_path.to_string_lossy() + }], + "environment": { + "kind": "builtin", + "name": "coin_flip" + }, + "interface": { + "observation_bits": 1, + "observation_stream_len": 1, + "observation_key_mode": "full_stream", + "reward_bits": 1, + "agent_actions": 2 + }, + "controller": { + "kind": "aiqi_warmstart_exact_jh", + "predictor": { + "kind": "ctw", + "depth": 4 + }, + "return_horizon": 2, + "return_bins": 3, + "label_phase_period": 2, + "teacher_dataset_asset": "teacher", + "planner_simulations_per_step": 1 + }, + "runtime": { + "random_seed": 7, + "learn_cycles": 1, + "eval_cycles": 0, + "terminate_lifetime": 1, + "log_every": 1, + "perf": false, + "vm_perf_only": false, + "explore_epsilon": 0.0, + "explore_gamma": 1.0 + } + })); + let contract = + standalone_warmstart_teacher_contract_for_compiled_planner_run(&warmstart_target) + .expect("build standalone teacher contract"); + let trace = warmstart_teacher_trace_from_jsonl_path(&jsonl_path, &contract, 2) + .expect("MC-AIXI JSONL should convert to a warm-start teacher trace"); + assert_eq!(trace.transitions.len(), 2); + + let jsonl = std::fs::read_to_string(&jsonl_path).expect("read JSONL trace"); + let records = jsonl + .lines() + .map(serde_json::from_str::) + .collect::, _>>() + .expect("parse JSONL records"); + assert_eq!(records.len(), 5); + let final_record = records.last().expect("terminal percept record"); + assert_eq!(final_record["kind"].as_str(), Some("percept")); + assert_eq!(final_record["t"].as_u64(), Some(2)); + + let bits = std::fs::read(&bits_path).expect("read bits01 trace"); + assert_eq!( + bits.len(), + 8, + "two MC-AIXI cycles should record p0,a0,p1,a1,p2 with 1-bit actions and 2-bit percepts" + ); + + let _ = std::fs::remove_file(bits_path); + let _ = std::fs::remove_file(jsonl_path); + let _ = std::fs::remove_file(teacher_path); + } + + #[cfg(all(feature = "backend-ctw", feature = "tuner"))] + #[test] + fn run_aixi_mode_rejects_non_planner_spec_documents() { + use infotheory::api::CanonicalJson; + let path = unique_temp_path("infotheory-spec-kind", ".json"); + let doc_value = json!({ + "schema_version": 1, + "kind": "tune", + "assets": [ + { + "id": "dataset", + "path": "input.bin" + } + ], + "input_asset": "dataset", + "baseline_candidate": { + "kind": "rate-ac", + "rate_backend": { + "kind": "ctw", + "depth": 8 + }, + "framing": "framed" + }, + "controller": { + "kind": "annealed_hill_climbing", + "max_mutation_radius": 1 + }, + "bounds": { + "allowed_backends": ["ctw"], + "forbidden_backends": [], + "parameter_ranges": [], + "max_experts": 2, + "max_mixture_nesting_depth": 1, + "min_experts": 1, + "allow_duplicate_experts": false, + "required_experts": [], + "forbidden_expert_pairs": [] + }, + "eval_time_limit_seconds": 1.0, + "time_budget_seconds": 2.0, + "min_throughput_bytes_per_second": 1.0, + "max_memory_bytes": 1024, + "output_config_path": "best.json", + "seed": 7, + "report_path": null + }); + let doc = + infotheory::spec::SpecDocument::parse_json_value(&doc_value, std::path::Path::new(".")) + .expect("canonical tune document"); + std::fs::write(&path, doc.to_canonical_json().expect("canonical json")) + .expect("write temp spec"); + + let err = run_aixi_mode(path.to_str().expect("utf8 path")) + .expect_err("non planner spec should be rejected"); + assert!(err.to_string().contains("planner_run document")); + + let _ = std::fs::remove_file(path); + } + + #[cfg(all(not(feature = "backend-ctw"), feature = "tuner"))] + #[test] + fn run_aixi_mode_surfaces_backend_validation_for_non_planner_documents() { + let path = unique_temp_path("infotheory-canonical-non-planner-no-ctw", ".json"); + let doc_value = json!({ + "schema_version": 1, + "kind": "tune", + "assets": [ + { + "id": "dataset", + "path": "input.bin" + } + ], + "input_asset": "dataset", + "baseline_candidate": { + "kind": "rate-ac", + "rate_backend": { + "kind": "ctw", + "depth": 8 + }, + "framing": "framed" + }, + "controller": { + "kind": "annealed_hill_climbing", + "max_mutation_radius": 1 + }, + "bounds": { + "allowed_backends": ["ctw"], + "forbidden_backends": [], + "parameter_ranges": [], + "max_experts": 2, + "max_mixture_nesting_depth": 1, + "min_experts": 1, + "allow_duplicate_experts": false, + "required_experts": [], + "forbidden_expert_pairs": [] + }, + "eval_time_limit_seconds": 1.0, + "time_budget_seconds": 2.0, + "min_throughput_bytes_per_second": 1.0, + "max_memory_bytes": 1024, + "output_config_path": "best.json", + "seed": 7, + "report_path": null + }); + std::fs::write( + &path, + serde_json::to_vec(&doc_value).expect("serialize canonical json"), + ) + .expect("write temp spec"); + + let err = run_aixi_mode(path.to_str().expect("utf8 path")) + .expect_err("missing backend feature should be surfaced"); + assert!( + err.to_string() + .contains("requires infotheory feature 'backend-ctw'"), + "{err}" + ); + + let _ = std::fs::remove_file(path); + } + + #[cfg(all(feature = "backend-ctw", feature = "aixi-gameengine"))] + #[test] + fn run_aixi_mode_accepts_canonical_planner_run_documents() { + use infotheory::api::CanonicalJson; + let path = unique_temp_path("infotheory-planner-run", ".json"); + let doc_value = json!({ + "schema_version": 1, + "kind": "planner_run", + "assets": [], + "environment": { + "kind": "builtin", + "name": "coin_flip" + }, + "interface": { + "observation_bits": 1, + "observation_stream_len": 1, + "observation_key_mode": "full_stream", + "reward_bits": 1, + "agent_actions": 2 + }, + "controller": { + "kind": "mc_aixi", + "predictor": { + "kind": "ctw", + "depth": 8 + }, + "bit_stream_semantics": { "kind": "binary_tokens" }, + "agent_horizon": 1, + "num_simulations": 1, + "mcts_strategy": { + "kind": "rho_uct" + }, + "exploration_exploitation_ratio": 1.0, + "discount_gamma": 1.0 + }, + "runtime": { + "random_seed": 7, + "learn_cycles": 1, + "eval_cycles": 0, + "terminate_lifetime": 1, + "log_every": 1, + "perf": false, + "vm_perf_only": false, + "explore_epsilon": 0.0, + "explore_gamma": 1.0 + } + }); + let doc = + infotheory::spec::SpecDocument::parse_json_value(&doc_value, std::path::Path::new(".")) + .expect("canonical planner document"); + std::fs::write(&path, doc.to_canonical_json().expect("canonical json")) + .expect("write temp planner spec"); + + run_aixi_mode(path.to_str().expect("utf8 path")) + .expect("canonical planner_run document should execute"); + + let _ = std::fs::remove_file(path); + } + + #[cfg(all(feature = "backend-ctw", feature = "aixi-gameengine"))] + #[test] + fn run_aixi_mode_rejects_legacy_interface_reward_range_fields() { + let path = unique_temp_path("infotheory-planner-run-invalid-reward", ".json"); + let legacy_doc = json!({ + "schema_version": 1, + "kind": "planner_run", + "assets": [], + "environment": { + "kind": "builtin", + "name": "coin_flip" + }, + "interface": { + "observation_bits": 1, + "observation_stream_len": 1, + "observation_key_mode": "full_stream", + "reward_bits": 1, + "agent_actions": 2, + "min_reward": 0, + "max_reward": 100, + "reward_offset": 0 + }, + "controller": { + "kind": "mc_aixi", + "predictor": { + "kind": "ctw", + "depth": 8 + }, + "bit_stream_semantics": { "kind": "binary_tokens" }, + "agent_horizon": 1, + "num_simulations": 1, + "mcts_strategy": { + "kind": "rho_uct" + }, + "exploration_exploitation_ratio": 1.0, + "discount_gamma": 1.0 + }, + "runtime": { + "random_seed": 7, + "learn_cycles": 1, + "eval_cycles": 0, + "terminate_lifetime": 1, + "log_every": 1, + "perf": false, + "vm_perf_only": false, + "explore_epsilon": 0.0, + "explore_gamma": 1.0 + } + }); + std::fs::write( + &path, + serde_json::to_vec_pretty(&legacy_doc).expect("legacy planner json"), + ) + .expect("write temp planner spec"); + + let err = run_aixi_mode(path.to_str().expect("utf8 path")) + .expect_err("legacy interface reward range fields should be rejected"); + let message = err.to_string(); + assert!( + message.contains("unknown interface field 'min_reward'") + || message.contains("unknown interface field 'max_reward'"), + "{message}" + ); + + let _ = std::fs::remove_file(path); + } + + #[test] + fn run_aixi_mode_rejects_legacy_planner_json_documents() { + let path = unique_temp_path("infotheory-legacy-planner", ".json"); + let legacy = serde_json::json!({ + "environment": "coin-flip", + "planner": "mc-aixi", + "algorithm": "ctw", + "ct_depth": 8, + "agent_horizon": 1, + "observation_bits": 1, + "observation_stream_len": 1, + "observation_key_mode": "full_stream", + "reward_bits": 1, + "agent_actions": 2, + "num_simulations": 1, + "discount_gamma": 1.0, + }); + std::fs::write( + &path, + serde_json::to_vec_pretty(&legacy).expect("legacy planner json"), + ) + .expect("write temp legacy planner config"); + + let err = run_aixi_mode(path.to_str().expect("utf8 path")) + .expect_err("legacy planner json should be rejected"); + let message = err.to_string(); + assert!(message.contains("legacy aixi JSON configs are no longer executable")); + assert!(message.contains("planner_run")); + assert!(message.contains("schema_version")); + assert!(message.contains("kind")); + + let _ = std::fs::remove_file(path); + } +} diff --git a/crates/infotheory/src/cli/warmstart.rs b/crates/infotheory/src/cli/warmstart.rs new file mode 100644 index 00000000..3eb60903 --- /dev/null +++ b/crates/infotheory/src/cli/warmstart.rs @@ -0,0 +1,607 @@ +use infotheory::aixi::planner_agent::{ + NullPlannerObserver, PlannerControllerAgent, PlannerRunSession, build_planner_environment, + compile_planner_run_document, +}; +use infotheory::aixi::warmstart::{ + WarmStartExactJhTeacherDataset, WarmStartExactJhTeacherTrace, WarmStartExactJhTransition, + merge_warmstart_teacher_traces_deterministic, read_warmstart_teacher_dataset_path, + standalone_warmstart_teacher_contract_for_compiled_planner_run, + validate_warmstart_teacher_dataset_for_compiled_planner_run, warmstart_target_return_horizon, + warmstart_teacher_trace_from_jsonl_path, write_warmstart_teacher_dataset_path, +}; +use infotheory::spec::{CanonicalJson, CompiledPlannerRunSpec}; + +/// Request for exporting a warm-start teacher dataset from a planner run. +pub(crate) struct WarmStartTeacherPlannerRunRequest { + /// Target warm-start planner-run path. + pub target_path: String, + /// Teacher planner-run path. + pub teacher_path: String, + /// Output teacher dataset path. + pub out_path: String, +} + +/// Request for converting planner JSONL to a warm-start teacher dataset. +pub(crate) struct WarmStartTeacherJsonlRequest { + /// Target warm-start planner-run path. + pub target_path: String, + /// Input JSONL path. + pub jsonl_path: String, + /// Output teacher dataset path. + pub out_path: String, +} + +/// Request for merging warm-start teacher datasets. +pub(crate) struct WarmStartTeacherMergeRequest { + /// Target warm-start planner-run path. + pub target_path: String, + /// Input teacher dataset paths. + pub teacher_paths: Vec, + /// Output teacher dataset path. + pub out_path: String, +} + +pub(crate) enum WarmStartCommand { + PlannerRun(WarmStartTeacherPlannerRunRequest), + FromJsonl(WarmStartTeacherJsonlRequest), + Merge(WarmStartTeacherMergeRequest), +} + +fn option_value( + args: &[String], + index: &mut usize, + option: &str, + expected: &str, +) -> anyhow::Result { + *index += 1; + args.get(*index) + .cloned() + .ok_or_else(|| anyhow::anyhow!("{option} requires {expected}")) +} + +fn set_once(slot: &mut Option, value: String, option: &str) -> anyhow::Result<()> { + if slot.replace(value).is_some() { + return Err(anyhow::anyhow!("duplicate {option}")); + } + Ok(()) +} + +fn parse_teacher_planner_run(args: &[String]) -> anyhow::Result { + let mut target_path = None; + let mut teacher_path = None; + let mut out_path = None; + let mut i = 4usize; + while i < args.len() { + match args[i].as_str() { + "--target" => set_once( + &mut target_path, + option_value(args, &mut i, "--target", "a warmstart planner_run path")?, + "--target", + )?, + "--teacher" => set_once( + &mut teacher_path, + option_value(args, &mut i, "--teacher", "a teacher planner_run path")?, + "--teacher", + )?, + "--out" => set_once( + &mut out_path, + option_value(args, &mut i, "--out", "an output teacher JSON path")?, + "--out", + )?, + other => { + return Err(anyhow::anyhow!( + "unknown warmstart teacher planner-run option '{other}'" + )); + } + } + i += 1; + } + Ok(WarmStartTeacherPlannerRunRequest { + target_path: target_path.ok_or_else(|| anyhow::anyhow!("missing required --target"))?, + teacher_path: teacher_path.ok_or_else(|| anyhow::anyhow!("missing required --teacher"))?, + out_path: out_path.ok_or_else(|| anyhow::anyhow!("missing required --out"))?, + }) +} + +fn parse_teacher_from_jsonl(args: &[String]) -> anyhow::Result { + let mut target_path = None; + let mut jsonl_path = None; + let mut out_path = None; + let mut i = 4usize; + while i < args.len() { + match args[i].as_str() { + "--target" => set_once( + &mut target_path, + option_value(args, &mut i, "--target", "a warmstart planner_run path")?, + "--target", + )?, + "--jsonl" => set_once( + &mut jsonl_path, + option_value(args, &mut i, "--jsonl", "a JSONL trace path")?, + "--jsonl", + )?, + "--out" => set_once( + &mut out_path, + option_value(args, &mut i, "--out", "an output teacher JSON path")?, + "--out", + )?, + other => { + return Err(anyhow::anyhow!( + "unknown warmstart teacher from-jsonl option '{other}'" + )); + } + } + i += 1; + } + Ok(WarmStartTeacherJsonlRequest { + target_path: target_path.ok_or_else(|| anyhow::anyhow!("missing required --target"))?, + jsonl_path: jsonl_path.ok_or_else(|| anyhow::anyhow!("missing required --jsonl"))?, + out_path: out_path.ok_or_else(|| anyhow::anyhow!("missing required --out"))?, + }) +} + +fn parse_teacher_merge(args: &[String]) -> anyhow::Result { + let mut target_path = None; + let mut teacher_paths = Vec::new(); + let mut out_path = None; + let mut i = 4usize; + while i < args.len() { + match args[i].as_str() { + "--target" => set_once( + &mut target_path, + option_value(args, &mut i, "--target", "a warmstart planner_run path")?, + "--target", + )?, + "--teacher" => teacher_paths.push(option_value( + args, + &mut i, + "--teacher", + "a teacher JSON path", + )?), + "--out" => set_once( + &mut out_path, + option_value(args, &mut i, "--out", "an output teacher JSON path")?, + "--out", + )?, + other => { + return Err(anyhow::anyhow!( + "unknown warmstart teacher merge option '{other}'" + )); + } + } + i += 1; + } + if teacher_paths.is_empty() { + return Err(anyhow::anyhow!("missing required --teacher")); + } + Ok(WarmStartTeacherMergeRequest { + target_path: target_path.ok_or_else(|| anyhow::anyhow!("missing required --target"))?, + teacher_paths, + out_path: out_path.ok_or_else(|| anyhow::anyhow!("missing required --out"))?, + }) +} + +pub(crate) fn parse_warmstart_command(args: &[String]) -> anyhow::Result { + match ( + args.get(1).map(String::as_str), + args.get(2).map(String::as_str), + args.get(3).map(String::as_str), + ) { + (Some("warmstart"), Some("teacher"), Some("planner-run")) => Ok( + WarmStartCommand::PlannerRun(parse_teacher_planner_run(args)?), + ), + (Some("warmstart"), Some("teacher"), Some("from-jsonl")) => { + Ok(WarmStartCommand::FromJsonl(parse_teacher_from_jsonl(args)?)) + } + (Some("warmstart"), Some("teacher"), Some("merge")) => { + Ok(WarmStartCommand::Merge(parse_teacher_merge(args)?)) + } + _ => Err(anyhow::anyhow!( + "usage: infotheory warmstart teacher ..." + )), + } +} + +fn compile_target(path: &str) -> anyhow::Result { + compile_planner_run_document(path, "warmstart teacher --target") +} + +fn ensure_teacher_interface_matches_target( + target: &CompiledPlannerRunSpec, + teacher: &CompiledPlannerRunSpec, +) -> anyhow::Result<()> { + if target.interface().agent_actions != teacher.interface().agent_actions + || target.interface().observation_bits != teacher.interface().observation_bits + || target.interface().observation_stream_len != teacher.interface().observation_stream_len + || target.interface().reward_bits != teacher.interface().reward_bits + || target.interface().observation_key_mode != teacher.interface().observation_key_mode + { + return Err(anyhow::anyhow!( + "teacher planner interface does not match target warmstart planner interface" + )); + } + Ok(()) +} + +fn compiled_environment_value( + compiled: &CompiledPlannerRunSpec, +) -> anyhow::Result { + let value = compiled + .canonical_spec() + .to_canonical_json_value() + .map_err(anyhow::Error::msg)?; + value + .get("environment") + .cloned() + .ok_or_else(|| anyhow::anyhow!("compiled planner canonical JSON is missing environment")) +} + +fn validate_warmstart_teacher_export_compatibility( + target: &CompiledPlannerRunSpec, + teacher: &CompiledPlannerRunSpec, +) -> anyhow::Result<()> { + ensure_teacher_interface_matches_target(target, teacher)?; + if target.runtime().vm_perf_only || teacher.runtime().vm_perf_only { + return Err(anyhow::anyhow!( + "warmstart teacher planner-run export does not support vm_perf_only planner specs" + )); + } + let target_environment = compiled_environment_value(target)?; + let teacher_environment = compiled_environment_value(teacher)?; + if target_environment != teacher_environment { + return Err(anyhow::anyhow!( + "teacher planner environment does not match target warmstart planner environment" + )); + } + Ok(()) +} + +pub(crate) fn run_warmstart_teacher_planner_run_export( + request: &WarmStartTeacherPlannerRunRequest, +) -> anyhow::Result<()> { + let target = compile_target(&request.target_path)?; + let teacher = compile_planner_run_document( + &request.teacher_path, + "warmstart teacher planner-run --teacher", + )?; + validate_warmstart_teacher_export_compatibility(&target, &teacher)?; + let contract = standalone_warmstart_teacher_contract_for_compiled_planner_run(&target) + .map_err(|err| anyhow::anyhow!("{err}"))?; + let (env, _) = build_planner_environment(&teacher)?; + let agent = PlannerControllerAgent::from_compiled(&teacher) + .map_err(|err| anyhow::anyhow!("failed to construct teacher planner: {err}"))?; + let mut session = PlannerRunSession::new(&teacher, agent, env) + .map_err(|err| anyhow::anyhow!("failed to initialize teacher planner session: {err}"))?; + let mut observer = NullPlannerObserver; + let mut transitions = Vec::new(); + while let Some(outcome) = session + .run_next_cycle(&mut observer) + .map_err(|err| anyhow::anyhow!("{err}"))? + { + transitions.push(WarmStartExactJhTransition::new( + outcome.action, + outcome.observations, + outcome.reward, + )); + } + let trace = WarmStartExactJhTeacherTrace::new(transitions); + let dataset = WarmStartExactJhTeacherDataset::new(contract, vec![trace]); + validate_warmstart_teacher_dataset_for_compiled_planner_run(&target, &dataset) + .map_err(|err| anyhow::anyhow!("{err}"))?; + write_warmstart_teacher_dataset_path(&request.out_path, &dataset) + .map_err(|err| anyhow::anyhow!("{err}"))?; + println!( + "Warm-start teacher dataset written to {} ({} trace).", + request.out_path, + dataset.traces.len() + ); + Ok(()) +} + +pub(crate) fn run_warmstart_teacher_from_jsonl_export( + request: &WarmStartTeacherJsonlRequest, +) -> anyhow::Result<()> { + let target = compile_target(&request.target_path)?; + let contract = standalone_warmstart_teacher_contract_for_compiled_planner_run(&target) + .map_err(|err| anyhow::anyhow!("{err}"))?; + let return_horizon = + warmstart_target_return_horizon(&target).map_err(|err| anyhow::anyhow!("{err}"))?; + let trace = + warmstart_teacher_trace_from_jsonl_path(&request.jsonl_path, &contract, return_horizon) + .map_err(|err| anyhow::anyhow!("{err}"))?; + let dataset = WarmStartExactJhTeacherDataset::new(contract, vec![trace]); + validate_warmstart_teacher_dataset_for_compiled_planner_run(&target, &dataset) + .map_err(|err| anyhow::anyhow!("{err}"))?; + write_warmstart_teacher_dataset_path(&request.out_path, &dataset) + .map_err(|err| anyhow::anyhow!("{err}"))?; + println!( + "Warm-start teacher dataset written to {} from {}.", + request.out_path, request.jsonl_path + ); + Ok(()) +} + +pub(crate) fn run_warmstart_teacher_merge( + request: &WarmStartTeacherMergeRequest, +) -> anyhow::Result<()> { + let target = compile_target(&request.target_path)?; + let contract = standalone_warmstart_teacher_contract_for_compiled_planner_run(&target) + .map_err(|err| anyhow::anyhow!("{err}"))?; + let mut merged = WarmStartExactJhTeacherDataset::new(contract, Vec::new()); + let mut expected_contract = None; + let mut inserted_total = 0usize; + for path in &request.teacher_paths { + let dataset = + read_warmstart_teacher_dataset_path(path).map_err(|err| anyhow::anyhow!("{err}"))?; + validate_warmstart_teacher_dataset_for_compiled_planner_run(&target, &dataset) + .map_err(|err| anyhow::anyhow!("{err}"))?; + if let Some(contract) = &expected_contract { + if contract != &dataset.contract { + return Err(anyhow::anyhow!( + "warmstart teacher merge input '{}' has a contract different from earlier inputs", + path + )); + } + } else { + expected_contract = Some(dataset.contract.clone()); + } + let (inserted, _) = + merge_warmstart_teacher_traces_deterministic(&mut merged.traces, dataset.traces); + inserted_total = inserted_total.saturating_add(inserted); + } + if merged.traces.is_empty() { + return Err(anyhow::anyhow!( + "merged teacher dataset would contain no traces" + )); + } + validate_warmstart_teacher_dataset_for_compiled_planner_run(&target, &merged) + .map_err(|err| anyhow::anyhow!("{err}"))?; + write_warmstart_teacher_dataset_path(&request.out_path, &merged) + .map_err(|err| anyhow::anyhow!("{err}"))?; + println!( + "Warm-start teacher dataset written to {} ({} traces, {} inserted).", + request.out_path, + merged.traces.len(), + inserted_total + ); + Ok(()) +} + +pub(crate) fn run_warmstart_command(command: WarmStartCommand) -> anyhow::Result<()> { + match command { + WarmStartCommand::PlannerRun(request) => run_warmstart_teacher_planner_run_export(&request), + WarmStartCommand::FromJsonl(request) => run_warmstart_teacher_from_jsonl_export(&request), + WarmStartCommand::Merge(request) => run_warmstart_teacher_merge(&request), + } +} + +#[cfg(all(test, feature = "backend-ctw"))] +mod tests { + use super::*; + use infotheory::aixi::planner_agent::PlannerActionProvenance; + use infotheory::aixi::warmstart::{ + WarmStartExactJhTeacherTrace, WarmStartExactJhTransition, warmstart_jsonl_action_record, + warmstart_jsonl_percept_record, + }; + use infotheory::aixi::warmstart_contract::{ + TaskFingerprint, warmstart_exact_jh_planner_task_fingerprint, + }; + use infotheory::spec::SpecDocument; + use serde_json::json; + use std::path::{Path, PathBuf}; + use std::sync::atomic::{AtomicU64, Ordering}; + use std::time::{SystemTime, UNIX_EPOCH}; + + static TEMP_TEST_PATH_COUNTER: AtomicU64 = AtomicU64::new(0); + + fn unique_temp_path(prefix: &str, suffix: &str) -> PathBuf { + let counter = TEMP_TEST_PATH_COUNTER.fetch_add(1, Ordering::Relaxed); + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_nanos()) + .unwrap_or(0); + std::env::temp_dir().join(format!( + "{prefix}-{}-{nanos}-{counter}{suffix}", + std::process::id() + )) + } + + fn planner_run_value( + environment_name: &str, + teacher_path: &Path, + return_horizon: usize, + vm_perf_only: bool, + ) -> serde_json::Value { + json!({ + "schema_version": 1, + "kind": "planner_run", + "assets": [{ + "id": "teacher", + "path": teacher_path.to_string_lossy() + }], + "environment": { + "kind": "builtin", + "name": environment_name + }, + "interface": { + "observation_bits": 2, + "observation_stream_len": 1, + "observation_key_mode": "full_stream", + "reward_bits": 2, + "agent_actions": 2 + }, + "controller": { + "kind": "aiqi_warmstart_exact_jh", + "predictor": { + "kind": "ctw", + "depth": 4 + }, + "return_horizon": return_horizon, + "return_bins": return_horizon + 1, + "label_phase_period": return_horizon, + "teacher_dataset_asset": "teacher", + "planner_simulations_per_step": 1 + }, + "runtime": { + "random_seed": 7, + "learn_cycles": 1, + "eval_cycles": 0, + "terminate_lifetime": 1, + "log_every": 1, + "perf": false, + "vm_perf_only": vm_perf_only, + "explore_epsilon": 0.0, + "explore_gamma": 1.0 + } + }) + } + + fn write_planner_run( + path: &Path, + environment_name: &str, + teacher_path: &Path, + return_horizon: usize, + vm_perf_only: bool, + ) { + std::fs::write( + path, + serde_json::to_vec(&planner_run_value( + environment_name, + teacher_path, + return_horizon, + vm_perf_only, + )) + .expect("serialize planner_run"), + ) + .expect("write planner_run"); + } + + fn compile_planner_value( + environment_name: &str, + teacher_path: &Path, + return_horizon: usize, + vm_perf_only: bool, + ) -> CompiledPlannerRunSpec { + let document = SpecDocument::parse_json_value( + &planner_run_value(environment_name, teacher_path, return_horizon, vm_perf_only), + Path::new("."), + ) + .expect("parse planner_run"); + let SpecDocument::PlannerRun(spec) = document else { + panic!("expected planner_run document"); + }; + spec.compile().expect("compile planner_run") + } + + fn write_teacher_dataset_for_target( + path: &Path, + compiled: &CompiledPlannerRunSpec, + task_fingerprint_override: Option<&str>, + ) { + let mut contract = standalone_warmstart_teacher_contract_for_compiled_planner_run(compiled) + .expect("standalone contract"); + if let Some(task_fingerprint) = task_fingerprint_override { + contract.task_fingerprint = TaskFingerprint::parse_hex(task_fingerprint) + .expect("test task fingerprint override must be canonical hex"); + } + let dataset = WarmStartExactJhTeacherDataset::new( + contract, + vec![WarmStartExactJhTeacherTrace::new(vec![ + WarmStartExactJhTransition::new(0, vec![1], 1), + ])], + ); + write_warmstart_teacher_dataset_path(path, &dataset).expect("write teacher dataset"); + } + + #[test] + fn planner_run_export_compatibility_rejects_environment_and_vm_perf_mismatch() { + let teacher_path = unique_temp_path("warmstart-cli-teacher", ".json"); + let target = compile_planner_value("coin_flip", &teacher_path, 1, false); + let other_environment = + compile_planner_value("biased_rock_paper_scissor", &teacher_path, 1, false); + let err = validate_warmstart_teacher_export_compatibility(&target, &other_environment) + .expect_err("environment mismatch must fail before export"); + assert!( + err.to_string().contains("environment does not match"), + "{err}" + ); + + let vm_perf_only = compile_planner_value("coin_flip", &teacher_path, 1, true); + let err = validate_warmstart_teacher_export_compatibility(&target, &vm_perf_only) + .expect_err("vm_perf_only teacher must fail before export"); + assert!(err.to_string().contains("vm_perf_only"), "{err}"); + } + + #[test] + fn from_jsonl_export_rejects_short_trace_before_writing() { + let target_path = unique_temp_path("warmstart-cli-target", ".json"); + let teacher_asset_path = unique_temp_path("warmstart-cli-target-teacher", ".json"); + let jsonl_path = unique_temp_path("warmstart-cli-short", ".jsonl"); + let out_path = unique_temp_path("warmstart-cli-short-out", ".json"); + write_planner_run(&target_path, "coin_flip", &teacher_asset_path, 2, false); + let jsonl = [ + warmstart_jsonl_action_record(0, 0, PlannerActionProvenance::Greedy).to_string(), + warmstart_jsonl_percept_record(0, &[1], 1).to_string(), + ] + .join("\n"); + std::fs::write(&jsonl_path, jsonl).expect("write jsonl"); + + let err = run_warmstart_teacher_from_jsonl_export(&WarmStartTeacherJsonlRequest { + target_path: target_path.to_string_lossy().into_owned(), + jsonl_path: jsonl_path.to_string_lossy().into_owned(), + out_path: out_path.to_string_lossy().into_owned(), + }) + .expect_err("short JSONL trace must fail before write"); + assert!(err.to_string().contains("return_horizon is 2"), "{err}"); + assert!( + !out_path.exists(), + "failed export must not leave a teacher output file" + ); + + let _ = std::fs::remove_file(target_path); + let _ = std::fs::remove_file(jsonl_path); + } + + #[test] + fn merge_rejects_contract_mismatch_before_writing() { + let target_path = unique_temp_path("warmstart-cli-merge-target", ".json"); + let target_teacher_asset = unique_temp_path("warmstart-cli-merge-target-teacher", ".json"); + let teacher_a_path = unique_temp_path("warmstart-cli-merge-a", ".json"); + let teacher_b_path = unique_temp_path("warmstart-cli-merge-b", ".json"); + let out_path = unique_temp_path("warmstart-cli-merge-out", ".json"); + write_planner_run(&target_path, "coin_flip", &target_teacher_asset, 1, false); + let target = compile_target(target_path.to_str().expect("utf-8 target path")) + .expect("compile target"); + let task_fingerprint = + warmstart_exact_jh_planner_task_fingerprint(&target).expect("task fingerprint"); + write_teacher_dataset_for_target(&teacher_a_path, &target, None); + write_teacher_dataset_for_target( + &teacher_b_path, + &target, + Some("deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef"), + ); + + let err = run_warmstart_teacher_merge(&WarmStartTeacherMergeRequest { + target_path: target_path.to_string_lossy().into_owned(), + teacher_paths: vec![ + teacher_a_path.to_string_lossy().into_owned(), + teacher_b_path.to_string_lossy().into_owned(), + ], + out_path: out_path.to_string_lossy().into_owned(), + }) + .expect_err("mismatched merge input contract must fail before write"); + assert!(err.to_string().contains("task_fingerprint"), "{err}"); + assert!( + err.to_string().contains(&task_fingerprint.to_string()), + "{err}" + ); + assert!( + !out_path.exists(), + "failed merge must not leave a teacher output file" + ); + + let _ = std::fs::remove_file(target_path); + let _ = std::fs::remove_file(teacher_a_path); + let _ = std::fs::remove_file(teacher_b_path); + } +} diff --git a/src/coders/ac.rs b/crates/infotheory/src/coders/ac.rs similarity index 94% rename from src/coders/ac.rs rename to crates/infotheory/src/coders/ac.rs index f85e9ec9..93dce49e 100644 --- a/src/coders/ac.rs +++ b/crates/infotheory/src/coders/ac.rs @@ -453,6 +453,62 @@ impl<'a> ArithmeticDecoder<'a> { Some(bit) } + #[inline] + fn advance_counts(&mut self, c_lo: u64, c_hi: u64, total: u32) { + let range = (self.high - self.low + 1) as u128; + let low_u = self.low as u128; + let total_u128 = total as u128; + let new_low = low_u + (range * (c_lo as u128)) / total_u128; + let new_high = low_u + (range * (c_hi as u128)) / total_u128 - 1; + + self.low = new_low as u64; + self.high = new_high as u64; + + loop { + if self.high < self.b_to_pm1 { + // Lower half; no offset adjustment is needed. + } else if self.low >= self.b_to_pm1 { + self.low -= self.b_to_pm1; + self.high -= self.b_to_pm1; + self.code -= self.b_to_pm1; + } else if self.low >= self.b_to_pm2 && self.high < self.b_to_pm2 * 3 { + self.low -= self.b_to_pm2; + self.high -= self.b_to_pm2; + self.code -= self.b_to_pm2; + } else { + break; + } + self.low = (self.low << 1) & self.mask; + self.high = ((self.high << 1) & self.mask) | 1; + self.code = ((self.code << 1) & self.mask) | (self.get_bit().unwrap_or(1) as u64); + } + } + + /// Decode a binary symbol from the two intervals `[0, split)` and + /// `[split, total)`. + /// + /// This is equivalent to decoding with the CDF `[0, split, total]`, but it + /// avoids the generic CDF search in bitwise arithmetic-coding hot paths. + #[inline] + pub(crate) fn decode_binary_counts(&mut self, split: u32, total: u32) -> anyhow::Result { + debug_assert!(split > 0); + debug_assert!(split < total); + + let total_u = total as u64; + let range = self.high - self.low + 1; + let value = + (((self.code - self.low + 1) as u128 * (total_u as u128)) - 1) / (range as u128); + let value_u = value as u32; + + if value_u < split { + self.advance_counts(0, split as u64, total); + Ok(0) + } else { + self.advance_counts(split as u64, total as u64, total); + Ok(1) + } + } + /// Decode a symbol using integer CDF. /// /// # Arguments @@ -483,35 +539,7 @@ impl<'a> ArithmeticDecoder<'a> { let c_lo = cdf[s] as u64; let c_hi = cdf[s + 1] as u64; - // Update range - let range = (self.high - self.low + 1) as u128; - let low_u = self.low as u128; - let total_u128 = total as u128; - let new_low = low_u + (range * (c_lo as u128)) / total_u128; - let new_high = low_u + (range * (c_hi as u128)) / total_u128 - 1; - - self.low = new_low as u64; - self.high = new_high as u64; - - // Renormalize - loop { - if self.high < self.b_to_pm1 { - // nothing - } else if self.low >= self.b_to_pm1 { - self.low -= self.b_to_pm1; - self.high -= self.b_to_pm1; - self.code -= self.b_to_pm1; - } else if self.low >= self.b_to_pm2 && self.high < self.b_to_pm2 * 3 { - self.low -= self.b_to_pm2; - self.high -= self.b_to_pm2; - self.code -= self.b_to_pm2; - } else { - break; - } - self.low = (self.low << 1) & self.mask; - self.high = ((self.high << 1) & self.mask) | 1; - self.code = ((self.code << 1) & self.mask) | (self.get_bit().unwrap_or(1) as u64); - } + self.advance_counts(c_lo, c_hi, total); Ok(s) } diff --git a/src/coders/mod.rs b/crates/infotheory/src/coders/mod.rs similarity index 100% rename from src/coders/mod.rs rename to crates/infotheory/src/coders/mod.rs diff --git a/src/coders/rans.rs b/crates/infotheory/src/coders/rans.rs similarity index 86% rename from src/coders/rans.rs rename to crates/infotheory/src/coders/rans.rs index aac77d34..a725d691 100644 --- a/src/coders/rans.rs +++ b/crates/infotheory/src/coders/rans.rs @@ -662,6 +662,93 @@ mod tests { } } + #[test] + fn test_pdf_helpers_roundtrip_and_size_estimate_match() { + let pdf: Vec = vec![0.5, 0.3, 0.15, 0.05]; + let symbols: Vec = vec![0, 1, 0, 2, 1, 0, 3, 0, 1, 0, 2]; + + let mut enc = RansEncoder::new(); + for &sym in symbols.iter().rev() { + enc.encode_pdf(&pdf, sym); + } + let estimated: usize = enc.size_estimate(); + let encoded: Vec = enc.finish(); + assert_eq!(estimated, encoded.len()); + + let mut dec = RansDecoder::new(&encoded).expect("decode state"); + for &expected in &symbols { + let got: usize = dec.decode_pdf(&pdf).expect("decode symbol"); + assert_eq!(got, expected); + } + } + + #[test] + fn test_decoder_rejects_short_input() { + let err = match RansDecoder::new(&[1, 2, 3]) { + Ok(_) => panic!("short rANS payload must fail"), + Err(err) => err, + }; + assert!(err.to_string().contains("rANS input too short")); + } + + #[test] + fn test_blocked_encoder_finish_empty_and_decoder_contract_errors() { + let empty_blocks: Vec> = BlockedRansEncoder::new().finish(); + assert!(empty_blocks.is_empty()); + + let bogus_block: [u8; 4] = [0, 0, 0, 0]; + let mismatch_err = match BlockedRansDecoder::new(vec![&bogus_block], 0) { + Ok(_) => panic!("zero symbols must require zero blocks"), + Err(err) => err, + }; + assert!( + mismatch_err + .to_string() + .contains("blocked rANS expected 0 blocks"), + "unexpected mismatch error: {mismatch_err}" + ); + + let cdf: Vec = quantize_pdf_to_rans_cdf(&[1.0]); + let mut empty_decoder = + BlockedRansDecoder::new(Vec::new(), 0).expect("empty stream descriptor is valid"); + let exhausted_err = empty_decoder + .decode(&cdf) + .expect_err("decoding from empty stream must fail"); + assert!( + exhausted_err + .to_string() + .contains("No more blocks to decode"), + "unexpected exhausted error: {exhausted_err}" + ); + } + + #[cfg(target_arch = "x86_64")] + #[test] + fn test_simd_roundtrip_and_constructor_guard() { + let pdf: Vec = vec![1.0]; + let cdf: Vec = quantize_pdf_to_rans_cdf(&pdf); + let symbols: Vec = vec![0usize; 64]; + + let mut enc = SimdRansEncoder::new(); + for &sym in symbols.iter().rev() { + enc.encode(&cdf_for_symbol(&cdf, sym)); + } + let encoded: Vec = enc.finish(); + + let short_len: usize = RANS_LANES * 4 - 1; + let short_err = match SimdRansDecoder::new(&encoded[..short_len]) { + Ok(_) => panic!("short SIMD rANS payload must fail"), + Err(err) => err, + }; + assert!(short_err.to_string().contains("SIMD rANS input too short")); + + let mut dec = SimdRansDecoder::new(&encoded).expect("simd decoder"); + for &expected in &symbols { + let got: usize = dec.decode(&cdf).expect("simd decode symbol"); + assert_eq!(got, expected); + } + } + #[test] fn test_blocked_rans_roundtrip_across_block_boundary() { let pdf = vec![0.5, 0.25, 0.125, 0.125]; diff --git a/src/compression/mod.rs b/crates/infotheory/src/compression/mod.rs similarity index 62% rename from src/compression/mod.rs rename to crates/infotheory/src/compression/mod.rs index b5920700..bf4416f7 100644 --- a/src/compression/mod.rs +++ b/crates/infotheory/src/compression/mod.rs @@ -4,20 +4,37 @@ //! - a predictive rate model (`RateBackend`) that emits per-symbol PDFs, //! - an entropy coder (`AC` or `rANS`), //! - optional framing metadata for robust decompression. +#![cfg_attr( + not(feature = "all-backends"), + allow(dead_code, unused_imports, unused_variables, unused_mut) +)] use anyhow::{Result, bail}; +use crate::api::{MixtureKind, MixtureScheduleMode}; +#[cfg(test)] +use crate::api::{MixtureSpec, RateBackend}; +#[cfg(feature = "backend-calibrated")] use crate::backends::calibration::CalibratorCore; +#[cfg(feature = "backend-ctw")] +use crate::backends::ctw::{ContextTree, FacContextTree, ctw_symbol_bit_msb}; +#[cfg(feature = "backend-match")] use crate::backends::match_model::MatchModel; +#[cfg(feature = "backend-ppmd")] use crate::backends::ppmd::PpmdModel; +#[cfg(feature = "backend-rosa")] +use crate::backends::rosaplus::RosaPlus; +#[cfg(feature = "backend-sequitur")] use crate::backends::sequitur::SequiturModel; +#[cfg(feature = "backend-match")] use crate::backends::sparse_match::SparseMatchModel; use crate::backends::text_context::TextContextAnalyzer; +#[cfg(feature = "backend-zpaq")] +use crate::backends::zpaq_rate::ZpaqRateModel; use crate::coders::{ ANS_TOTAL, ArithmeticDecoder, ArithmeticEncoder, BlockedRansDecoder, BlockedRansEncoder, CDF_TOTAL, Cdf, CoderType, crc32, quantize_pdf_to_rans_cdf_with_buffer, }; -use crate::ctw::FacContextTree; #[cfg(feature = "backend-mamba")] use crate::mambazip; use crate::mixture::{ @@ -25,11 +42,9 @@ use crate::mixture::{ switching_alpha_for_update, }; use crate::neural_mix::NeuralMixCore; -use crate::rosaplus::RosaPlus; #[cfg(feature = "backend-rwkv")] use crate::rwkvzip; -use crate::zpaq_rate::ZpaqRateModel; -use crate::{CalibratedSpec, MixtureKind, MixtureScheduleMode, MixtureSpec, RateBackend}; +use crate::spec::CompiledRateBackend; use rayon::{ThreadPool, prelude::*}; const FRAMED_MAGIC: u32 = 0x4354_4946; // "FITC" @@ -37,11 +52,6 @@ const FRAMED_VERSION: u8 = 1; const PDF_MIN: f64 = DEFAULT_MIN_PROB; const DIAGNOSTIC_PARALLEL_THRESHOLD: usize = 4; -#[inline] -fn build_calibrator(spec: &CalibratedSpec) -> CalibratorCore { - CalibratorCore::new(spec.context, spec.bins, spec.learning_rate, spec.bias_clip) -} - #[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] /// Wire format mode for rate-coded payloads. pub enum FramingMode { @@ -65,13 +75,14 @@ impl FramedHeader { const SIZE: usize = 4 + 1 + 1 + 8 + 4; fn new(coder: CoderType, original_len: u64, crc32: u32) -> Self { + let coder = match coder { + CoderType::AC => 0, + CoderType::RANS => 1, + }; Self { magic: FRAMED_MAGIC, version: FRAMED_VERSION, - coder: match coder { - CoderType::AC => 0, - CoderType::RANS => 1, - }, + coder, original_len, crc32, } @@ -119,9 +130,17 @@ impl FramedHeader { } } +#[cfg(feature = "backend-ctw")] +#[derive(Clone)] +enum CtwCompressionTree { + Ac(Box), + Fac(FacContextTree), +} + +#[cfg(feature = "backend-ctw")] #[derive(Clone)] -struct CtwPredictor { - tree: FacContextTree, +pub(crate) struct CtwPredictor { + tree: CtwCompressionTree, bits_per_symbol: usize, msb_first: bool, pdf: Vec, @@ -129,10 +148,11 @@ struct CtwPredictor { valid: bool, } +#[cfg(feature = "backend-ctw")] impl CtwPredictor { - fn new_ctw(depth: usize) -> Self { + pub(crate) fn new_ctw(depth: usize) -> Self { Self { - tree: FacContextTree::new(depth, 8), + tree: CtwCompressionTree::Ac(Box::new(ContextTree::new(depth))), bits_per_symbol: 8, msb_first: true, pdf: vec![0.0; 256], @@ -141,11 +161,16 @@ impl CtwPredictor { } } - fn new_fac(base_depth: usize, bits_per_symbol: usize) -> Self { + pub(crate) fn new_fac( + base_depth: usize, + bits_per_symbol: usize, + msb_first: Option, + ) -> Self { + let effective_msb_first: bool = msb_first.unwrap_or(bits_per_symbol == 8); Self { - tree: FacContextTree::new(base_depth, bits_per_symbol), + tree: CtwCompressionTree::Fac(FacContextTree::new(base_depth, bits_per_symbol)), bits_per_symbol, - msb_first: false, + msb_first: effective_msb_first, pdf: vec![0.0; 256], pattern_logps: vec![f64::NEG_INFINITY; 256], valid: false, @@ -153,79 +178,136 @@ impl CtwPredictor { } fn fill_pattern_log_probs(&mut self) -> usize { - fn rec( - tree: &mut FacContextTree, - bits: usize, - msb_first: bool, - depth: usize, - pattern: usize, - log_before: f64, - out: &mut [f64], - ) { - if depth == bits { - out[pattern] = tree.get_log_block_probability() - log_before; - return; - } - for bit in [false, true] { - tree.update(bit, depth); - let next_pattern = if msb_first { - (pattern << 1) | (bit as usize) - } else { - pattern | ((bit as usize) << depth) - }; + let bits = self.bits_per_symbol.clamp(1, 8); + let patterns = 1usize << bits; + self.pattern_logps[..patterns].fill(f64::NEG_INFINITY); + match &mut self.tree { + CtwCompressionTree::Ac(tree) => { + fn rec( + tree: &mut ContextTree, + depth: usize, + bits: usize, + pattern: usize, + log_before: f64, + out: &mut [f64], + ) { + if depth == bits { + out[pattern] = tree.get_log_block_probability() - log_before; + return; + } + for bit in [false, true] { + tree.update(bit); + rec( + tree, + depth + 1, + bits, + (pattern << 1) | (bit as usize), + log_before, + out, + ); + tree.revert(); + } + } + + let log_before = tree.get_log_block_probability(); rec( tree, + 0, bits, - msb_first, - depth + 1, - next_pattern, + 0, log_before, - out, + &mut self.pattern_logps[..patterns], ); - tree.revert(depth); } - } + CtwCompressionTree::Fac(tree) => { + fn rec( + tree: &mut FacContextTree, + bits: usize, + msb_first: bool, + depth: usize, + pattern: usize, + log_before: f64, + out: &mut [f64], + ) { + if depth == bits { + out[pattern] = tree.get_log_block_probability() - log_before; + return; + } + for bit in [false, true] { + tree.update(bit, depth); + let next_pattern = if msb_first { + (pattern << 1) | (bit as usize) + } else { + pattern | ((bit as usize) << depth) + }; + rec( + tree, + bits, + msb_first, + depth + 1, + next_pattern, + log_before, + out, + ); + tree.revert(depth); + } + } - let bits = self.bits_per_symbol.clamp(1, 8); - let patterns = 1usize << bits; - let log_before = self.tree.get_log_block_probability(); - self.pattern_logps[..patterns].fill(f64::NEG_INFINITY); - rec( - &mut self.tree, - bits, - self.msb_first, - 0, - 0, - log_before, - &mut self.pattern_logps[..patterns], - ); + let log_before = tree.get_log_block_probability(); + rec( + tree, + bits, + self.msb_first, + 0, + 0, + log_before, + &mut self.pattern_logps[..patterns], + ); + } + } patterns } #[cfg(test)] fn log_prob_symbol_bruteforce(&mut self, symbol: u8) -> f64 { let bits = self.bits_per_symbol.clamp(1, 8); - let before = self.tree.get_log_block_probability(); - if self.msb_first { - for bit_idx in 0..bits { - let bit = ((symbol >> (7 - bit_idx)) & 1) == 1; - self.tree.update(bit, bit_idx); - } - let after = self.tree.get_log_block_probability(); - for bit_idx in (0..bits).rev() { - self.tree.revert(bit_idx); - } - after - before - } else { - for bit_idx in 0..bits { - let bit = ((symbol >> bit_idx) & 1) == 1; - self.tree.update(bit, bit_idx); - } - let after = self.tree.get_log_block_probability(); - for bit_idx in (0..bits).rev() { - self.tree.revert(bit_idx); + match &mut self.tree { + CtwCompressionTree::Ac(tree) => { + debug_assert!(self.msb_first); + let before = tree.get_log_block_probability(); + for bit_idx in 0..bits { + tree.update(ctw_symbol_bit_msb(symbol, bits, bit_idx)); + } + let after = tree.get_log_block_probability(); + for _ in 0..bits { + tree.revert(); + } + after - before + } + CtwCompressionTree::Fac(tree) => { + let before = tree.get_log_block_probability(); + if self.msb_first { + for bit_idx in 0..bits { + let bit = ((symbol >> (7 - bit_idx)) & 1) == 1; + tree.update(bit, bit_idx); + } + let after = tree.get_log_block_probability(); + for bit_idx in (0..bits).rev() { + tree.revert(bit_idx); + } + after - before + } else { + for bit_idx in 0..bits { + let bit = ((symbol >> bit_idx) & 1) == 1; + tree.update(bit, bit_idx); + } + let after = tree.get_log_block_probability(); + for bit_idx in (0..bits).rev() { + tree.revert(bit_idx); + } + after - before + } } - after - before } } @@ -275,20 +357,39 @@ impl CtwPredictor { } fn update(&mut self, symbol: u8) { - if self.msb_first { - for bit_idx in 0..self.bits_per_symbol { - let bit = ((symbol >> (7 - bit_idx)) & 1) == 1; - self.tree.update(bit, bit_idx); + match &mut self.tree { + CtwCompressionTree::Ac(tree) => { + debug_assert!(self.msb_first); + for bit_idx in 0..self.bits_per_symbol { + tree.update(ctw_symbol_bit_msb(symbol, self.bits_per_symbol, bit_idx)); + } } - } else { - for bit_idx in 0..self.bits_per_symbol { - let bit = ((symbol >> bit_idx) & 1) == 1; - self.tree.update(bit, bit_idx); + CtwCompressionTree::Fac(tree) => { + if self.msb_first { + for bit_idx in 0..self.bits_per_symbol { + let bit = ((symbol >> (7 - bit_idx)) & 1) == 1; + tree.update(bit, bit_idx); + } + } else { + for bit_idx in 0..self.bits_per_symbol { + let bit = ((symbol >> bit_idx) & 1) == 1; + tree.update(bit, bit_idx); + } + } } } self.valid = false; } + fn reserve_for_symbols(&mut self, total_symbols: usize) { + match &mut self.tree { + CtwCompressionTree::Ac(tree) => tree.reserve_for_symbols( + total_symbols.saturating_mul(self.bits_per_symbol.clamp(1, 8)), + ), + CtwCompressionTree::Fac(tree) => tree.reserve_for_symbols(total_symbols), + } + } + #[inline] fn can_fast_ac_bitwise(&self) -> bool { self.bits_per_symbol == 8 && self.msb_first @@ -297,19 +398,28 @@ impl CtwPredictor { #[inline] fn bit_prob_one_msb(&mut self, bit_idx: usize) -> f64 { debug_assert!(self.can_fast_ac_bitwise()); - self.tree.predict_one(bit_idx).clamp(PDF_MIN, 1.0 - PDF_MIN) + match &mut self.tree { + CtwCompressionTree::Ac(tree) => tree.predict(true).clamp(PDF_MIN, 1.0 - PDF_MIN), + CtwCompressionTree::Fac(tree) => { + tree.predict_one(bit_idx).clamp(PDF_MIN, 1.0 - PDF_MIN) + } + } } #[inline] fn update_bit_msb(&mut self, bit_idx: usize, bit: bool) { debug_assert!(self.can_fast_ac_bitwise()); - self.tree.update_predicted(bit, bit_idx); + match &mut self.tree { + CtwCompressionTree::Ac(tree) => tree.update(bit), + CtwCompressionTree::Fac(tree) => tree.update_predicted(bit, bit_idx), + } self.valid = false; } } +#[cfg(feature = "backend-rosa")] #[derive(Clone)] -struct RosaPredictor { +pub(crate) struct RosaPredictor { model: RosaPlus, pdf: Vec, cdf: [f64; 257], @@ -317,8 +427,9 @@ struct RosaPredictor { cdf_valid: bool, } +#[cfg(feature = "backend-rosa")] impl RosaPredictor { - fn new(max_order: i64) -> Self { + pub(crate) fn new(max_order: i64) -> Self { let mut model = RosaPlus::new(max_order, false, 0, 42); model.build_lm_full_bytes_no_finalize_endpos(); Self { @@ -370,7 +481,7 @@ impl RosaPredictor { #[derive(Clone)] #[cfg(feature = "backend-mamba")] -struct MambaPredictor { +pub(crate) struct MambaPredictor { compressor: mambazip::Compressor, primed: bool, pdf: Vec, @@ -381,23 +492,25 @@ struct MambaPredictor { #[derive(Clone)] #[cfg(feature = "backend-rwkv")] -struct RwkvPredictor { +pub(crate) struct RwkvPredictor { compressor: rwkvzip::Compressor, primed: bool, cdf: [f64; 257], cdf_valid: bool, } +#[cfg(feature = "backend-zpaq")] #[derive(Clone)] -struct ZpaqPredictor { +pub(crate) struct ZpaqPredictor { method: String, history: Vec, pdf: Vec, valid: bool, } +#[cfg(feature = "backend-zpaq")] impl ZpaqPredictor { - fn new(method: String) -> Self { + pub(crate) fn new(method: String) -> Self { Self { method, history: Vec::new(), @@ -430,21 +543,14 @@ impl ZpaqPredictor { #[cfg(feature = "backend-mamba")] impl MambaPredictor { - fn from_model(model: std::sync::Arc) -> Self { - let compressor = mambazip::Compressor::new_from_model(model); - let vocab = compressor.vocab_size(); - Self { - compressor, - primed: false, - pdf: vec![0.0; vocab], - cdf: uniform_cdf_row(), - valid: false, - cdf_valid: false, - } + #[cfg(test)] + fn from_method(method: &str) -> Result { + let spec = mambazip::parse_method_spec(method)?; + Self::from_method_spec(&spec) } - fn from_method(method: &str) -> Result { - let compressor = mambazip::Compressor::new_from_method(method)?; + pub(crate) fn from_method_spec(method: &mambazip::MethodSpec) -> Result { + let compressor = mambazip::Compressor::new_from_method_spec(method)?; let vocab = compressor.vocab_size(); Ok(Self { compressor, @@ -513,18 +619,14 @@ impl MambaPredictor { #[cfg(feature = "backend-rwkv")] impl RwkvPredictor { - fn from_model(model: std::sync::Arc) -> Self { - let compressor = rwkvzip::Compressor::new_from_model(model); - Self { - compressor, - primed: false, - cdf: uniform_cdf_row(), - cdf_valid: false, - } + #[cfg(test)] + fn from_method(method: &str) -> Result { + let spec = rwkvzip::parse_method_spec(method)?; + Self::from_method_spec(&spec) } - fn from_method(method: &str) -> Result { - let compressor = rwkvzip::Compressor::new_from_method(method)?; + pub(crate) fn from_method_spec(method: &rwkvzip::MethodSpec) -> Result { + let compressor = rwkvzip::Compressor::new_from_method_spec(method)?; Ok(Self { compressor, primed: false, @@ -581,6 +683,84 @@ struct MixExpert { cum_log_loss: f64, } +#[derive(Clone, Debug)] +enum PredictorBitwiseStepState { + NativeRecursive, + CachedCdf { lo: usize, hi: usize }, + PdfPrefix { cdf: Vec, lo: usize, hi: usize }, +} + +impl Default for PredictorBitwiseStepState { + fn default() -> Self { + Self::PdfPrefix { + cdf: Vec::new(), + lo: 0, + hi: 256, + } + } +} + +impl PredictorBitwiseStepState { + fn prepare(&mut self, predictor: &mut RatePdfPredictor) -> Result<()> { + if predictor.begin_native_recursive_bitwise_byte_step()? { + *self = Self::NativeRecursive; + return Ok(()); + } + if predictor.prepare_cached_cdf_fast_bitwise()? { + *self = Self::CachedCdf { lo: 0, hi: 256 }; + return Ok(()); + } + + let mut cdf = match std::mem::take(self) { + Self::PdfPrefix { cdf, .. } => cdf, + _ => Vec::new(), + }; + rebuild_bitwise_prefix_cdf_row(&mut cdf, predictor.pdf_next()?); + *self = Self::PdfPrefix { + cdf, + lo: 0, + hi: 256, + }; + Ok(()) + } + + fn bit_prob_one_msb( + &mut self, + predictor: &mut RatePdfPredictor, + bit_idx: usize, + ) -> Result { + match self { + Self::NativeRecursive => predictor.native_recursive_bit_prob_one_msb(bit_idx), + Self::CachedCdf { lo, hi } => Ok(predictor + .cached_cdf_bit_prob_one_msb(*lo, *hi) + .expect("CachedCdf state invariant violated: missing cached CDF entry")), + Self::PdfPrefix { cdf, lo, hi } => Ok(cdf_bit_prob_one_msb(cdf, *lo, *hi)), + } + } + + fn observe_bit_msb( + &mut self, + predictor: &mut RatePdfPredictor, + bit_idx: usize, + bit: bool, + ) -> Result<()> { + match self { + Self::NativeRecursive => predictor.native_recursive_observe_bit_msb(bit_idx, bit), + Self::CachedCdf { lo, hi } | Self::PdfPrefix { lo, hi, .. } => { + advance_msb_prefix_range(lo, hi, bit); + Ok(()) + } + } + } + + fn finish_symbol(&mut self, predictor: &mut RatePdfPredictor, symbol: u8) -> Result<()> { + match self { + Self::NativeRecursive => predictor.finish_native_recursive_bitwise_byte_step(symbol), + Self::CachedCdf { .. } | Self::PdfPrefix { .. } => predictor.update(symbol), + } + } +} + #[derive(Clone, Copy, Debug, Default)] pub(crate) struct AcLogLossNodeValue { pub(crate) prob: f64, @@ -605,7 +785,7 @@ pub(crate) struct AcLogLossRootSnapshot { } #[derive(Clone)] -struct MixturePredictor { +pub(crate) struct MixturePredictor { kind: MixtureKind, schedule: MixtureScheduleMode, alpha: f64, @@ -614,11 +794,11 @@ struct MixturePredictor { prior_weights: Vec, neural: NeuralMixCore, analyzer: TextContextAnalyzer, - neural_logps: Vec, - neural_bit_modes: Vec, - neural_lo: Vec, - neural_hi: Vec, - neural_pdf_cdf_rows: Vec>, + bitwise_expert_states: Vec, + // Reused per-expert observation scratch: symbol paths store log p(symbol), + // while bitwise AC temporarily stages p(bit = 1) before collapsing back to + // the symbol log-probability at byte completion. + expert_observation_scratch: Vec, scratch: Vec, scratch2: Vec, projection_scratch: Vec, @@ -629,17 +809,27 @@ struct MixturePredictor { } impl MixturePredictor { - fn new(spec: &MixtureSpec) -> Result { - spec.validate().map_err(anyhow::Error::msg)?; - let mut experts = Vec::with_capacity(spec.experts.len()); - for e in &spec.experts { + pub(crate) fn new_from_compiled(backend: &CompiledRateBackend) -> Result { + let crate::spec::core::RateBackendPlan::Mixture { + kind, + schedule, + alpha, + decay, + experts: plan_experts, + .. + } = backend.plan() + else { + bail!("compiled backend is not a mixture backend"); + }; + let mut experts = Vec::with_capacity(plan_experts.len()); + for expert_plan in plan_experts.iter() { + let compiled = + crate::spec::core::compiled_rate_backend_from_plan(expert_plan.backend.clone()) + .map_err(anyhow::Error::msg)?; experts.push(MixExpert { - predictor: Box::new(RatePdfPredictor::from_rate_backend( - e.backend.clone(), - e.max_order, - )?), - log_weight: e.log_prior, - log_prior: e.log_prior, + predictor: Box::new(crate::runtime::build_rate_pdf_predictor(&compiled)?), + log_weight: expert_plan.log_prior, + log_prior: expert_plan.log_prior, cum_log_loss: 0.0, }); } @@ -655,7 +845,7 @@ impl MixturePredictor { *weight = weight.clamp(PDF_MIN, 1.0 - PDF_MIN); } - let base_lr = spec.alpha.abs().clamp(1e-6, 1.0); + let base_lr = alpha.abs().clamp(1e-6, 1.0); let effective_lr = (base_lr * 25.0).clamp(1e-6, 1.0); let analyzer = TextContextAnalyzer::new(); let mut neural = NeuralMixCore::new( @@ -667,19 +857,16 @@ impl MixturePredictor { ); neural.set_context_state(analyzer.state()); Ok(Self { - kind: spec.kind, - schedule: spec.schedule, - alpha: spec.alpha, - decay: spec.decay.unwrap_or(1.0).clamp(0.0, 1.0), + kind: *kind, + schedule: *schedule, + alpha: *alpha, + decay: decay.unwrap_or(1.0).clamp(0.0, 1.0), experts, prior_weights, neural, analyzer, - neural_logps: vec![0.0; spec.experts.len()], - neural_bit_modes: vec![0; spec.experts.len()], - neural_lo: vec![0; spec.experts.len()], - neural_hi: vec![256; spec.experts.len()], - neural_pdf_cdf_rows: vec![vec![0.0; 257]; spec.experts.len()], + bitwise_expert_states: Vec::new(), + expert_observation_scratch: vec![0.0; plan_experts.len()], scratch: Vec::new(), scratch2: Vec::new(), projection_scratch: Vec::new(), @@ -782,11 +969,7 @@ impl MixturePredictor { return Ok(&self.pdf); } let weights = self.predictive_weights(); - if weights.len() == 1 && matches!(self.kind, MixtureKind::Mdl | MixtureKind::Neural) { - self.pdf.fill(0.0); - } else { - self.pdf.fill(0.0); - } + self.pdf.fill(0.0); for (index, expert) in self.experts.iter_mut().enumerate() { let weight = weights.get(index).copied().unwrap_or(0.0); if weight <= 0.0 { @@ -808,6 +991,7 @@ impl MixturePredictor { match &mut *expert.predictor { // Direct CTW benefits from pre-reserving, but inside mixtures that extra // headroom can dominate peak RSS without a proportional runtime gain. + #[cfg(feature = "backend-ctw")] RatePdfPredictor::Ctw(_) | RatePdfPredictor::FacCtw(_) => {} _ => expert.predictor.begin_stream(total_len)?, } @@ -1072,16 +1256,17 @@ impl MixturePredictor { } let n = self.experts.len(); self.neural.set_context_state(self.analyzer.state()); - self.neural_logps.resize(n, 0.0); + self.expert_observation_scratch.resize(n, 0.0); for i in 0..n { let p = self.experts[i].predictor.pdf_next()?[y].max(PDF_MIN); let lp = p.ln(); - self.neural_logps[i] = lp; + self.expert_observation_scratch[i] = lp; self.experts[i].cum_log_loss -= lp; } - self.neural.evaluate_symbol(&self.neural_logps, PDF_MIN); self.neural - .update_weights_symbol(&self.neural_logps, PDF_MIN); + .evaluate_symbol(&self.expert_observation_scratch, PDF_MIN); + self.neural + .update_weights_symbol(&self.expert_observation_scratch, PDF_MIN); for e in &mut self.experts { e.predictor.update(symbol)?; } @@ -1102,20 +1287,17 @@ impl MixturePredictor { } #[inline] - fn can_fast_ac_bitwise(&self) -> bool { - self.experts.iter().any(|e| { - if let RatePdfPredictor::Ctw(ctw) = &*e.predictor { - ctw.can_fast_ac_bitwise() - } else { - false - } - }) + fn has_recursive_native_bitwise_expert(&self) -> bool { + self.experts + .iter() + .any(|expert| expert.predictor.has_recursive_native_bitwise_path()) } - fn ac_step_bitwise(&mut self, mut choose_bit: F) -> Result - where - F: FnMut(usize, f64) -> Result, - { + fn begin_bitwise_byte_step(&mut self) -> Result { + if !self.has_recursive_native_bitwise_expert() { + return Ok(false); + } + let n = self.experts.len(); self.scratch.resize(n, 0.0); match self.kind { @@ -1124,14 +1306,6 @@ impl MixturePredictor { self.neural.evaluate_expert_weights(); self.scratch.copy_from_slice(self.neural.expert_weights()); } - MixtureKind::FadingBayes => { - let weights = self.predictive_weights(); - self.scratch.copy_from_slice(&weights); - } - MixtureKind::Mdl => { - let weights = self.predictive_weights(); - self.scratch.copy_from_slice(&weights); - } _ => { let weights = self.predictive_weights(); self.scratch.copy_from_slice(&weights); @@ -1139,140 +1313,74 @@ impl MixturePredictor { } self.scratch2.resize(n, 1.0); self.scratch2.fill(1.0); - self.neural_logps.resize(n, 0.0); - self.neural_bit_modes.resize(n, 0); - self.neural_lo.resize(n, 0); - self.neural_hi.resize(n, 256); - if self.neural_pdf_cdf_rows.len() < n { - self.neural_pdf_cdf_rows.resize_with(n, || vec![0.0; 257]); - } - + self.expert_observation_scratch.resize(n, 0.0); + self.bitwise_expert_states + .resize_with(n, PredictorBitwiseStepState::default); for i in 0..n { - self.neural_bit_modes[i] = 1; - self.neural_lo[i] = 0; - self.neural_hi[i] = 256; - - let mut handled_ctw = false; - if let RatePdfPredictor::Ctw(ctw) = &mut *self.experts[i].predictor - && ctw.can_fast_ac_bitwise() - { - self.neural_bit_modes[i] = 0; - handled_ctw = true; - } - if handled_ctw { - continue; - } - - if self.experts[i] - .predictor - .prepare_cached_cdf_fast_bitwise()? - { - self.neural_bit_modes[i] = 2; - continue; - } - - let pdf = self.experts[i].predictor.pdf_next()?; - let row = &mut self.neural_pdf_cdf_rows[i]; - if row.len() != 257 { - row.resize(257, 0.0); - } - row[0] = 0.0; - for b in 0..256usize { - row[b + 1] = row[b] + pdf[b].max(PDF_MIN); - } - if !row[256].is_finite() || row[256] <= 0.0 { - for (j, v) in row.iter_mut().enumerate() { - *v = (j as f64) / 256.0; - } - } + self.bitwise_expert_states[i].prepare(&mut self.experts[i].predictor)?; } + Ok(true) + } - let mut symbol = 0u8; - for bit_idx in 0..8usize { - let mut denom = 0.0; - let mut numer1 = 0.0; - - for i in 0..n { - let p1 = if self.neural_bit_modes[i] == 0 { - match &mut *self.experts[i].predictor { - RatePdfPredictor::Ctw(ctw) => ctw.bit_prob_one_msb(bit_idx), - _ => 0.5, - } - } else if self.neural_bit_modes[i] == 2 { - self.experts[i] - .predictor - .cached_cdf_bit_prob_one_msb(self.neural_lo[i], self.neural_hi[i]) - .unwrap_or(0.5) - } else { - let lo = self.neural_lo[i]; - let hi = self.neural_hi[i]; - let mid = (lo + hi) >> 1; - let row = &self.neural_pdf_cdf_rows[i]; - let total = (row[hi] - row[lo]).max(PDF_MIN); - let one = (row[hi] - row[mid]).max(0.0); - (one / total).clamp(PDF_MIN, 1.0 - PDF_MIN) - }; - self.neural_logps[i] = p1; - let wp = self.scratch[i] * self.scratch2[i]; - denom += wp; - numer1 += wp * p1; - } - - let p1_mix = if denom.is_finite() && denom > 0.0 { - (numer1 / denom).clamp(PDF_MIN, 1.0 - PDF_MIN) - } else { - 0.5 - }; - let bit = choose_bit(bit_idx, p1_mix)? & 1; - symbol |= bit << (7 - bit_idx); - - for i in 0..n { - let p1 = self.neural_logps[i]; - let pb = if bit == 1 { p1 } else { 1.0 - p1 }; - self.scratch2[i] = (self.scratch2[i] * pb).max(PDF_MIN); + fn bit_prob_one_msb(&mut self, bit_idx: usize) -> Result { + let mut denom = 0.0; + let mut numer1 = 0.0; + for i in 0..self.experts.len() { + let p1 = self.bitwise_expert_states[i] + .bit_prob_one_msb(&mut self.experts[i].predictor, bit_idx)?; + self.expert_observation_scratch[i] = p1; + let wp = self.scratch[i] * self.scratch2[i]; + denom += wp; + numer1 += wp * p1; + } + Ok(if denom.is_finite() && denom > 0.0 { + (numer1 / denom).clamp(PDF_MIN, 1.0 - PDF_MIN) + } else { + panic!( + "MixturePredictor bit_prob_one_msb: invalid denom (finite>0 violated); \ + this is an internal invariant failure in the bitwise mixture state machine" + ) + }) + } - if self.neural_bit_modes[i] == 0 { - if let RatePdfPredictor::Ctw(ctw) = &mut *self.experts[i].predictor { - ctw.update_bit_msb(bit_idx, bit == 1); - } - } else { - let lo = self.neural_lo[i]; - let hi = self.neural_hi[i]; - let mid = (lo + hi) >> 1; - if bit == 1 { - self.neural_lo[i] = mid; - self.neural_hi[i] = hi; - } else { - self.neural_lo[i] = lo; - self.neural_hi[i] = mid; - } - } - } + fn observe_bit_msb(&mut self, bit_idx: usize, bit: bool) -> Result<()> { + for i in 0..self.experts.len() { + let p1 = self.expert_observation_scratch[i]; + let pb = if bit { p1 } else { 1.0 - p1 }; + self.scratch2[i] = (self.scratch2[i] * pb).max(PDF_MIN); + self.bitwise_expert_states[i].observe_bit_msb( + &mut self.experts[i].predictor, + bit_idx, + bit, + )?; } + Ok(()) + } + fn finish_bitwise_symbol(&mut self, symbol: u8) -> Result<()> { + let n = self.experts.len(); for i in 0..n { let lp = self.scratch2[i].max(PDF_MIN).ln(); - self.neural_logps[i] = lp; + self.expert_observation_scratch[i] = lp; self.experts[i].cum_log_loss -= lp; - if self.neural_bit_modes[i] != 0 { - self.experts[i].predictor.update(symbol)?; - } + self.bitwise_expert_states[i].finish_symbol(&mut self.experts[i].predictor, symbol)?; } match self.kind { MixtureKind::Bayes => { for i in 0..n { - self.scratch[i] = self.experts[i].log_weight + self.neural_logps[i]; + self.scratch[i] = + self.experts[i].log_weight + self.expert_observation_scratch[i]; } let log_mix = logsumexp_slice(&self.scratch[..n]); for i in 0..n { - self.experts[i].log_weight += self.neural_logps[i] - log_mix; + self.experts[i].log_weight += self.expert_observation_scratch[i] - log_mix; } } MixtureKind::FadingBayes => { for i in 0..n { - self.scratch[i] = - self.decay * self.experts[i].log_weight + self.neural_logps[i]; + self.scratch[i] = self.decay * self.experts[i].log_weight + + self.expert_observation_scratch[i]; } let log_mix = logsumexp_slice(&self.scratch[..n]); for i in 0..n { @@ -1281,7 +1389,8 @@ impl MixturePredictor { } MixtureKind::Switching => { for i in 0..n { - self.scratch[i] = self.experts[i].log_weight + self.neural_logps[i]; + self.scratch[i] = + self.experts[i].log_weight + self.expert_observation_scratch[i]; } let log_mix = logsumexp_slice(&self.scratch[..n]); for weight in &mut self.scratch[..n] { @@ -1305,7 +1414,7 @@ impl MixturePredictor { self.scratch2[i] = self.experts[i].log_weight.exp(); } let mix_prob = self - .neural_logps + .expert_observation_scratch .iter() .zip(self.scratch2.iter()) .map(|(&lp, &w)| w * lp.exp()) @@ -1316,7 +1425,7 @@ impl MixturePredictor { let eta = convex_step_size_for_update(self.schedule, self.alpha, self.convex_updates); for i in 0..n { - let grad = -(self.neural_logps[i] - log_mix).exp(); + let grad = -(self.expert_observation_scratch[i] - log_mix).exp(); self.scratch2[i] -= eta * grad; } project_simplex_with_scratch(&mut self.scratch2[..n], &mut self.projection_scratch); @@ -1328,16 +1437,17 @@ impl MixturePredictor { MixtureKind::Neural => { if n > 1 { self.neural.set_context_state(self.analyzer.state()); - self.neural.evaluate_symbol(&self.neural_logps, PDF_MIN); self.neural - .update_weights_symbol(&self.neural_logps, PDF_MIN); + .evaluate_symbol(&self.expert_observation_scratch, PDF_MIN); + self.neural + .update_weights_symbol(&self.expert_observation_scratch, PDF_MIN); } self.analyzer.update(symbol); self.neural.set_context_state(self.analyzer.state()); } } self.valid = false; - Ok(symbol) + Ok(()) } } @@ -1346,9 +1456,15 @@ pub(crate) struct DiagnosticRatePredictor { } impl DiagnosticRatePredictor { - pub(crate) fn from_rate_backend(backend: RateBackend, max_order: i64) -> Result { + #[cfg(test)] + pub(crate) fn from_rate_backend(backend: RateBackend) -> Result { + let compiled = backend.compile().map_err(anyhow::Error::msg)?; + Self::from_compiled(&compiled) + } + + pub(crate) fn from_compiled(backend: &CompiledRateBackend) -> Result { Ok(Self { - inner: RatePdfPredictor::from_rate_backend(backend, max_order)?, + inner: crate::runtime::build_rate_pdf_predictor(backend)?, }) } @@ -1383,190 +1499,161 @@ impl DiagnosticRatePredictor { &mut self, symbol: u8, encoder: &mut ArithmeticEncoder, + cdf: &mut [u32; 257], ) -> Result<()> { - self.inner.encode_symbol_ac_step(symbol, encoder) + self.inner.encode_symbol_ac_step(symbol, encoder, cdf) } } #[derive(Clone)] #[allow(clippy::large_enum_variant)] -enum RatePdfPredictor { +pub(crate) enum RatePdfPredictor { + #[cfg(feature = "backend-rosa")] Rosa(RosaPredictor), - Match { - model: MatchModel, - }, - SparseMatch { - model: SparseMatchModel, - }, - Ppmd { - model: PpmdModel, - }, - Sequitur { - model: SequiturModel, - }, + #[cfg(feature = "backend-match")] + Match { model: MatchModel }, + #[cfg(feature = "backend-match")] + SparseMatch { model: SparseMatchModel }, + #[cfg(feature = "backend-ppmd")] + Ppmd { model: PpmdModel }, + #[cfg(feature = "backend-sequitur")] + Sequitur { model: SequiturModel }, + #[cfg(feature = "backend-ctw")] Ctw(CtwPredictor), + #[cfg(feature = "backend-ctw")] FacCtw(CtwPredictor), #[cfg(feature = "backend-mamba")] Mamba(MambaPredictor), #[cfg(feature = "backend-rwkv")] Rwkv(RwkvPredictor), + #[cfg(feature = "backend-zpaq")] Zpaq(ZpaqPredictor), + #[cfg(feature = "backend-mixture")] Mixture(MixturePredictor), - Particle(crate::particle::ParticleRuntime), + #[cfg(feature = "backend-particle")] + Particle(crate::backends::particle::ParticleRuntime), + #[cfg(feature = "backend-calibrated")] Calibrated { base: Box, core: CalibratorCore, pdf: Vec, valid: bool, }, + #[allow(dead_code)] + Disabled { reason: String }, } impl RatePdfPredictor { - fn from_rate_backend(backend: RateBackend, max_order: i64) -> Result { - match backend { - RateBackend::RosaPlus => Ok(Self::Rosa(RosaPredictor::new(max_order))), - RateBackend::Match { - hash_bits, - min_len, - max_len, - base_mix, - confidence_scale, - } => Ok(Self::Match { - model: MatchModel::new_contiguous( - hash_bits, - min_len, - max_len, - base_mix, - confidence_scale, - ), - }), - RateBackend::SparseMatch { - hash_bits, - min_len, - max_len, - gap_min, - gap_max, - base_mix, - confidence_scale, - } => Ok(Self::SparseMatch { - model: SparseMatchModel::new( - hash_bits, - min_len, - max_len, - gap_min, - gap_max, - base_mix, - confidence_scale, - ), - }), - RateBackend::Ppmd { order, memory_mb } => Ok(Self::Ppmd { - model: PpmdModel::new(order, memory_mb), - }), - RateBackend::Sequitur { context_bytes } => Ok(Self::Sequitur { - model: SequiturModel::new(context_bytes), - }), - RateBackend::Ctw { depth } => Ok(Self::Ctw(CtwPredictor::new_ctw(depth))), - RateBackend::FacCtw { - base_depth, - num_percept_bits: _, - encoding_bits, - } => { - let bits = encoding_bits.clamp(1, 8); - Ok(Self::FacCtw(CtwPredictor::new_fac(base_depth, bits))) - } - #[cfg(feature = "backend-mamba")] - RateBackend::Mamba { model } => Ok(Self::Mamba(MambaPredictor::from_model(model))), - #[cfg(feature = "backend-mamba")] - RateBackend::MambaMethod { method } => { - Ok(Self::Mamba(MambaPredictor::from_method(&method)?)) - } - #[cfg(feature = "backend-rwkv")] - RateBackend::Rwkv7 { model } => Ok(Self::Rwkv(RwkvPredictor::from_model(model))), - #[cfg(feature = "backend-rwkv")] - RateBackend::Rwkv7Method { method } => { - Ok(Self::Rwkv(RwkvPredictor::from_method(&method)?)) - } - RateBackend::Zpaq { method } => Ok(Self::Zpaq(ZpaqPredictor::new(method))), - RateBackend::Mixture { spec } => { - Ok(Self::Mixture(MixturePredictor::new(spec.as_ref())?)) - } - RateBackend::Particle { spec } => Ok(Self::Particle( - crate::particle::ParticleRuntime::new(spec.as_ref()), - )), - RateBackend::Calibrated { spec } => Ok(Self::Calibrated { - base: Box::new(Self::from_rate_backend(spec.base.clone(), max_order)?), - core: build_calibrator(spec.as_ref()), - pdf: vec![1.0 / 256.0; 256], - valid: false, - }), - } + #[cfg(test)] + fn from_compiled(backend: &CompiledRateBackend) -> Result { + crate::runtime::build_rate_pdf_predictor(backend) + } + + #[cfg(test)] + pub(crate) fn from_rate_backend(backend: RateBackend) -> Result { + let compiled = backend.compile().map_err(anyhow::Error::msg)?; + Self::from_compiled(&compiled) } fn begin_stream(&mut self, total_len: usize) -> Result<()> { self.finish_stream()?; match self { + #[cfg(feature = "backend-rosa")] Self::Rosa(m) => { m.begin_stream(total_len); Ok(()) } - Self::Match { .. } - | Self::SparseMatch { .. } - | Self::Ppmd { .. } - | Self::Zpaq(_) - | Self::Particle(_) => Ok(()), + #[cfg(feature = "backend-match")] + Self::Match { .. } => Ok(()), + #[cfg(feature = "backend-match")] + Self::SparseMatch { .. } => Ok(()), + #[cfg(feature = "backend-ppmd")] + Self::Ppmd { .. } => Ok(()), + #[cfg(feature = "backend-zpaq")] + Self::Zpaq(_) => Ok(()), + #[cfg(feature = "backend-particle")] + Self::Particle(_) => Ok(()), + #[cfg(feature = "backend-sequitur")] Self::Sequitur { model } => { model.begin_stream(Some(total_len as u64)); Ok(()) } + #[cfg(feature = "backend-ctw")] Self::Ctw(m) | Self::FacCtw(m) => { - m.tree.reserve_for_symbols(total_len); + m.reserve_for_symbols(total_len); Ok(()) } #[cfg(feature = "backend-mamba")] Self::Mamba(m) => m.begin_stream(total_len), #[cfg(feature = "backend-rwkv")] Self::Rwkv(m) => m.begin_stream(total_len), + #[cfg(feature = "backend-mixture")] Self::Mixture(m) => m.begin_stream(total_len), + #[cfg(feature = "backend-calibrated")] Self::Calibrated { base, .. } => base.begin_stream(total_len), + Self::Disabled { reason } => bail!("{reason}"), } } fn finish_stream(&mut self) -> Result<()> { match self { - Self::Rosa(_) - | Self::Match { .. } - | Self::SparseMatch { .. } - | Self::Ppmd { .. } - | Self::Sequitur { .. } - | Self::Ctw(_) - | Self::FacCtw(_) - | Self::Zpaq(_) - | Self::Particle(_) => Ok(()), + #[cfg(feature = "backend-rosa")] + Self::Rosa(_) => Ok(()), + #[cfg(feature = "backend-match")] + Self::Match { .. } => Ok(()), + #[cfg(feature = "backend-match")] + Self::SparseMatch { .. } => Ok(()), + #[cfg(feature = "backend-ppmd")] + Self::Ppmd { .. } => Ok(()), + #[cfg(feature = "backend-ctw")] + Self::Ctw(_) => Ok(()), + #[cfg(feature = "backend-ctw")] + Self::FacCtw(_) => Ok(()), + #[cfg(feature = "backend-zpaq")] + Self::Zpaq(_) => Ok(()), + #[cfg(feature = "backend-particle")] + Self::Particle(_) => Ok(()), + #[cfg(feature = "backend-sequitur")] + Self::Sequitur { .. } => Ok(()), #[cfg(feature = "backend-mamba")] - Self::Mamba(_) => Ok(()), + Self::Mamba(m) => m.compressor.finish_online_policy_stream(), #[cfg(feature = "backend-rwkv")] Self::Rwkv(m) => m.finish_stream(), + #[cfg(feature = "backend-mixture")] Self::Mixture(m) => m.finish_stream(), + #[cfg(feature = "backend-calibrated")] Self::Calibrated { base, .. } => base.finish_stream(), + Self::Disabled { .. } => Ok(()), } } fn pdf_next(&mut self) -> Result<&[f64]> { match self { + #[cfg(feature = "backend-rosa")] Self::Rosa(m) => Ok(m.pdf_next()), + #[cfg(feature = "backend-match")] Self::Match { model } => Ok(model.pdf()), + #[cfg(feature = "backend-ctw")] Self::Ctw(m) => Ok(m.pdf_next()), + #[cfg(feature = "backend-ctw")] Self::FacCtw(m) => Ok(m.pdf_next()), #[cfg(feature = "backend-mamba")] Self::Mamba(m) => Ok(m.pdf_next()), #[cfg(feature = "backend-rwkv")] Self::Rwkv(m) => Ok(m.pdf_next()), + #[cfg(feature = "backend-zpaq")] Self::Zpaq(m) => Ok(m.pdf_next()), + #[cfg(feature = "backend-mixture")] Self::Mixture(m) => m.ensure_pdf(), + #[cfg(feature = "backend-particle")] Self::Particle(m) => Ok(m.pdf_next()), + #[cfg(feature = "backend-match")] Self::SparseMatch { model } => Ok(model.pdf()), + #[cfg(feature = "backend-ppmd")] Self::Ppmd { model } => Ok(model.pdf()), + #[cfg(feature = "backend-sequitur")] Self::Sequitur { model } => Ok(model.pdf()), + #[cfg(feature = "backend-calibrated")] Self::Calibrated { base, core, @@ -1581,35 +1668,43 @@ impl RatePdfPredictor { } Ok(pdf) } + Self::Disabled { reason } => bail!("{reason}"), } } fn update(&mut self, symbol: u8) -> Result<()> { match self { + #[cfg(feature = "backend-rosa")] Self::Rosa(m) => { m.update(symbol); Ok(()) } + #[cfg(feature = "backend-match")] Self::Match { model } => { model.update(symbol); Ok(()) } + #[cfg(feature = "backend-match")] Self::SparseMatch { model } => { model.update(symbol); Ok(()) } + #[cfg(feature = "backend-ppmd")] Self::Ppmd { model } => { model.update(symbol); Ok(()) } + #[cfg(feature = "backend-sequitur")] Self::Sequitur { model } => { model.update(symbol); Ok(()) } + #[cfg(feature = "backend-ctw")] Self::Ctw(m) => { m.update(symbol); Ok(()) } + #[cfg(feature = "backend-ctw")] Self::FacCtw(m) => { m.update(symbol); Ok(()) @@ -1618,15 +1713,19 @@ impl RatePdfPredictor { Self::Mamba(m) => m.update(symbol), #[cfg(feature = "backend-rwkv")] Self::Rwkv(m) => m.update(symbol), + #[cfg(feature = "backend-zpaq")] Self::Zpaq(m) => { m.update(symbol); Ok(()) } + #[cfg(feature = "backend-mixture")] Self::Mixture(m) => m.update(symbol), + #[cfg(feature = "backend-particle")] Self::Particle(m) => { m.step(symbol); Ok(()) } + #[cfg(feature = "backend-calibrated")] Self::Calibrated { base, core, @@ -1643,23 +1742,28 @@ impl RatePdfPredictor { *valid = false; Ok(()) } + Self::Disabled { reason } => bail!("{reason}"), } } fn prepare_cached_cdf_fast_bitwise(&mut self) -> Result { match self { + #[cfg(feature = "backend-rosa")] Self::Rosa(m) => { let _ = m.cdf_next(); Ok(true) } + #[cfg(feature = "backend-match")] Self::Match { model } => { let _ = model.cdf(); Ok(true) } + #[cfg(feature = "backend-match")] Self::SparseMatch { model } => { let _ = model.cdf(); Ok(true) } + #[cfg(feature = "backend-ppmd")] Self::Ppmd { model } => { let _ = model.cdf(); Ok(true) @@ -1680,9 +1784,13 @@ impl RatePdfPredictor { fn cached_cdf_bit_prob_one_msb(&mut self, lo: usize, hi: usize) -> Option { match self { + #[cfg(feature = "backend-rosa")] Self::Rosa(m) => Some(cdf_bit_prob_one_msb(&m.cdf, lo, hi)), + #[cfg(feature = "backend-match")] Self::Match { model } => Some(cdf_bit_prob_one_msb(model.cdf(), lo, hi)), + #[cfg(feature = "backend-match")] Self::SparseMatch { model } => Some(cdf_bit_prob_one_msb(model.cdf(), lo, hi)), + #[cfg(feature = "backend-ppmd")] Self::Ppmd { model } => Some(cdf_bit_prob_one_msb(model.cdf(), lo, hi)), #[cfg(feature = "backend-mamba")] Self::Mamba(m) => Some(cdf_bit_prob_one_msb(m.cdf_next(), lo, hi)), @@ -1693,23 +1801,92 @@ impl RatePdfPredictor { } #[inline] - fn can_fast_ac_bitwise(&self) -> bool { + fn has_recursive_native_bitwise_path(&self) -> bool { match self { - Self::Ctw(m) => m.can_fast_ac_bitwise(), - Self::Mixture(m) => m.can_fast_ac_bitwise(), + #[cfg(feature = "backend-ctw")] + Self::Ctw(m) | Self::FacCtw(m) => m.can_fast_ac_bitwise(), + #[cfg(feature = "backend-mixture")] + Self::Mixture(m) => m.has_recursive_native_bitwise_expert(), _ => false, } } - fn ac_step_fast_bitwise(&mut self, choose_bit: F) -> Result + fn begin_native_recursive_bitwise_byte_step(&mut self) -> Result { + match self { + #[cfg(feature = "backend-ctw")] + Self::Ctw(m) | Self::FacCtw(m) => Ok(m.can_fast_ac_bitwise()), + #[cfg(feature = "backend-mixture")] + Self::Mixture(m) => m.begin_bitwise_byte_step(), + _ => Ok(false), + } + } + + fn native_recursive_bit_prob_one_msb(&mut self, bit_idx: usize) -> Result { + match self { + #[cfg(feature = "backend-ctw")] + Self::Ctw(m) | Self::FacCtw(m) => Ok(m.bit_prob_one_msb(bit_idx)), + #[cfg(feature = "backend-mixture")] + Self::Mixture(m) => m.bit_prob_one_msb(bit_idx), + _ => bail!("native recursive bitwise stepping is unavailable for this predictor"), + } + } + + fn native_recursive_observe_bit_msb(&mut self, bit_idx: usize, bit: bool) -> Result<()> { + match self { + #[cfg(feature = "backend-ctw")] + Self::Ctw(m) | Self::FacCtw(m) => { + m.update_bit_msb(bit_idx, bit); + Ok(()) + } + #[cfg(feature = "backend-mixture")] + Self::Mixture(m) => m.observe_bit_msb(bit_idx, bit), + _ => bail!("native recursive bitwise stepping is unavailable for this predictor"), + } + } + + fn finish_native_recursive_bitwise_byte_step(&mut self, symbol: u8) -> Result<()> { + match self { + #[cfg(feature = "backend-ctw")] + Self::Ctw(_) | Self::FacCtw(_) => Ok(()), + #[cfg(feature = "backend-mixture")] + Self::Mixture(m) => m.finish_bitwise_symbol(symbol), + _ => bail!("native recursive bitwise stepping is unavailable for this predictor"), + } + } + + #[inline] + fn can_fast_ac_bitwise(&self) -> bool { + self.has_recursive_native_bitwise_path() + } + + // Keep this separate from the live AC payload path so framed AC preserves + // the v1 wire contract while still allowing backend-agnostic bitwise + // stepping as an internal utility. + fn ac_step_bitwise(&mut self, mut choose_bit: F) -> Result where F: FnMut(usize, f64) -> Result, { - match self { - Self::Ctw(m) => ctw_ac_step_bitwise(m, choose_bit), - Self::Mixture(m) => m.ac_step_bitwise(choose_bit), - _ => unreachable!("fast bitwise path requested for unsupported predictor"), + let mut state = PredictorBitwiseStepState::default(); + state.prepare(self)?; + let mut symbol = 0u8; + for bit_idx in 0..8usize { + let p1 = state.bit_prob_one_msb(self, bit_idx)?; + let bit = choose_bit(bit_idx, p1)? & 1; + if bit == 1 { + symbol |= 1u8 << (7 - bit_idx); + } + state.observe_bit_msb(self, bit_idx, bit == 1)?; } + state.finish_symbol(self, symbol)?; + Ok(symbol) + } + + fn ac_step_fast_bitwise(&mut self, choose_bit: F) -> Result + where + F: FnMut(usize, f64) -> Result, + { + debug_assert!(self.can_fast_ac_bitwise()); + self.ac_step_bitwise(choose_bit) } fn diagnostic_snapshot_subtree( @@ -1720,6 +1897,7 @@ impl RatePdfPredictor { pool: Option<&ThreadPool>, ) -> Result { match self { + #[cfg(feature = "backend-mixture")] Self::Mixture(m) => { m.diagnostic_subtree_snapshot(symbol, local_weight, effective_weight, pool) } @@ -1737,6 +1915,9 @@ impl RatePdfPredictor { } } + // The mixture implementation mutates the Vec allocation; no-mixture builds + // only see this forwarding signature and would otherwise flag it as `ptr_arg`. + #[allow(clippy::ptr_arg)] fn diagnostic_root_snapshot( &mut self, symbol: u8, @@ -1744,6 +1925,7 @@ impl RatePdfPredictor { out: &mut Vec, ) -> Result { match self { + #[cfg(feature = "backend-mixture")] Self::Mixture(m) => m.diagnostic_root_snapshot(symbol, pool, out), _ => anyhow::bail!("AC log-loss diagnostics require a top-level mixture backend"), } @@ -1753,6 +1935,7 @@ impl RatePdfPredictor { &mut self, symbol: u8, encoder: &mut ArithmeticEncoder, + cdf: &mut [u32; 257], ) -> Result<()> { if self.can_fast_ac_bitwise() { self.ac_step_fast_bitwise(|bit_idx, p1_mix| { @@ -1769,9 +1952,10 @@ impl RatePdfPredictor { } let pdf = self.pdf_next()?; - let mut cdf = vec![0u32; 257]; crate::coders::quantize_pdf_to_integer_cdf_dense_positive_with_buffer( - pdf, CDF_TOTAL, &mut cdf, + pdf, + CDF_TOTAL, + cdf.as_mut_slice(), ); let sym = symbol as usize; encoder.encode_counts(cdf[sym] as u64, cdf[sym + 1] as u64, CDF_TOTAL as u64)?; @@ -1779,21 +1963,6 @@ impl RatePdfPredictor { } } -fn ctw_ac_step_bitwise(ctw: &mut CtwPredictor, mut choose_bit: F) -> Result -where - F: FnMut(usize, f64) -> Result, -{ - debug_assert!(ctw.can_fast_ac_bitwise()); - let mut symbol = 0u8; - for bit_idx in 0..8usize { - let p1 = ctw.bit_prob_one_msb(bit_idx); - let bit = choose_bit(bit_idx, p1)? & 1; - symbol |= bit << (7 - bit_idx); - ctw.update_bit_msb(bit_idx, bit == 1); - } - Ok(symbol) -} - #[inline] fn binary_split_from_prob_one(p1: f64) -> u32 { let p1 = p1.clamp(PDF_MIN, 1.0 - PDF_MIN); @@ -1807,13 +1976,83 @@ fn binary_split_from_prob_one(p1: f64) -> u32 { split } +/// This function should be considered when fine-tuning Compression/decompression for a particular runtime case. In particular, my benchmarking has shown that inlining is non-obvious in how it affects performance +/// Inlining both encode and decode seems to cause performance issues with Match+AC decompression specifically, hence the odd configuration here for balance. +/// Encode default: inline +/// Technical note: this fast-path preserves the same bit ordering and CDF split mapping as the generic AC path (MSB-first with `binary_split_from_prob_one`). +#[cfg_attr(not(infotheory_ac_encode_deinline), inline(always))] +#[cfg_attr(infotheory_ac_encode_deinline, inline(never))] +fn encode_payload_ac_fast_bitwise( + data: &[u8], + predictor: &mut RatePdfPredictor, +) -> Result> { + let mut out = Vec::new(); + { + let mut enc = ArithmeticEncoder::new(&mut out); + for &symbol in data { + predictor.ac_step_fast_bitwise(|bit_idx, p1_mix| { + let bit = (symbol >> (7 - bit_idx)) & 1; + let split = binary_split_from_prob_one(p1_mix); + if bit == 0 { + enc.encode_counts(0, split as u64, CDF_TOTAL as u64)?; + } else { + enc.encode_counts(split as u64, CDF_TOTAL as u64, CDF_TOTAL as u64)?; + } + Ok(bit) + })?; + } + let _ = enc.finish()?; + } + Ok(out) +} + +/// This function should be considered when fine-tuning Compression/decompression for a particular runtime case. In particular, my benchmarking has shown that inlining is non-obvious in how it affects performance +/// Inlining both encode and decode seems to cause performance issues with Match+AC decompression specifically, hence the odd configuration here for balance. +/// Decode default: deinline +/// Technical note: this decodes exactly `out_len` symbols from the same binary CDF domain (`CDF_TOTAL`) used by the paired encode fast-path. +#[cfg_attr(infotheory_ac_decode_inline, inline(always))] +#[cfg_attr(not(infotheory_ac_decode_inline), inline(never))] +fn decode_payload_ac_fast_bitwise( + payload: &[u8], + out_len: usize, + predictor: &mut RatePdfPredictor, +) -> Result> { + let mut dec = ArithmeticDecoder::new(payload)?; + let mut out = Vec::with_capacity(out_len); + for _ in 0..out_len { + let symbol = predictor.ac_step_fast_bitwise(|_, p1_mix| { + let split = binary_split_from_prob_one(p1_mix); + dec.decode_binary_counts(split, CDF_TOTAL) + })?; + out.push(symbol); + } + Ok(out) +} + fn encode_payload_ac(data: &[u8], predictor: &mut RatePdfPredictor) -> Result> { predictor.begin_stream(data.len())?; + + if predictor.can_fast_ac_bitwise() { + let out = encode_payload_ac_fast_bitwise(data, predictor)?; + predictor.finish_stream()?; + return Ok(out); + } + let mut out = Vec::new(); { let mut enc = ArithmeticEncoder::new(&mut out); + // Reuse one CDF scratch buffer for the full stream to avoid per-symbol allocation. + let mut cdf = [0u32; 257]; for &symbol in data { - predictor.encode_symbol_ac_step(symbol, &mut enc)?; + let pdf = predictor.pdf_next()?; + crate::coders::quantize_pdf_to_integer_cdf_dense_positive_with_buffer( + pdf, + CDF_TOTAL, + cdf.as_mut_slice(), + ); + let sym = symbol as usize; + enc.encode_counts(cdf[sym] as u64, cdf[sym + 1] as u64, CDF_TOTAL as u64)?; + predictor.update(symbol)?; } let _ = enc.finish()?; } @@ -1827,17 +2066,9 @@ fn decode_payload_ac( predictor: &mut RatePdfPredictor, ) -> Result> { predictor.begin_stream(out_len)?; + if predictor.can_fast_ac_bitwise() { - let mut dec = ArithmeticDecoder::new(payload)?; - let mut out = Vec::with_capacity(out_len); - for _ in 0..out_len { - let symbol = predictor.ac_step_fast_bitwise(|_, p1_mix| { - let split = binary_split_from_prob_one(p1_mix); - let cdf = [0u32, split, CDF_TOTAL]; - Ok(dec.decode_symbol_counts(&cdf, CDF_TOTAL)? as u8) - })?; - out.push(symbol); - } + let out = decode_payload_ac_fast_bitwise(payload, out_len, predictor)?; predictor.finish_stream()?; return Ok(out); } @@ -1935,12 +2166,11 @@ fn decode_payload_rans( /// with payload metadata and CRC for safer transport/storage. pub fn compress_rate_bytes( data: &[u8], - rate_backend: &RateBackend, - max_order: i64, + rate_backend: &CompiledRateBackend, coder: CoderType, framing: FramingMode, ) -> Result> { - let mut predictor = RatePdfPredictor::from_rate_backend(rate_backend.clone(), max_order)?; + let mut predictor = crate::runtime::build_rate_pdf_predictor(rate_backend)?; let payload = match coder { CoderType::AC => encode_payload_ac(data, &mut predictor)?, CoderType::RANS => encode_payload_rans(data, &mut predictor)?, @@ -1960,20 +2190,18 @@ pub fn compress_rate_bytes( /// Return compressed size (in bytes) for `data` using rate coding. pub fn compress_rate_size( data: &[u8], - rate_backend: &RateBackend, - max_order: i64, + rate_backend: &CompiledRateBackend, coder: CoderType, framing: FramingMode, ) -> Result { - let encoded = compress_rate_bytes(data, rate_backend, max_order, coder, framing)?; + let encoded = compress_rate_bytes(data, rate_backend, coder, framing)?; Ok(encoded.len() as u64) } /// Return compressed size (in bytes) for concatenated slices under one stream. pub fn compress_rate_size_chain( parts: &[&[u8]], - rate_backend: &RateBackend, - max_order: i64, + rate_backend: &CompiledRateBackend, coder: CoderType, framing: FramingMode, ) -> Result { @@ -1982,14 +2210,13 @@ pub fn compress_rate_size_chain( for p in parts { data.extend_from_slice(p); } - compress_rate_size(&data, rate_backend, max_order, coder, framing) + compress_rate_size(&data, rate_backend, coder, framing) } /// Decompress bytes produced by [`compress_rate_bytes`]. pub fn decompress_rate_bytes( input: &[u8], - rate_backend: &RateBackend, - max_order: i64, + rate_backend: &CompiledRateBackend, _coder: CoderType, framing: FramingMode, ) -> Result> { @@ -2006,7 +2233,7 @@ pub fn decompress_rate_bytes( }; let _ = coder; - let mut predictor = RatePdfPredictor::from_rate_backend(rate_backend.clone(), max_order)?; + let mut predictor = crate::runtime::build_rate_pdf_predictor(rate_backend)?; let decoded = match coder { CoderType::AC => decode_payload_ac(payload, out_len, &mut predictor)?, CoderType::RANS => decode_payload_rans(payload, out_len, &mut predictor)?, @@ -2065,7 +2292,7 @@ fn build_cdf_row_from_pdf_slice(pdf: &[f64], cdf: &mut [f64; 257]) { } } -fn normalize_pdf_vec_and_maybe_build_cdf(pdf: &mut [f64], mut cdf: Option<&mut [f64; 257]>) { +fn normalize_pdf_vec_and_maybe_build_cdf(pdf: &mut [f64], cdf: Option<&mut [f64; 257]>) { let mut sum = 0.0; for p in pdf.iter_mut() { *p = if p.is_finite() { @@ -2078,13 +2305,13 @@ fn normalize_pdf_vec_and_maybe_build_cdf(pdf: &mut [f64], mut cdf: Option<&mut [ if !(sum.is_finite()) || sum <= 0.0 { let u = 1.0 / (pdf.len() as f64); pdf.fill(u); - if let Some(cdf) = cdf.as_deref_mut() { + if let Some(cdf) = cdf { *cdf = uniform_cdf_row(); } return; } let inv = 1.0 / sum; - if let Some(cdf) = cdf.as_deref_mut() { + if let Some(cdf) = cdf { cdf[0] = 0.0; let mut acc = 0.0; for i in 0..256 { @@ -2099,14 +2326,52 @@ fn normalize_pdf_vec_and_maybe_build_cdf(pdf: &mut [f64], mut cdf: Option<&mut [ } } +/// Build a 257-slot MSB-prefix CDF row from a 256-symbol PDF. +/// +/// Predictors must emit valid PDFs; this wrapper does not repair invalid totals in +/// release builds. Invalid rows are caught via `debug_assert` in debug builds only. #[inline] -fn cdf_bit_prob_one_msb(cdf: &[f64; 257], lo: usize, hi: usize) -> f64 { +fn rebuild_bitwise_prefix_cdf_row(cdf: &mut Vec, pdf: &[f64]) { + debug_assert_eq!( + pdf.len(), + 256, + "rebuild_bitwise_prefix_cdf_row requires a full 256-element PDF (caller invariant)" + ); + debug_assert!( + pdf.iter().all(|&p| p.is_finite() && p >= 0.0), + "Predictor contract violation: predictor emitted non-finite or negative PDF mass" + ); + cdf.resize(257, 0.0); + cdf[0] = 0.0; + for idx in 0..256usize { + let p: f64 = pdf[idx].max(PDF_MIN); // direct index per invariant + cdf[idx + 1] = cdf[idx] + p; + } + debug_assert!( + cdf[256].is_finite() && cdf[256] > 0.0, + "Predictor contract violation: invalid prefix-CDF total ({})", + cdf[256] + ); +} + +#[inline] +fn cdf_bit_prob_one_msb(cdf: &[f64], lo: usize, hi: usize) -> f64 { let mid = (lo + hi) >> 1; let total = (cdf[hi] - cdf[lo]).max(PDF_MIN); let one = (cdf[hi] - cdf[mid]).max(0.0); (one / total).clamp(PDF_MIN, 1.0 - PDF_MIN) } +#[inline] +fn advance_msb_prefix_range(lo: &mut usize, hi: &mut usize, bit: bool) { + let mid = (*lo + *hi) >> 1; + if bit { + *lo = mid; + } else { + *hi = mid; + } +} + #[inline] fn logsumexp_slice(vals: &[f64]) -> f64 { let mut m = f64::NEG_INFINITY; @@ -2244,13 +2509,80 @@ fn apply_switching_weights( } #[allow(dead_code)] +#[cfg(feature = "backend-zpaq")] fn _zpaq_marker(_: &ZpaqRateModel) {} #[cfg(test)] +mod rebuild_bitwise_prefix_cdf_row_contract_tests { + use super::rebuild_bitwise_prefix_cdf_row; + + #[test] + #[cfg(debug_assertions)] + fn rebuild_bitwise_prefix_cdf_row_rejects_nan_pdf_in_debug() { + let mut pdf = [1.0 / 256.0; 256]; + pdf[17] = f64::NAN; + + let result = std::panic::catch_unwind(|| { + let mut cdf: Vec = Vec::new(); + rebuild_bitwise_prefix_cdf_row(&mut cdf, &pdf); + }); + + assert!(result.is_err()); + } + + #[test] + #[cfg(debug_assertions)] + fn rebuild_bitwise_prefix_cdf_row_rejects_negative_pdf_in_debug() { + let mut pdf = [1.0 / 256.0; 256]; + pdf[17] = -0.1; + + let result = std::panic::catch_unwind(|| { + let mut cdf: Vec = Vec::new(); + rebuild_bitwise_prefix_cdf_row(&mut cdf, &pdf); + }); + + assert!(result.is_err()); + } +} + +#[cfg(all(test, feature = "all-backends"))] mod tests { use super::*; use std::sync::Arc; + fn compiled_rate_backend(backend: &RateBackend) -> CompiledRateBackend { + backend + .compile() + .unwrap_or_else(|err| panic!("failed to compile rate backend for test: {err}")) + } + + fn compress_rate_bytes( + data: &[u8], + rate_backend: &RateBackend, + coder: CoderType, + framing: FramingMode, + ) -> Result> { + super::compress_rate_bytes(data, &compiled_rate_backend(rate_backend), coder, framing) + } + + fn compress_rate_size( + data: &[u8], + rate_backend: &RateBackend, + coder: CoderType, + framing: FramingMode, + ) -> Result { + super::compress_rate_size(data, &compiled_rate_backend(rate_backend), coder, framing) + } + + fn decompress_rate_bytes( + input: &[u8], + rate_backend: &RateBackend, + coder: CoderType, + framing: FramingMode, + ) -> Result> { + super::decompress_rate_bytes(input, &compiled_rate_backend(rate_backend), coder, framing) + } + fn assert_pdf_close(lhs: &[f64], rhs: &[f64], tol: f64) { assert_eq!(lhs.len(), rhs.len()); for (idx, (&a, &b)) in lhs.iter().zip(rhs.iter()).enumerate() { @@ -2320,7 +2652,7 @@ mod tests { #[test] fn fac_pdf_fast_matches_bruteforce_subbyte() { - let mut predictor = CtwPredictor::new_fac(5, 5); + let mut predictor = CtwPredictor::new_fac(5, 5, None); for &b in b"fac ctw subbyte regression corpus abcdefghijklmnopqrstuvwxyz" { predictor.update(b); } @@ -2340,37 +2672,47 @@ mod tests { } } + #[test] + fn fac_ctw_default_bit_order_is_byte_msb_and_subbyte_lsb() { + let byte_default = CtwPredictor::new_fac(5, 8, None); + assert!( + byte_default.can_fast_ac_bitwise(), + "8-bit FacCtw without explicit order should use MSB-first native bitwise path" + ); + + let subbyte_default = CtwPredictor::new_fac(5, 5, None); + assert!( + !subbyte_default.can_fast_ac_bitwise(), + "non-byte FacCtw without explicit order keeps legacy LSB-first behavior" + ); + + let explicit_lsb = CtwPredictor::new_fac(5, 8, Some(false)); + assert!( + !explicit_lsb.can_fast_ac_bitwise(), + "explicit msb_first=false must preserve legacy LSB-first behavior" + ); + + let explicit_subbyte_msb = CtwPredictor::new_fac(5, 5, Some(true)); + assert!( + !explicit_subbyte_msb.can_fast_ac_bitwise(), + "subbyte widths do not use byte-packed native fast path even when MSB-first" + ); + } + fn assert_ctw_pdf_next_preserves_state(mut predictor: CtwPredictor) { for &b in b"ctw predictor state preservation payload" { predictor.update(b); } - let mut before_p0 = [0.0f64; 8]; - let mut before_p1 = [0.0f64; 8]; - for bit_idx in 0..8usize { - before_p0[bit_idx] = predictor.tree.predict(false, bit_idx); - before_p1[bit_idx] = predictor.tree.predict(true, bit_idx); + let mut baseline = [0.0f64; 256]; + for (sym, slot) in baseline.iter_mut().enumerate() { + *slot = predictor.log_prob_symbol_bruteforce(sym as u8); } - let log_before = predictor.tree.get_log_block_probability(); let _ = predictor.pdf_next(); - let log_after = predictor.tree.get_log_block_probability(); - assert!( - (log_before - log_after).abs() < 1e-12, - "log drift: before={log_before} after={log_after}" - ); - for bit_idx in 0..8usize { - let after_p0 = predictor.tree.predict(false, bit_idx); - let after_p1 = predictor.tree.predict(true, bit_idx); - assert!( - (before_p0[bit_idx] - after_p0).abs() < 1e-12, - "bit {bit_idx} p0 drift: {} vs {}", - before_p0[bit_idx], - after_p0 - ); + for (sym, &expected) in baseline.iter().enumerate() { + let after = predictor.log_prob_symbol_bruteforce(sym as u8); assert!( - (before_p1[bit_idx] - after_p1).abs() < 1e-12, - "bit {bit_idx} p1 drift: {} vs {}", - before_p1[bit_idx], - after_p1 + (expected - after).abs() < 1e-12, + "symbol {sym} drift: {expected} vs {after}" ); } } @@ -2382,7 +2724,7 @@ mod tests { #[test] fn fac_pdf_next_preserves_state() { - assert_ctw_pdf_next_preserves_state(CtwPredictor::new_fac(7, 8)); + assert_ctw_pdf_next_preserves_state(CtwPredictor::new_fac(7, 8, None)); } fn assert_fill_pattern_preserves_symbol_log_probs(mut predictor: CtwPredictor) { @@ -2411,7 +2753,7 @@ mod tests { #[test] fn fac_fill_pattern_preserves_symbol_log_probs() { - assert_fill_pattern_preserves_symbol_log_probs(CtwPredictor::new_fac(7, 8)); + assert_fill_pattern_preserves_symbol_log_probs(CtwPredictor::new_fac(7, 8, None)); } fn assert_pdf_then_update_matches_plain_update(mut base: CtwPredictor) { @@ -2444,17 +2786,16 @@ mod tests { #[test] fn fac_pdf_then_update_matches_plain_update() { - assert_pdf_then_update_matches_plain_update(CtwPredictor::new_fac(7, 8)); + assert_pdf_then_update_matches_plain_update(CtwPredictor::new_fac(7, 8, None)); } #[test] fn roundtrip_rate_ac_ctw() { let data = b"ctw backend roundtrip payload"; let backend = RateBackend::Ctw { depth: 8 }; - let enc = - compress_rate_bytes(data, &backend, -1, CoderType::AC, FramingMode::Framed).unwrap(); + let enc = compress_rate_bytes(data, &backend, CoderType::AC, FramingMode::Framed).unwrap(); let dec = - decompress_rate_bytes(&enc, &backend, -1, CoderType::AC, FramingMode::Framed).unwrap(); + decompress_rate_bytes(&enc, &backend, CoderType::AC, FramingMode::Framed).unwrap(); assert_eq!(dec, data); } @@ -2483,17 +2824,63 @@ mod tests { memory_mb: 8, }, ] { - let enc = compress_rate_bytes(data, &backend, -1, CoderType::AC, FramingMode::Framed) - .unwrap(); - let dec = decompress_rate_bytes(&enc, &backend, -1, CoderType::AC, FramingMode::Framed) - .unwrap(); + let enc = + compress_rate_bytes(data, &backend, CoderType::AC, FramingMode::Framed).unwrap(); + let dec = + decompress_rate_bytes(&enc, &backend, CoderType::AC, FramingMode::Framed).unwrap(); assert_eq!(dec, data); } } + #[test] + fn framed_rate_ac_keeps_v1_coder_byte_for_ctw() { + let data = b"legacy framed ac header payload"; + let backend = RateBackend::Ctw { depth: 8 }; + let enc = compress_rate_bytes(data, &backend, CoderType::AC, FramingMode::Framed).unwrap(); + let hdr = FramedHeader::read(&enc).expect("framed header"); + assert_eq!(hdr.coder_type(), CoderType::AC); + assert_eq!(hdr.coder, 0); + let dec = + decompress_rate_bytes(&enc, &backend, CoderType::AC, FramingMode::Framed).unwrap(); + assert_eq!(dec, data); + } + + #[test] + fn framed_rate_rans_keeps_v1_coder_byte() { + let data = b"legacy framed rans header payload"; + let backend = RateBackend::Ctw { depth: 8 }; + let enc = + compress_rate_bytes(data, &backend, CoderType::RANS, FramingMode::Framed).unwrap(); + let hdr = FramedHeader::read(&enc).expect("framed header"); + assert_eq!(hdr.coder_type(), CoderType::RANS); + assert_eq!(hdr.coder, 1); + let dec = + decompress_rate_bytes(&enc, &backend, CoderType::RANS, FramingMode::Framed).unwrap(); + assert_eq!(dec, data); + } + + #[test] + fn framed_rate_ac_keeps_byte_prefix_models_on_legacy_path() { + let data = b"byte prefix adapter exists but byte ac remains default"; + let backend = RateBackend::Match { + hash_bits: 20, + min_len: 4, + max_len: 255, + base_mix: 0.02, + confidence_scale: 1.0, + }; + let enc = compress_rate_bytes(data, &backend, CoderType::AC, FramingMode::Framed).unwrap(); + let hdr = FramedHeader::read(&enc).expect("framed header"); + assert_eq!(hdr.coder_type(), CoderType::AC); + assert_eq!(hdr.coder, 0); + let dec = + decompress_rate_bytes(&enc, &backend, CoderType::AC, FramingMode::Framed).unwrap(); + assert_eq!(dec, data); + } + #[test] fn roundtrip_rate_ac_ppmd_high_order_text_payload() { - let seed = include_bytes!("../../README.md"); + let seed = include_bytes!("../../../../README.md"); let mut data = Vec::with_capacity(4096); while data.len() < 4096 { data.extend_from_slice(seed); @@ -2504,9 +2891,9 @@ mod tests { order: 12, memory_mb: 256, }; - let enc = compress_rate_bytes(&data, &backend, -1, CoderType::AC, FramingMode::Framed) + let enc = compress_rate_bytes(&data, &backend, CoderType::AC, FramingMode::Framed) .expect("ppmd high-order compression"); - let dec = decompress_rate_bytes(&enc, &backend, -1, CoderType::AC, FramingMode::Framed) + let dec = decompress_rate_bytes(&enc, &backend, CoderType::AC, FramingMode::Framed) .expect("ppmd high-order decompression"); assert_eq!(dec, data); } @@ -2523,10 +2910,9 @@ mod tests { bias_clip: 4.0, }), }; - let enc = - compress_rate_bytes(data, &backend, -1, CoderType::AC, FramingMode::Framed).unwrap(); + let enc = compress_rate_bytes(data, &backend, CoderType::AC, FramingMode::Framed).unwrap(); let dec = - decompress_rate_bytes(&enc, &backend, -1, CoderType::AC, FramingMode::Framed).unwrap(); + decompress_rate_bytes(&enc, &backend, CoderType::AC, FramingMode::Framed).unwrap(); assert_eq!(dec, data); } @@ -2538,7 +2924,6 @@ mod tests { vec![crate::MixtureExpertSpec { name: Some("ctw".to_string()), log_prior: 0.0, - max_order: -1, backend: RateBackend::Ctw { depth: 8 }, }], ) @@ -2546,10 +2931,9 @@ mod tests { let backend = RateBackend::Mixture { spec: Arc::new(spec), }; - let enc = - compress_rate_bytes(data, &backend, -1, CoderType::AC, FramingMode::Framed).unwrap(); + let enc = compress_rate_bytes(data, &backend, CoderType::AC, FramingMode::Framed).unwrap(); let dec = - decompress_rate_bytes(&enc, &backend, -1, CoderType::AC, FramingMode::Framed).unwrap(); + decompress_rate_bytes(&enc, &backend, CoderType::AC, FramingMode::Framed).unwrap(); assert_eq!(dec, data); } @@ -2561,7 +2945,6 @@ mod tests { vec![crate::MixtureExpertSpec { name: Some("ctw".to_string()), log_prior: 0.0, - max_order: -1, backend: RateBackend::Ctw { depth: 8 }, }], ) @@ -2569,10 +2952,9 @@ mod tests { let backend = RateBackend::Mixture { spec: Arc::new(spec), }; - let enc = - compress_rate_bytes(data, &backend, -1, CoderType::AC, FramingMode::Framed).unwrap(); + let enc = compress_rate_bytes(data, &backend, CoderType::AC, FramingMode::Framed).unwrap(); let dec = - decompress_rate_bytes(&enc, &backend, -1, CoderType::AC, FramingMode::Framed).unwrap(); + decompress_rate_bytes(&enc, &backend, CoderType::AC, FramingMode::Framed).unwrap(); assert_eq!(dec, data); } @@ -2585,17 +2967,16 @@ mod tests { crate::MixtureExpertSpec { name: Some("ctw".to_string()), log_prior: 0.0, - max_order: -1, backend: RateBackend::Ctw { depth: 6 }, }, crate::MixtureExpertSpec { name: Some("fac".to_string()), log_prior: 0.0, - max_order: -1, backend: RateBackend::FacCtw { base_depth: 6, num_percept_bits: 8, encoding_bits: 8, + msb_first: None, }, }, ], @@ -2606,7 +2987,6 @@ mod tests { crate::MixtureExpertSpec { name: Some("nested".to_string()), log_prior: 0.0, - max_order: -1, backend: RateBackend::Mixture { spec: Arc::new(nested), }, @@ -2614,9 +2994,8 @@ mod tests { crate::MixtureExpertSpec { name: Some("zpaq".to_string()), log_prior: 0.0, - max_order: -1, backend: RateBackend::Zpaq { - method: "1".to_string(), + method: crate::api::ZpaqMethodSpec::literal("1"), }, }, ], @@ -2627,9 +3006,9 @@ mod tests { spec: Arc::new(root), }; let enc = - compress_rate_bytes(data, &backend, -1, CoderType::RANS, FramingMode::Framed).unwrap(); - let dec = decompress_rate_bytes(&enc, &backend, -1, CoderType::RANS, FramingMode::Framed) - .unwrap(); + compress_rate_bytes(data, &backend, CoderType::RANS, FramingMode::Framed).unwrap(); + let dec = + decompress_rate_bytes(&enc, &backend, CoderType::RANS, FramingMode::Framed).unwrap(); assert_eq!(dec, data); } @@ -2642,17 +3021,16 @@ mod tests { crate::MixtureExpertSpec { name: Some("ctw".to_string()), log_prior: 0.0, - max_order: -1, backend: RateBackend::Ctw { depth: 6 }, }, crate::MixtureExpertSpec { name: Some("fac".to_string()), log_prior: 0.0, - max_order: -1, backend: RateBackend::FacCtw { base_depth: 6, num_percept_bits: 8, encoding_bits: 8, + msb_first: None, }, }, ], @@ -2663,7 +3041,6 @@ mod tests { crate::MixtureExpertSpec { name: Some("nested".to_string()), log_prior: 0.0, - max_order: -1, backend: RateBackend::Mixture { spec: Arc::new(inner), }, @@ -2671,9 +3048,8 @@ mod tests { crate::MixtureExpertSpec { name: Some("zpaq".to_string()), log_prior: 0.0, - max_order: -1, backend: RateBackend::Zpaq { - method: "1".to_string(), + method: crate::api::ZpaqMethodSpec::literal("1"), }, }, ], @@ -2683,10 +3059,22 @@ mod tests { let backend = RateBackend::Mixture { spec: Arc::new(root), }; - let enc = - compress_rate_bytes(data, &backend, -1, CoderType::AC, FramingMode::Framed).unwrap(); + let enc = compress_rate_bytes(data, &backend, CoderType::AC, FramingMode::Framed).unwrap(); let dec = - decompress_rate_bytes(&enc, &backend, -1, CoderType::AC, FramingMode::Framed).unwrap(); + decompress_rate_bytes(&enc, &backend, CoderType::AC, FramingMode::Framed).unwrap(); + assert_eq!(dec, data); + } + + #[test] + fn roundtrip_rate_ac_recursive_native_bitwise_mixture() { + let data = b"recursive native bitwise mixture payload"; + let backend = recursive_native_bitwise_backend(); + let predictor = RatePdfPredictor::from_rate_backend(backend.clone()).unwrap(); + assert!(predictor.can_fast_ac_bitwise()); + + let enc = compress_rate_bytes(data, &backend, CoderType::AC, FramingMode::Framed).unwrap(); + let dec = + decompress_rate_bytes(&enc, &backend, CoderType::AC, FramingMode::Framed).unwrap(); assert_eq!(dec, data); } @@ -2694,7 +3082,7 @@ mod tests { let backend = RateBackend::Mixture { spec: Arc::new(spec.clone()), }; - let mut predictor = RatePdfPredictor::from_rate_backend(backend, -1).unwrap(); + let mut predictor = RatePdfPredictor::from_rate_backend(backend).unwrap(); let experts = spec.build_experts(); let mut runtime = crate::mixture::build_mixture_runtime(&spec, &experts).unwrap(); @@ -2716,17 +3104,16 @@ mod tests { crate::MixtureExpertSpec { name: Some("ctw".to_string()), log_prior: 0.0, - max_order: -1, backend: RateBackend::Ctw { depth: 7 }, }, crate::MixtureExpertSpec { name: Some("fac".to_string()), log_prior: -0.7, - max_order: -1, backend: RateBackend::FacCtw { base_depth: 7, num_percept_bits: 8, encoding_bits: 8, + msb_first: None, }, }, ] @@ -2821,7 +3208,6 @@ mod tests { crate::MixtureExpertSpec { name: Some("nested".to_string()), log_prior: 0.0, - max_order: -1, backend: RateBackend::Mixture { spec: Arc::new(nested), }, @@ -2829,7 +3215,6 @@ mod tests { crate::MixtureExpertSpec { name: Some("ppmd".to_string()), log_prior: -0.2, - max_order: -1, backend: RateBackend::Ppmd { order: 5, memory_mb: 8, @@ -2845,6 +3230,183 @@ mod tests { ); } + fn recursive_native_bitwise_backend() -> RateBackend { + let nested = MixtureSpec::new(MixtureKind::Bayes, alignment_experts()); + let root = MixtureSpec::new( + MixtureKind::Switching, + vec![ + crate::MixtureExpertSpec { + name: Some("nested".to_string()), + log_prior: 0.0, + backend: RateBackend::Mixture { + spec: Arc::new(nested), + }, + }, + crate::MixtureExpertSpec { + name: Some("ppmd".to_string()), + log_prior: -0.2, + backend: RateBackend::Ppmd { + order: 5, + memory_mb: 8, + }, + }, + ], + ) + .with_alpha(0.13); + RateBackend::Mixture { + spec: Arc::new(root), + } + } + + fn assert_bitwise_byte_step_matches_pdf_and_plain_update( + mut predictor: RatePdfPredictor, + data: &[u8], + tol_prob: f64, + tol_pdf: f64, + ) { + for &symbol in data { + let expected_pdf = predictor.pdf_next().unwrap().to_vec(); + let expected_prob = expected_pdf[symbol as usize]; + + let mut stepped = predictor.clone(); + let mut product = 1.0f64; + let produced = stepped + .ac_step_bitwise(|bit_idx, p1| { + let bit = (symbol >> (7 - bit_idx)) & 1; + let pb = if bit == 1 { p1 } else { 1.0 - p1 }; + product *= pb; + Ok(bit) + }) + .unwrap(); + assert_eq!(produced, symbol); + assert!( + (product - expected_prob).abs() <= tol_prob, + "symbol={symbol} product={product} expected_prob={expected_prob}" + ); + + let mut plain = predictor.clone(); + plain.update(symbol).unwrap(); + let stepped_pdf = stepped.pdf_next().unwrap().to_vec(); + let plain_pdf = plain.pdf_next().unwrap().to_vec(); + assert_pdf_close(&stepped_pdf, &plain_pdf, tol_pdf); + + predictor.update(symbol).unwrap(); + } + } + + #[test] + fn bitwise_byte_step_matches_pdf_and_plain_update_for_native_and_recursive_mixtures() { + assert_bitwise_byte_step_matches_pdf_and_plain_update( + RatePdfPredictor::from_rate_backend(RateBackend::Ctw { depth: 7 }).unwrap(), + b"direct ctw bitwise byte step parity", + 1e-12, + 1e-12, + ); + + let direct_fac = RateBackend::FacCtw { + base_depth: 7, + num_percept_bits: 8, + encoding_bits: 8, + msb_first: None, + }; + let direct_fac_predictor = RatePdfPredictor::from_rate_backend(direct_fac).unwrap(); + assert!( + direct_fac_predictor.can_fast_ac_bitwise(), + "byte-wide MSB fac-ctw must expose its native recursive AC path" + ); + assert_bitwise_byte_step_matches_pdf_and_plain_update( + direct_fac_predictor, + b"direct fac ctw bitwise byte step parity", + 1e-12, + 1e-12, + ); + + let single_expert = RateBackend::Mixture { + spec: Arc::new(MixtureSpec::new( + MixtureKind::Bayes, + vec![crate::MixtureExpertSpec { + name: Some("ctw".to_string()), + log_prior: 0.0, + backend: RateBackend::Ctw { depth: 7 }, + }], + )), + }; + assert_bitwise_byte_step_matches_pdf_and_plain_update( + RatePdfPredictor::from_rate_backend(single_expert).unwrap(), + b"single expert ctw mixture bitwise byte step parity", + 1e-12, + 1e-12, + ); + + let single_fac_neural = RateBackend::Mixture { + spec: Arc::new(MixtureSpec::new( + MixtureKind::Neural, + vec![crate::MixtureExpertSpec { + name: Some("fac-ctw".to_string()), + log_prior: 0.0, + backend: RateBackend::FacCtw { + base_depth: 7, + num_percept_bits: 8, + encoding_bits: 8, + msb_first: None, + }, + }], + )), + }; + let single_fac_neural_predictor = + RatePdfPredictor::from_rate_backend(single_fac_neural).unwrap(); + assert!( + single_fac_neural_predictor.can_fast_ac_bitwise(), + "neural mixtures containing only byte-wide MSB fac-ctw must not fall back to PDF-prefix AC" + ); + assert_bitwise_byte_step_matches_pdf_and_plain_update( + single_fac_neural_predictor, + b"single expert fac neural mixture bitwise byte step parity", + 1e-12, + 1e-12, + ); + + let mixed_direct = RateBackend::Mixture { + spec: Arc::new(MixtureSpec::new( + MixtureKind::Bayes, + vec![ + crate::MixtureExpertSpec { + name: Some("ctw".to_string()), + log_prior: 0.0, + backend: RateBackend::Ctw { depth: 7 }, + }, + crate::MixtureExpertSpec { + name: Some("match".to_string()), + log_prior: -0.3, + backend: RateBackend::Match { + hash_bits: 20, + min_len: 4, + max_len: 255, + base_mix: 0.02, + confidence_scale: 1.0, + }, + }, + ], + )), + }; + assert_bitwise_byte_step_matches_pdf_and_plain_update( + RatePdfPredictor::from_rate_backend(mixed_direct).unwrap(), + b"mixed direct mixture bitwise byte step parity", + 1e-11, + 1e-11, + ); + + let recursive = recursive_native_bitwise_backend(); + let predictor = RatePdfPredictor::from_rate_backend(recursive).unwrap(); + assert!(predictor.can_fast_ac_bitwise()); + assert_bitwise_byte_step_matches_pdf_and_plain_update( + predictor, + b"recursive nested mixture bitwise byte step parity", + 1e-10, + 1e-10, + ); + } + fn assert_cached_cdf_fast_bitwise_matches_pdf_rows(mut predictor: RatePdfPredictor) { let data = b"cached cdf parity check payload"; for &symbol in data { @@ -2883,49 +3445,37 @@ mod tests { #[test] fn cached_cdf_fast_bitwise_matches_pdf_rows_for_specialized_predictors() { assert_cached_cdf_fast_bitwise_matches_pdf_rows( - RatePdfPredictor::from_rate_backend(RateBackend::RosaPlus, -1).unwrap(), + RatePdfPredictor::from_rate_backend(RateBackend::RosaPlus { max_order: -1 }).unwrap(), ); assert_cached_cdf_fast_bitwise_matches_pdf_rows( - RatePdfPredictor::from_rate_backend( - RateBackend::Ppmd { - order: 6, - memory_mb: 8, - }, - -1, - ) + RatePdfPredictor::from_rate_backend(RateBackend::Ppmd { + order: 6, + memory_mb: 8, + }) .unwrap(), ); assert_cached_cdf_fast_bitwise_matches_pdf_rows( - RatePdfPredictor::from_rate_backend( - RateBackend::Match { - hash_bits: 20, - min_len: 4, - max_len: 255, - base_mix: 0.02, - confidence_scale: 1.0, - }, - -1, - ) + RatePdfPredictor::from_rate_backend(RateBackend::Match { + hash_bits: 20, + min_len: 4, + max_len: 255, + base_mix: 0.02, + confidence_scale: 1.0, + }) .unwrap(), ); #[cfg(feature = "backend-rwkv")] assert_cached_cdf_fast_bitwise_matches_pdf_rows( - RatePdfPredictor::from_rate_backend( - RateBackend::Rwkv7Method { - method: "cfg:hidden=64,layers=1,intermediate=64,decay_rank=8,a_rank=8,v_rank=8,g_rank=8,seed=11,train=none,lr=0.0,stride=1;policy:schedule=0..100:infer".to_string(), - }, - -1, - ) + RatePdfPredictor::from_rate_backend(RateBackend::Rwkv7Method { + method: crate::rwkvzip::parse_method_spec("cfg:hidden=64,layers=1,intermediate=64,decay_rank=8,a_rank=8,v_rank=8,g_rank=8,seed=11,train=none,lr=0.0,stride=1;policy:schedule=0..100:infer").expect("rwkv method spec"), + }) .unwrap(), ); #[cfg(feature = "backend-mamba")] assert_cached_cdf_fast_bitwise_matches_pdf_rows( - RatePdfPredictor::from_rate_backend( - RateBackend::MambaMethod { - method: "cfg:hidden=64,layers=1,intermediate=64,state=8,conv=3,dt_rank=4,seed=7,train=none,lr=0.0,stride=1;policy:schedule=0..100:infer".to_string(), - }, - -1, - ) + RatePdfPredictor::from_rate_backend(RateBackend::MambaMethod { + method: crate::mambazip::parse_method_spec("cfg:hidden=64,layers=1,intermediate=64,state=8,conv=3,dt_rank=4,seed=7,train=none,lr=0.0,stride=1;policy:schedule=0..100:infer").expect("mamba method spec"), + }) .unwrap(), ); } @@ -2933,10 +3483,10 @@ mod tests { #[test] fn raw_size_not_larger_than_framed_size() { let data = b"raw/framed size check payload"; - let backend = RateBackend::RosaPlus; - let raw = compress_rate_size(data, &backend, 8, CoderType::AC, FramingMode::Raw).unwrap(); + let backend = RateBackend::RosaPlus { max_order: 8 }; + let raw = compress_rate_size(data, &backend, CoderType::AC, FramingMode::Raw).unwrap(); let framed = - compress_rate_size(data, &backend, 8, CoderType::AC, FramingMode::Framed).unwrap(); + compress_rate_size(data, &backend, CoderType::AC, FramingMode::Framed).unwrap(); assert!(framed >= raw); } @@ -2945,12 +3495,11 @@ mod tests { fn roundtrip_rate_rwkv_method_cfg() { let data = b"rwkv cfg method backend"; let backend = RateBackend::Rwkv7Method { - method: "cfg:hidden=64,layers=1,intermediate=64,decay_rank=8,a_rank=8,v_rank=8,g_rank=8,seed=11,train=none,lr=0.0,stride=1;policy:schedule=0..100:infer".to_string(), + method: crate::rwkvzip::parse_method_spec("cfg:hidden=64,layers=1,intermediate=64,decay_rank=8,a_rank=8,v_rank=8,g_rank=8,seed=11,train=none,lr=0.0,stride=1;policy:schedule=0..100:infer").expect("rwkv method spec"), }; - let enc = - compress_rate_bytes(data, &backend, -1, CoderType::AC, FramingMode::Framed).unwrap(); + let enc = compress_rate_bytes(data, &backend, CoderType::AC, FramingMode::Framed).unwrap(); let dec = - decompress_rate_bytes(&enc, &backend, -1, CoderType::AC, FramingMode::Framed).unwrap(); + decompress_rate_bytes(&enc, &backend, CoderType::AC, FramingMode::Framed).unwrap(); assert_eq!(dec, data); } @@ -2974,6 +3523,34 @@ mod tests { assert_pdf_close(predictor.pdf_next(), &direct, 1e-18); } + #[cfg(feature = "backend-rwkv")] + #[test] + fn compiled_rwkv_rate_pdf_predictor_preserves_backend_pdf_exactly() { + let method = "cfg:hidden=64,layers=1,intermediate=64,decay_rank=8,a_rank=8,v_rank=8,g_rank=8,seed=11,train=none,lr=0.0,stride=1;policy:schedule=0..100:infer"; + let backend = RateBackend::Rwkv7Method { + method: crate::rwkvzip::parse_method_spec(method).expect("rwkv method spec"), + } + .compile() + .expect("compiled rwkv backend"); + let spec = rwkvzip::parse_method_spec(method).expect("parsed rwkv spec"); + let mut predictor = + RatePdfPredictor::from_compiled(&backend).expect("compiled rwkv predictor"); + let mut direct = + rwkvzip::Compressor::new_from_method_spec(&spec).expect("rwkv backend from spec"); + let mut pdf = vec![0.0; direct.vocab_size()]; + + let predicted = predictor.pdf_next().expect("predictor pdf").to_vec(); + direct.forward_to_pdf(0, &mut pdf); + assert_pdf_close(&predicted, &pdf, 1e-18); + + predictor.update(b'x').expect("predictor update"); + direct + .online_update_from_pdf(b'x', &pdf) + .expect("backend update"); + direct.forward_to_pdf(u32::from(b'x'), &mut pdf); + assert_pdf_close(predictor.pdf_next().expect("predictor pdf"), &pdf, 1e-18); + } + #[cfg(feature = "backend-rwkv")] #[test] fn rwkv_rate_predictor_matches_backend_after_partial_tbptt_stream() { @@ -3014,18 +3591,21 @@ mod tests { #[test] fn roundtrip_rate_rwkv_two_json_method_2m() { let two_json: serde_json::Value = - serde_json::from_str(include_str!("../../examples/two.json")).unwrap(); - let method = two_json["experts"] + serde_json::from_str(include_str!("../../../../configs/bench/two.json")).unwrap(); + let experts = two_json["experts"] .as_array() - .unwrap() + .expect("two.json must define experts array"); + let method = experts .iter() - .find(|expert| expert["name"].as_str() == Some("rwkv")) + .find(|expert| expert["kind"].as_str() == Some("rwkv7")) .and_then(|expert| expert["method"].as_str()) - .unwrap() + .expect("two.json must include rwkv7 expert with string method") .to_string(); - let backend = RateBackend::Rwkv7Method { method }; - let seed = include_bytes!("../../README.md"); + let backend = RateBackend::Rwkv7Method { + method: crate::rwkvzip::parse_method_spec(&method).expect("rwkv method spec"), + }; + let seed = include_bytes!("../../../../README.md"); let target_len = 2_097_152usize; let mut data = Vec::with_capacity(target_len); while data.len() < target_len { @@ -3033,13 +3613,28 @@ mod tests { data.extend_from_slice(&seed[..seed.len().min(remaining)]); } - let enc = - compress_rate_bytes(&data, &backend, -1, CoderType::AC, FramingMode::Framed).unwrap(); + let enc = compress_rate_bytes(&data, &backend, CoderType::AC, FramingMode::Framed).unwrap(); let dec = - decompress_rate_bytes(&enc, &backend, -1, CoderType::AC, FramingMode::Framed).unwrap(); + decompress_rate_bytes(&enc, &backend, CoderType::AC, FramingMode::Framed).unwrap(); assert_eq!(dec, data); } + #[test] + fn benchmark_two_json_matches_examples_and_historical_alpha() { + let canonical: serde_json::Value = + serde_json::from_str(include_str!("../../../../configs/bench/two.json")).unwrap(); + let example: serde_json::Value = + serde_json::from_str(include_str!("../../../../examples/two.json")).unwrap(); + + assert_eq!(canonical, example, "benchmark specs drifted"); + assert_eq!(canonical["kind"].as_str(), Some("neural")); + let alpha = canonical["alpha"].as_f64().expect("neural alpha"); + assert!( + (alpha - 0.03).abs() <= 1e-12, + "expected historical neural alpha 0.03, got {alpha}" + ); + } + #[cfg(feature = "backend-mamba")] #[test] fn mamba_rate_predictor_preserves_backend_pdf_exactly() { @@ -3060,6 +3655,34 @@ mod tests { assert_pdf_close(predictor.pdf_next(), &direct, 1e-18); } + #[cfg(feature = "backend-mamba")] + #[test] + fn compiled_mamba_rate_pdf_predictor_preserves_backend_pdf_exactly() { + let method = "cfg:hidden=64,layers=1,intermediate=64,state=8,conv=3,dt_rank=4,seed=7,train=none,lr=0.0,stride=1;policy:schedule=0..100:infer"; + let backend = RateBackend::MambaMethod { + method: crate::mambazip::parse_method_spec(method).expect("mamba method spec"), + } + .compile() + .expect("compiled mamba backend"); + let spec = mambazip::parse_method_spec(method).expect("parsed mamba spec"); + let mut predictor = + RatePdfPredictor::from_compiled(&backend).expect("compiled mamba predictor"); + let mut direct = + mambazip::Compressor::new_from_method_spec(&spec).expect("mamba backend from spec"); + let mut pdf = vec![0.0; direct.vocab_size()]; + + let predicted = predictor.pdf_next().expect("predictor pdf").to_vec(); + direct.forward_to_pdf(0, &mut pdf); + assert_pdf_close(&predicted, &pdf, 1e-18); + + predictor.update(b'x').expect("predictor update"); + direct + .online_update_from_pdf(b'x', &pdf) + .expect("backend update"); + direct.forward_to_pdf(u32::from(b'x'), &mut pdf); + assert_pdf_close(predictor.pdf_next().expect("predictor pdf"), &pdf, 1e-18); + } + #[test] fn roundtrip_rate_ac_particle() { let spec = crate::ParticleSpec { @@ -3077,10 +3700,9 @@ mod tests { let backend = RateBackend::Particle { spec: Arc::new(spec), }; - let enc = - compress_rate_bytes(data, &backend, -1, CoderType::AC, FramingMode::Framed).unwrap(); + let enc = compress_rate_bytes(data, &backend, CoderType::AC, FramingMode::Framed).unwrap(); let dec = - decompress_rate_bytes(&enc, &backend, -1, CoderType::AC, FramingMode::Framed).unwrap(); + decompress_rate_bytes(&enc, &backend, CoderType::AC, FramingMode::Framed).unwrap(); assert_eq!(dec, data); } @@ -3102,9 +3724,9 @@ mod tests { spec: Arc::new(spec), }; let enc = - compress_rate_bytes(data, &backend, -1, CoderType::RANS, FramingMode::Framed).unwrap(); - let dec = decompress_rate_bytes(&enc, &backend, -1, CoderType::RANS, FramingMode::Framed) - .unwrap(); + compress_rate_bytes(data, &backend, CoderType::RANS, FramingMode::Framed).unwrap(); + let dec = + decompress_rate_bytes(&enc, &backend, CoderType::RANS, FramingMode::Framed).unwrap(); assert_eq!(dec, data); } @@ -3127,7 +3749,6 @@ mod tests { crate::MixtureExpertSpec { name: Some("particle".to_string()), log_prior: 0.0, - max_order: -1, backend: RateBackend::Particle { spec: Arc::new(particle_spec), }, @@ -3135,7 +3756,6 @@ mod tests { crate::MixtureExpertSpec { name: Some("ctw".to_string()), log_prior: 0.0, - max_order: -1, backend: RateBackend::Ctw { depth: 6 }, }, ], @@ -3144,10 +3764,9 @@ mod tests { spec: Arc::new(spec), }; let data = b"mixture with particle expert roundtrip"; - let enc = - compress_rate_bytes(data, &backend, -1, CoderType::AC, FramingMode::Framed).unwrap(); + let enc = compress_rate_bytes(data, &backend, CoderType::AC, FramingMode::Framed).unwrap(); let dec = - decompress_rate_bytes(&enc, &backend, -1, CoderType::AC, FramingMode::Framed).unwrap(); + decompress_rate_bytes(&enc, &backend, CoderType::AC, FramingMode::Framed).unwrap(); assert_eq!(dec, data); } } diff --git a/src/datagen.rs b/crates/infotheory/src/datagen.rs similarity index 100% rename from src/datagen.rs rename to crates/infotheory/src/datagen.rs diff --git a/src/diagnostics.rs b/crates/infotheory/src/diagnostics.rs similarity index 80% rename from src/diagnostics.rs rename to crates/infotheory/src/diagnostics.rs index 144b8437..455ff8e5 100644 --- a/src/diagnostics.rs +++ b/crates/infotheory/src/diagnostics.rs @@ -2,9 +2,11 @@ use anyhow::{Context, Result, bail}; +use crate::api::{CompiledRateBackend, MixtureSpec, RateBackend}; +#[cfg(all(test, feature = "backend-mixture"))] +use crate::api::{MixtureExpertSpec, MixtureKind}; use crate::compression::{AcLogLossNodeValue, DiagnosticRatePredictor}; -use crate::mixture::RateBackendPredictor; -use crate::{MixtureExpertSpec, MixtureKind, MixtureSpec, RateBackend}; +use crate::spec::core::{RateBackendPlan, RateBackendPlanExpert, compiled_rate_backend_from_plan}; use std::env; use std::fs::File; use std::io::{BufWriter, Write}; @@ -138,76 +140,10 @@ fn bits_from_prob(prob: f64) -> f64 { -prob.max(crate::mixture::DEFAULT_MIN_PROB).log2() } -fn mixture_kind_label(kind: MixtureKind) -> &'static str { - match kind { - MixtureKind::Bayes => "mixture:bayes", - MixtureKind::FadingBayes => "mixture:fading-bayes", - MixtureKind::Switching => "mixture:switching", - MixtureKind::Convex => "mixture:convex", - MixtureKind::Mdl => "mixture:mdl", - MixtureKind::Neural => "mixture:neural", - } -} - -fn backend_label(expert: &MixtureExpertSpec) -> String { - match &expert.backend { - RateBackend::RosaPlus => format!("rosaplus(max_order={})", expert.max_order), - RateBackend::Match { - hash_bits, - min_len, - max_len, - base_mix, - confidence_scale, - } => format!( - "match(hash_bits={hash_bits},min_len={min_len},max_len={max_len},base_mix={base_mix},confidence_scale={confidence_scale})" - ), - RateBackend::SparseMatch { - hash_bits, - min_len, - max_len, - gap_min, - gap_max, - base_mix, - confidence_scale, - } => format!( - "sparse-match(hash_bits={hash_bits},min_len={min_len},max_len={max_len},gap_min={gap_min},gap_max={gap_max},base_mix={base_mix},confidence_scale={confidence_scale})" - ), - RateBackend::Ppmd { order, memory_mb } => { - format!("ppmd(order={order},memory_mb={memory_mb})") - } - RateBackend::Sequitur { context_bytes } => { - format!("sequitur(context_bytes={context_bytes})") - } - RateBackend::Ctw { depth } => format!("ctw(depth={depth})"), - RateBackend::FacCtw { - base_depth, - num_percept_bits, - encoding_bits, - } => format!( - "fac-ctw(base_depth={base_depth},num_percept_bits={num_percept_bits},encoding_bits={encoding_bits})" - ), - #[cfg(feature = "backend-mamba")] - RateBackend::Mamba { .. } => "mamba".to_string(), - #[cfg(feature = "backend-mamba")] - RateBackend::MambaMethod { method } => format!("mamba(method={method})"), - #[cfg(feature = "backend-rwkv")] - RateBackend::Rwkv7 { .. } => "rwkv7".to_string(), - #[cfg(feature = "backend-rwkv")] - RateBackend::Rwkv7Method { method } => format!("rwkv7(method={method})"), - RateBackend::Zpaq { method } => format!("zpaq(method={method})"), - RateBackend::Mixture { spec } => mixture_kind_label(spec.kind).to_string(), - RateBackend::Particle { spec } => format!( - "particle(num_particles={},num_cells={})", - spec.num_particles, spec.num_cells - ), - RateBackend::Calibrated { spec } => format!( - "calibrated(context={:?},bins={},learning_rate={},bias_clip={})", - spec.context, spec.bins, spec.learning_rate, spec.bias_clip - ), - } -} - -fn flatten_mixture_spec(spec: &MixtureSpec) -> FlatSchema { +fn flatten_compiled_mixture(backend: &CompiledRateBackend) -> Result { + let RateBackendPlan::Mixture { experts, .. } = backend.plan() else { + unreachable!("compiled diagnostic root must be a mixture backend"); + }; let mut schema = FlatSchema { nodes: vec![FlatNodeMeta { id: 0, @@ -215,7 +151,7 @@ fn flatten_mixture_spec(spec: &MixtureSpec) -> FlatSchema { depth: 0, path: "0:root".to_string(), display_name: "root".to_string(), - backend_label: mixture_kind_label(spec.kind).to_string(), + backend_label: backend.display_label(), is_mixture: true, is_leaf: false, is_root_child: false, @@ -223,22 +159,41 @@ fn flatten_mixture_spec(spec: &MixtureSpec) -> FlatSchema { non_root_ids: Vec::new(), root_child_ids: Vec::new(), }; - flatten_experts(&mut schema, &spec.experts, 0, 1, "0:root", true); - schema + flatten_experts(&mut schema, experts.as_ref(), 0, 1, "0:root", true)?; + Ok(schema) +} + +#[cfg(all(test, feature = "backend-mixture"))] +fn flatten_mixture_spec(spec: &MixtureSpec) -> FlatSchema { + let backend = RateBackend::Mixture { + spec: Arc::new(spec.clone()), + } + .compile() + .unwrap_or_else(|err| panic!("failed to compile diagnostic mixture schema: {err}")); + flatten_compiled_mixture(&backend) + .unwrap_or_else(|err| panic!("failed to flatten diagnostic mixture schema: {err}")) } fn flatten_experts( schema: &mut FlatSchema, - experts: &[MixtureExpertSpec], + experts: &[RateBackendPlanExpert], parent_id: usize, depth: usize, parent_path: &str, root_level: bool, -) { +) -> Result<()> { for expert in experts { - let raw_display_name = expert.name.clone().unwrap_or_else(|| { - RateBackendPredictor::default_name(&expert.backend, expert.max_order) - }); + let backend = + compiled_rate_backend_from_plan(expert.backend.clone()).with_context(|| { + format!( + "failed to compile diagnostic mixture expert '{}'", + expert.name.as_deref().unwrap_or("") + ) + })?; + let raw_display_name = expert + .name + .clone() + .unwrap_or_else(|| backend.default_name()); let display_name = sanitize_tsv_text(&raw_display_name); let node_id = schema.nodes.len(); let path = format!( @@ -246,14 +201,14 @@ fn flatten_experts( node_id, sanitize_path_segment(&display_name) ); - let is_mixture = matches!(expert.backend, RateBackend::Mixture { .. }); + let is_mixture = matches!(expert.backend.as_ref(), RateBackendPlan::Mixture { .. }); let meta = FlatNodeMeta { id: node_id, parent_id: Some(parent_id), depth, path, display_name, - backend_label: sanitize_tsv_text(&backend_label(expert)), + backend_label: sanitize_tsv_text(&backend.display_label()), is_mixture, is_leaf: !is_mixture, is_root_child: root_level, @@ -263,11 +218,19 @@ fn flatten_experts( if root_level { schema.root_child_ids.push(node_id); } - if let RateBackend::Mixture { spec } = &expert.backend { + if let RateBackendPlan::Mixture { experts, .. } = expert.backend.as_ref() { let node_path = schema.nodes[node_id].path.clone(); - flatten_experts(schema, &spec.experts, node_id, depth + 1, &node_path, false); + flatten_experts( + schema, + experts.as_ref(), + node_id, + depth + 1, + &node_path, + false, + )?; } } + Ok(()) } fn write_nodes_tsv(path: &Path, schema: &FlatSchema) -> Result<()> { @@ -326,7 +289,12 @@ pub fn run_ac_log_loss_mixture_bytes( let trace_path = out_prefix.with_extension("trace.tsv"); let nodes_path = out_prefix.with_extension("nodes.tsv"); let summary_path = out_prefix.with_extension("summary.tsv"); - let schema = flatten_mixture_spec(spec); + let compiled_backend = RateBackend::Mixture { + spec: Arc::new(spec.clone()), + } + .compile() + .map_err(anyhow::Error::msg)?; + let schema = flatten_compiled_mixture(&compiled_backend)?; write_nodes_tsv(&nodes_path, &schema)?; let threads = parse_diagnostic_threads_from_env()?; @@ -341,12 +309,7 @@ pub fn run_ac_log_loss_mixture_bytes( None }; - let mut predictor = DiagnosticRatePredictor::from_rate_backend( - RateBackend::Mixture { - spec: Arc::new(spec.clone()), - }, - -1, - )?; + let mut predictor = DiagnosticRatePredictor::from_compiled(&compiled_backend)?; predictor.begin_stream(data.len())?; let trace_file = File::create(&trace_path) @@ -393,6 +356,7 @@ pub fn run_ac_log_loss_mixture_bytes( let mut counter = CountingWriter::default(); { let mut encoder = crate::coders::ArithmeticEncoder::new(&mut counter); + let mut cdf = [0u32; 257]; for (t, &byte) in data.iter().enumerate() { let root_snapshot = predictor.diagnostic_root_snapshot(byte, pool.as_ref(), &mut row_values)?; @@ -484,7 +448,7 @@ pub fn run_ac_log_loss_mixture_bytes( } writeln!(trace_writer, "{}", row.join("\t"))?; - predictor.encode_symbol_ac_step(byte, &mut encoder)?; + predictor.encode_symbol_ac_step(byte, &mut encoder, &mut cdf)?; } let _ = encoder.finish()?; } @@ -573,46 +537,36 @@ pub fn run_ac_log_loss_mixture_bytes( }) } -#[cfg(test)] +#[cfg(all(test, feature = "backend-mixture"))] mod tests { use super::*; - fn test_nested_spec() -> MixtureSpec { + #[cfg(feature = "backend-mixture")] + fn test_nested_spec(base: RateBackend) -> MixtureSpec { MixtureSpec::new( MixtureKind::Switching, vec![ MixtureExpertSpec { - name: Some("ctw".to_string()), + name: Some("leaf-a".to_string()), log_prior: 0.0, - max_order: -1, - backend: RateBackend::Ctw { depth: 6 }, + backend: base.clone(), }, MixtureExpertSpec { name: Some("nested".to_string()), log_prior: -0.1, - max_order: -1, backend: RateBackend::Mixture { spec: Arc::new(MixtureSpec::new( MixtureKind::Bayes, vec![ MixtureExpertSpec { - name: Some("fac".to_string()), + name: Some("leaf-b".to_string()), log_prior: 0.0, - max_order: -1, - backend: RateBackend::FacCtw { - base_depth: 5, - num_percept_bits: 8, - encoding_bits: 8, - }, + backend: base.clone(), }, MixtureExpertSpec { - name: Some("ppmd".to_string()), + name: Some("leaf-c".to_string()), log_prior: 0.0, - max_order: -1, - backend: RateBackend::Ppmd { - order: 4, - memory_mb: 8, - }, + backend: base, }, ], )), @@ -623,9 +577,13 @@ mod tests { .with_alpha(0.2) } + #[cfg(feature = "backend-mixture")] #[test] fn flatten_schema_includes_submixtures_and_descendants_in_preorder() { - let schema = flatten_mixture_spec(&test_nested_spec()); + let Some(base) = crate::runtime::first_enabled_default_rate_backend_spec() else { + return; + }; + let schema = flatten_mixture_spec(&test_nested_spec(base)); assert_eq!(schema.nodes.len(), 5); assert_eq!(schema.nodes[0].display_name, "root"); assert_eq!(schema.root_child_ids, vec![1, 2]); @@ -636,15 +594,13 @@ mod tests { assert_eq!(schema.nodes[4].parent_id, Some(2)); } + #[cfg(feature = "all-backends")] #[test] fn diagnostic_snapshot_matches_root_pdf_and_oracle_minimum() { - let spec = test_nested_spec(); - let mut predictor = DiagnosticRatePredictor::from_rate_backend( - RateBackend::Mixture { - spec: Arc::new(spec.clone()), - }, - -1, - ) + let spec = test_nested_spec(RateBackend::Ctw { depth: 6 }); + let mut predictor = DiagnosticRatePredictor::from_rate_backend(RateBackend::Mixture { + spec: Arc::new(spec.clone()), + }) .expect("predictor"); let data = b"nested diagnostic payload"; predictor.begin_stream(data.len()).expect("begin stream"); @@ -684,9 +640,10 @@ mod tests { predictor.finish_stream().expect("finish stream"); } + #[cfg(feature = "all-backends")] #[test] fn diagnostic_ac_payload_matches_raw_ac_compression_size() { - let spec = test_nested_spec(); + let spec = test_nested_spec(RateBackend::Ctw { depth: 6 }); let data = b"payload bits raw diagnostic parity"; let stamp = format!( "infotheory_ac_diag_{}_{}", @@ -702,10 +659,10 @@ mod tests { let backend = RateBackend::Mixture { spec: Arc::new(spec), }; + let backend = backend.compile().expect("compiled mixture backend"); let encoded = crate::compression::compress_rate_bytes( data, &backend, - -1, crate::coders::CoderType::AC, crate::compression::FramingMode::Raw, ) diff --git a/crates/infotheory/src/error.rs b/crates/infotheory/src/error.rs new file mode 100644 index 00000000..81fe9b68 --- /dev/null +++ b/crates/infotheory/src/error.rs @@ -0,0 +1,139 @@ +//! Shared public error types for fallible infotheory APIs. + +use std::error::Error; +use std::fmt; + +/// Result type used by fallible infotheory APIs. +pub type InfotheoryResult = Result; + +/// Public error type for spec validation, backend construction, and runtime failures. +#[derive(Debug)] +pub enum InfotheoryError { + /// Invalid or unsupported backend/spec configuration supplied by the caller. + InvalidBackendConfig(String), + /// Runtime execution failure while scoring, generating, or compressing. + Runtime(String), + /// Requested operation is not supported for the chosen backend. + Unsupported(String), + /// I/O failure surfaced through infotheory APIs. + Io(std::io::Error), + /// Shared spec/config parsing error. + Spec(crate::spec::SpecError), +} + +impl InfotheoryError { + /// Build an invalid-backend/spec configuration error. + pub fn invalid_backend_config(message: impl Into) -> Self { + Self::InvalidBackendConfig(message.into()) + } + + /// Build a runtime execution error. + pub fn runtime(message: impl Into) -> Self { + Self::Runtime(message.into()) + } + + /// Build an unsupported-operation error. + pub fn unsupported(message: impl Into) -> Self { + Self::Unsupported(message.into()) + } +} + +impl fmt::Display for InfotheoryError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::InvalidBackendConfig(message) => { + write!(f, "invalid backend configuration: {message}") + } + Self::Runtime(message) => write!(f, "runtime failure: {message}"), + Self::Unsupported(message) => write!(f, "unsupported operation: {message}"), + Self::Io(err) => write!(f, "i/o failure: {err}"), + Self::Spec(err) => write!(f, "spec error: {err}"), + } + } +} + +impl Error for InfotheoryError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Io(err) => Some(err), + Self::Spec(err) => Some(err), + _ => None, + } + } +} + +impl From for InfotheoryError { + fn from(value: std::io::Error) -> Self { + Self::Io(value) + } +} + +impl From for InfotheoryError { + fn from(value: crate::spec::SpecError) -> Self { + Self::Spec(value) + } +} + +impl From for InfotheoryError { + fn from(value: String) -> Self { + Self::Runtime(value) + } +} + +impl From<&str> for InfotheoryError { + fn from(value: &str) -> Self { + Self::Runtime(value.to_string()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn constructors_and_display_messages_are_stable() { + assert_eq!( + InfotheoryError::invalid_backend_config("bad depth").to_string(), + "invalid backend configuration: bad depth" + ); + assert_eq!( + InfotheoryError::runtime("decoder stalled").to_string(), + "runtime failure: decoder stalled" + ); + assert_eq!( + InfotheoryError::unsupported("requires vm").to_string(), + "unsupported operation: requires vm" + ); + } + + #[test] + fn source_and_from_conversions_preserve_error_context() { + let io = std::io::Error::other("disk broke"); + let io_error = InfotheoryError::from(io); + assert!(io_error.to_string().contains("i/o failure: disk broke")); + assert!(io_error.source().is_some()); + + let spec = crate::spec::SpecError::new("bad spec"); + let spec_error = InfotheoryError::from(spec.clone()); + assert_eq!(spec_error.to_string(), "spec error: bad spec"); + assert_eq!( + spec_error + .source() + .expect("spec error should retain source") + .to_string(), + spec.to_string() + ); + + let runtime_from_string = InfotheoryError::from("runtime text"); + assert_eq!( + runtime_from_string.to_string(), + "runtime failure: runtime text" + ); + + let runtime_from_owned = InfotheoryError::from(String::from("owned runtime")); + assert_eq!( + runtime_from_owned.to_string(), + "runtime failure: owned runtime" + ); + } +} diff --git a/crates/infotheory/src/lib.rs b/crates/infotheory/src/lib.rs new file mode 100644 index 00000000..0cef9215 --- /dev/null +++ b/crates/infotheory/src/lib.rs @@ -0,0 +1,1465 @@ +#![allow(unsafe_op_in_unsafe_fn)] + +//! # InfoTheory: Information Theoretic Estimators & Metrics +//! +//! This crate provides a comprehensive suite of information-theoretic primitives for +//! quantifying complexity, dependence, and similarity between data sequences. +//! +//! It implements two complementary classes of estimators: +//! 1. **Algorithmic information theory (predictive / Kolmogorov +//! complexity)**: estimates `K(·)`-flavored quantities by treating a +//! model as a description-length functional — either a compressor +//! `C(·)` or a sequential predictor used as a prequential code +//! `-log₂ p(x_t | x_{ runtime builders and backend registry metadata. +pub(crate) mod runtime; +/// Information-theoretic code search pipeline (3-stage: prefilter, filter, KMI rerank). +#[cfg(feature = "backend-rosa")] +pub mod search; +#[cfg(feature = "backend-particle")] +pub(crate) mod simd_math; +/// Shared backend/spec parsing and loading helpers. +pub mod spec; +/// Tuner runtime execution controls and CLI-facing tuning entrypoints. +#[cfg(feature = "tuner")] +pub mod tuner; +use crate::api::CompiledRateBackend; +#[cfg(all(test, feature = "all-backends"))] +pub(crate) use crate::api::{ + CalibratedSpec, CalibrationContextKind, MixtureExpertSpec, MixtureKind, MixtureSpec, + ParticleSpec, +}; +#[cfg(all(test, feature = "all-backends"))] +use crate::api::{ + CompressionBackend, GenerationConfig, InfotheoryCtx, NcdVariant, RateBackend, + RateBackendSession, d_kl_bytes, try_biased_entropy_rate_backend, + try_conditional_entropy_rate_bytes, try_cross_entropy_rate_backend, try_entropy_rate_backend, + try_entropy_rate_bytes, try_joint_entropy_rate_backend, try_joint_entropy_rate_bytes, + try_mutual_information_bytes, +}; +#[cfg(all(test, feature = "all-backends"))] +use crate::api::{ + empirical_entropy_bytes, empirical_joint_entropy_bytes, js_div_bytes, nhd_bytes, tvd_bytes, +}; +use crate::error::{InfotheoryError, InfotheoryResult}; +/// CTW and FAC-CTW backend types. +#[cfg(feature = "backend-ctw")] +pub use backends::ctw; +#[cfg(feature = "backend-mamba")] +/// Mamba backend types and compressor. +pub use backends::mambazip; +/// Match-based repeat predictor. +#[cfg(feature = "backend-match")] +pub use backends::match_model; +/// Particle-latent filter ensemble rate backend. +#[cfg(feature = "backend-particle")] +pub use backends::particle; +/// PPMD-style byte model. +#[cfg(feature = "backend-ppmd")] +pub use backends::ppmd; +/// ROSA+ backend types. +#[cfg(feature = "backend-rosa")] +pub use backends::rosaplus; +#[cfg(feature = "backend-rwkv")] +/// RWKV backend types and compressor. +pub use backends::rwkvzip; +/// Exact online Sequitur backend types. +#[cfg(feature = "backend-sequitur")] +pub use backends::sequitur; +/// Sparse/gapped match predictor. +#[cfg(feature = "backend-match")] +pub use backends::sparse_match; +/// ZPAQ rate-model adapter. +#[cfg(feature = "backend-zpaq")] +pub use backends::zpaq_rate; + +#[cfg(feature = "backend-rosa")] +use crate::backends::rosaplus::RosaPlus; +use crate::mixture::OnlineBytePredictor; +use std::cell::RefCell; +#[cfg(any(feature = "backend-rwkv", feature = "backend-mamba"))] +use std::collections::HashMap; +#[cfg(all(test, feature = "all-backends"))] +use std::sync::Arc; + +thread_local! { + #[cfg(feature = "backend-mamba")] + static MAMBA_METHOD_TLS: RefCell> = RefCell::new(HashMap::new()); + #[cfg(feature = "backend-rwkv")] + static RWKV_METHOD_TLS: RefCell> = RefCell::new(HashMap::new()); +} + +thread_local! { + static DEFAULT_CTX: RefCell> = const { RefCell::new(None) }; +} + +/// Returns the current default information theory context for the thread. +pub(crate) fn get_default_ctx() -> InfotheoryResult { + DEFAULT_CTX.with(|ctx| { + let mut slot = ctx.borrow_mut(); + if slot.is_none() { + *slot = Some(api::InfotheoryCtx::try_default()?); + } + Ok(slot + .as_ref() + .expect("default context initialized above") + .clone()) + }) +} + +/// Sets the default information theory context for the thread. +pub(crate) fn set_default_ctx(ctx: api::InfotheoryCtx) { + DEFAULT_CTX.with(|c| *c.borrow_mut() = Some(ctx)); +} + +#[inline(always)] +pub(crate) fn with_default_ctx( + f: impl FnOnce(&api::InfotheoryCtx) -> InfotheoryResult, +) -> InfotheoryResult { + let ctx = get_default_ctx()?; + f(&ctx) +} + +#[inline(always)] +pub(crate) fn aligned_prefix<'a>(x: &'a [u8], y: &'a [u8]) -> (&'a [u8], &'a [u8]) { + let n = x.len().min(y.len()); + (&x[..n], &y[..n]) +} + +#[cfg(feature = "backend-zpaq")] +#[inline(always)] +pub(crate) fn try_zpaq_compress_size_bytes(data: &[u8], method: &str) -> InfotheoryResult { + zpaq_rs::compress_size(data, method) + .map_err(|err| InfotheoryError::runtime(format!("zpaq size compression failed: {err}"))) +} + +#[cfg(not(feature = "backend-zpaq"))] +#[inline(always)] +pub(crate) fn try_zpaq_compress_size_bytes(_data: &[u8], _method: &str) -> InfotheoryResult { + Err(InfotheoryError::unsupported( + "CompressionBackend::Zpaq is unavailable: build with feature 'backend-zpaq'", + )) +} + +#[cfg(feature = "backend-zpaq")] +#[inline(always)] +pub(crate) fn try_zpaq_compress_size_parallel_bytes( + data: &[u8], + method: &str, + threads: usize, +) -> InfotheoryResult { + zpaq_rs::compress_size_parallel(data, method, threads).map_err(|err| { + InfotheoryError::runtime(format!("zpaq parallel size compression failed: {err}")) + }) +} + +#[cfg(not(feature = "backend-zpaq"))] +#[inline(always)] +pub(crate) fn try_zpaq_compress_size_parallel_bytes( + _data: &[u8], + _method: &str, + _threads: usize, +) -> InfotheoryResult { + Err(InfotheoryError::unsupported( + "CompressionBackend::Zpaq is unavailable: build with feature 'backend-zpaq'", + )) +} + +#[cfg(feature = "backend-zpaq")] +#[inline(always)] +pub(crate) fn try_zpaq_compress_size_stream( + reader: R, + method: &str, +) -> InfotheoryResult { + zpaq_rs::compress_size_stream(reader, method, None, None) + .map_err(|err| InfotheoryError::runtime(format!("zpaq stream compression failed: {err}"))) +} + +#[cfg(feature = "backend-zpaq")] +#[inline(always)] +pub(crate) fn try_zpaq_compress_size_stream_parallel( + reader: R, + method: &str, + threads: usize, +) -> InfotheoryResult { + zpaq_rs::compress_size_stream_parallel(reader, method, None, None, threads) + .map_err(|err| InfotheoryError::runtime(format!("zpaq stream compression failed: {err}"))) +} + +#[cfg(not(feature = "backend-zpaq"))] +#[inline(always)] +pub(crate) fn try_zpaq_compress_size_stream_parallel( + _reader: R, + _method: &str, + _threads: usize, +) -> InfotheoryResult { + Err(InfotheoryError::unsupported( + "CompressionBackend::Zpaq is unavailable: build with feature 'backend-zpaq'", + )) +} + +#[cfg(not(feature = "backend-zpaq"))] +#[inline(always)] +pub(crate) fn try_zpaq_compress_size_stream( + _reader: R, + _method: &str, +) -> InfotheoryResult { + Err(InfotheoryError::unsupported( + "CompressionBackend::Zpaq is unavailable: build with feature 'backend-zpaq'", + )) +} + +#[cfg(feature = "backend-zpaq")] +#[inline(always)] +pub(crate) fn zpaq_compress_to_vec(data: &[u8], method: &str) -> anyhow::Result> { + Ok(zpaq_rs::compress_to_vec(data, method)?) +} + +#[cfg(not(feature = "backend-zpaq"))] +#[inline(always)] +pub(crate) fn zpaq_compress_to_vec(_data: &[u8], _method: &str) -> anyhow::Result> { + anyhow::bail!("zpaq backend disabled at compile time (enable feature 'backend-zpaq')") +} + +#[cfg(feature = "backend-zpaq")] +#[inline(always)] +pub(crate) fn zpaq_decompress_to_vec(data: &[u8]) -> anyhow::Result> { + Ok(zpaq_rs::decompress_to_vec(data)?) +} + +#[cfg(not(feature = "backend-zpaq"))] +#[inline(always)] +pub(crate) fn zpaq_decompress_to_vec(_data: &[u8]) -> anyhow::Result> { + anyhow::bail!("zpaq backend disabled at compile time (enable feature 'backend-zpaq')") +} + +/// Validate that a ZPAQ method string is supported for rate estimation. +pub fn validate_zpaq_rate_method(method: &str) -> InfotheoryResult<()> { + #[cfg(feature = "backend-zpaq")] + { + zpaq_rate::validate_zpaq_rate_method(method) + .map_err(InfotheoryError::invalid_backend_config) + } + #[cfg(not(feature = "backend-zpaq"))] + { + let _ = method; + Err(InfotheoryError::unsupported( + "zpaq backend disabled at compile time", + )) + } +} + +#[cfg(feature = "backend-rwkv")] +pub(crate) fn with_rwkv_method_spec_tls( + method: &str, + spec: &rwkvzip::MethodSpec, + f: impl FnOnce(&mut rwkvzip::Compressor) -> R, +) -> R { + RWKV_METHOD_TLS.with(|cell| { + let mut map = cell.borrow_mut(); + let mut comp = if let Some(template) = map.get(method) { + template.clone() + } else { + let template = rwkvzip::Compressor::new_from_method_spec(spec).unwrap_or_else(|e| { + panic!("invalid rwkv method '{method}': {e:#}"); + }); + map.insert(method.to_string(), template.clone()); + template + }; + drop(map); + f(&mut comp) + }) +} + +#[cfg(feature = "backend-mamba")] +pub(crate) fn with_mamba_method_spec_tls( + method: &str, + spec: &mambazip::MethodSpec, + f: impl FnOnce(&mut mambazip::Compressor) -> R, +) -> R { + MAMBA_METHOD_TLS.with(|cell| { + let mut map = cell.borrow_mut(); + let mut comp = if let Some(template) = map.get(method) { + template.clone() + } else { + let template = mambazip::Compressor::new_from_method_spec(spec).unwrap_or_else(|e| { + panic!("invalid mamba method '{method}': {e:#}"); + }); + map.insert(method.to_string(), template.clone()); + template + }; + drop(map); + f(&mut comp) + }) +} + +pub(crate) fn try_prequential_rate_backend( + data: &[u8], + prefix_parts: &[&[u8]], + backend: &CompiledRateBackend, +) -> InfotheoryResult { + if data.is_empty() { + return Ok(0.0); + } + let total = prefix_parts + .iter() + .map(|p| p.len() as u64) + .sum::() + .saturating_add(data.len() as u64); + let mut predictor = + crate::runtime::build_rate_backend_predictor_default(backend).map_err(|e| { + InfotheoryError::runtime(format!("rate backend predictor init failed: {e}")) + })?; + predictor + .begin_stream(Some(total)) + .map_err(|e| InfotheoryError::runtime(format!("rate backend stream init failed: {e}")))?; + for prefix in prefix_parts { + for &b in *prefix { + predictor.update(b); + } + } + let mut bits = 0.0; + for &b in data { + bits -= predictor.log_prob(b) / std::f64::consts::LN_2; + predictor.update(b); + } + predictor.finish_stream().map_err(|e| { + InfotheoryError::runtime(format!("rate backend stream finalize failed: {e}")) + })?; + Ok(bits / (data.len() as f64)) +} + +pub(crate) fn try_frozen_plugin_rate_backend( + score_data: &[u8], + fit_parts: &[&[u8]], + backend: &CompiledRateBackend, +) -> InfotheoryResult { + if score_data.is_empty() { + return Ok(0.0); + } + #[cfg(feature = "backend-rosa")] + if let crate::spec::core::RateBackendPlan::RosaPlus { max_order } = backend.plan() { + let mut model = RosaPlus::new(*max_order, false, 0, 42); + let fit_total = fit_parts.iter().map(|part| part.len()).sum::(); + if fit_total > 0 { + model.reserve_for_stream(fit_total); + let mut non_empty_parts = fit_parts + .iter() + .copied() + .filter(|part| !part.is_empty()) + .peekable(); + while let Some(part) = non_empty_parts.next() { + if non_empty_parts.peek().is_some() { + model.train_sequence(part); + } else { + model.train_example(part); + } + } + } + model.build_lm(); + return Ok(model.cross_entropy(score_data)); + } + #[cfg(feature = "backend-rwkv")] + if let crate::spec::core::RateBackendPlan::Rwkv7 { + method, + parsed_method, + .. + } = backend.plan() + { + return with_rwkv_method_spec_tls(method, parsed_method, |c| { + c.cross_entropy_frozen_plugin_chain(fit_parts, score_data) + .map_err(|e| { + InfotheoryError::runtime(format!( + "rwkv method frozen-plugin scoring failed: {e:#}" + )) + }) + }); + } + #[cfg(feature = "backend-mamba")] + if let crate::spec::core::RateBackendPlan::Mamba { + method, + parsed_method, + .. + } = backend.plan() + { + return with_mamba_method_spec_tls(method, parsed_method, |c| { + c.cross_entropy_frozen_plugin_chain(fit_parts, score_data) + .map_err(|e| { + InfotheoryError::runtime(format!( + "mamba method frozen-plugin scoring failed: {e:#}" + )) + }) + }); + } + + let fit_total = fit_parts.iter().map(|part| part.len() as u64).sum::(); + let mut predictor = + crate::runtime::build_rate_backend_predictor_default(backend).map_err(|e| { + InfotheoryError::runtime(format!("rate backend predictor init failed: {e}")) + })?; + predictor + .begin_stream(Some(fit_total)) + .map_err(|e| InfotheoryError::runtime(format!("rate backend fit-pass init failed: {e}")))?; + for part in fit_parts { + for &byte in *part { + predictor.update(byte); + } + } + predictor.finish_stream().map_err(|e| { + InfotheoryError::runtime(format!("rate backend fit-pass finalize failed: {e}")) + })?; + predictor + .reset_frozen(Some(score_data.len() as u64)) + .map_err(|e| { + InfotheoryError::runtime(format!("rate backend frozen-score reset failed: {e}")) + })?; + let mut bits = 0.0; + for &byte in score_data { + bits -= predictor.log_prob(byte) / std::f64::consts::LN_2; + predictor.update_frozen(byte); + } + predictor.finish_stream().map_err(|e| { + InfotheoryError::runtime(format!("rate backend frozen-score finalize failed: {e}")) + })?; + Ok(bits / (score_data.len() as f64)) +} + +#[cfg(all(test, feature = "all-backends"))] +mod tests { + use super::*; + + #[cfg(not(feature = "backend-zpaq"))] + fn compress_size_backend(data: &[u8], backend: &CompressionBackend) -> u64 { + let compiled = backend + .compile() + .unwrap_or_else(|err| panic!("failed to compile compression backend for test: {err}")); + crate::api::try_compress_size_backend(data, &compiled).expect("compress_size_backend") + } + + fn compiled_rate_backend(backend: &RateBackend) -> crate::spec::CompiledRateBackend { + backend + .compile() + .unwrap_or_else(|err| panic!("failed to compile rate backend for test: {err}")) + } + + fn ctx(rate_backend: RateBackend, compression_backend: CompressionBackend) -> InfotheoryCtx { + InfotheoryCtx::from_specs(rate_backend, compression_backend) + .unwrap_or_else(|err| panic!("failed to build infotheory test context: {err}")) + } + + fn default_compression_backend() -> CompressionBackend { + CompressionBackend::try_default() + .unwrap_or_else(|err| panic!("failed to select default compression backend: {err}")) + } + + fn default_ctx() -> InfotheoryCtx { + InfotheoryCtx::try_default() + .unwrap_or_else(|err| panic!("failed to build default test context: {err}")) + } + + fn generate_rate_backend_chain( + prefix_parts: &[&[u8]], + bytes: usize, + backend: &RateBackend, + config: GenerationConfig, + ) -> Vec { + crate::api::generation::generate_rate_backend_chain( + prefix_parts, + bytes, + &compiled_rate_backend(backend), + config, + ) + } + + fn ncd_bytes(x: &[u8], y: &[u8], method: &str, variant: NcdVariant) -> f64 { + let backend = CompressionBackend::zpaq(method) + .compile() + .expect("compile zpaq backend"); + crate::api::try_ncd_bytes_backend(x, y, &backend, variant).expect("ncd_bytes") + } + + fn entropy_rate_bytes(data: &[u8]) -> f64 { + try_entropy_rate_bytes(data).expect("entropy_rate_bytes") + } + + fn entropy_rate_backend(data: &[u8], backend: &RateBackend) -> f64 { + try_entropy_rate_backend(data, &compiled_rate_backend(backend)) + .expect("entropy_rate_backend") + } + + fn biased_entropy_rate_backend(data: &[u8], backend: &RateBackend) -> f64 { + try_biased_entropy_rate_backend(data, &compiled_rate_backend(backend)) + .expect("biased_entropy_rate_backend") + } + + fn cross_entropy_rate_backend( + test_data: &[u8], + train_data: &[u8], + backend: &RateBackend, + ) -> f64 { + try_cross_entropy_rate_backend(test_data, train_data, &compiled_rate_backend(backend)) + .expect("cross_entropy_rate_backend") + } + + fn joint_entropy_rate_backend(x: &[u8], y: &[u8], backend: &RateBackend) -> f64 { + try_joint_entropy_rate_backend(x, y, &compiled_rate_backend(backend)) + .expect("joint_entropy_rate_backend") + } + + fn joint_entropy_rate_bytes(x: &[u8], y: &[u8]) -> f64 { + try_joint_entropy_rate_bytes(x, y).expect("joint_entropy_rate_bytes") + } + + fn conditional_entropy_rate_bytes(x: &[u8], y: &[u8]) -> f64 { + try_conditional_entropy_rate_bytes(x, y).expect("conditional_entropy_rate_bytes") + } + + fn mutual_information_bytes(x: &[u8], y: &[u8]) -> f64 { + try_mutual_information_bytes(x, y).expect("mutual_information_bytes") + } + + fn ned_bytes(x: &[u8], y: &[u8]) -> f64 { + crate::api::try_ned_bytes(x, y).expect("ned_bytes") + } + + fn nte_rate_backend(x: &[u8], y: &[u8], backend: &RateBackend) -> f64 { + crate::api::try_nte_rate_backend(x, y, &compiled_rate_backend(backend)) + .expect("nte_rate_backend") + } + + fn resistance_to_transformation_bytes(x: &[u8], tx: &[u8]) -> f64 { + crate::api::try_resistance_to_transformation_bytes(x, tx) + .expect("resistance_to_transformation_bytes") + } + + fn test_match_backend() -> RateBackend { + RateBackend::Match { + hash_bits: 12, + min_len: 2, + max_len: 16, + base_mix: 0.01, + confidence_scale: 1.0, + } + } + + fn test_ppmd_backend() -> RateBackend { + RateBackend::Ppmd { + order: 4, + memory_mb: 1, + } + } + + fn test_calibrated_backend() -> RateBackend { + RateBackend::Calibrated { + spec: Arc::new(CalibratedSpec { + base: test_match_backend(), + context: CalibrationContextKind::Text, + bins: 16, + learning_rate: 0.05, + bias_clip: 4.0, + }), + } + } + + fn test_mixture_backend() -> RateBackend { + RateBackend::Mixture { + spec: Arc::new(MixtureSpec::new( + MixtureKind::Bayes, + vec![ + MixtureExpertSpec { + name: Some("match".to_string()), + log_prior: 0.0, + backend: test_match_backend(), + }, + MixtureExpertSpec { + name: Some("ppmd".to_string()), + log_prior: 0.0, + backend: test_ppmd_backend(), + }, + ], + )), + } + } + + fn test_particle_backend() -> RateBackend { + RateBackend::Particle { + spec: Arc::new(ParticleSpec { + num_particles: 4, + num_cells: 4, + cell_dim: 8, + num_rules: 2, + selector_hidden: 16, + rule_hidden: 16, + context_window: 8, + unroll_steps: 1, + ..ParticleSpec::default() + }), + } + } + + fn continuation_prompt() -> &'static [u8] { + b"If a frog is green, dogs are red.\nIf a toad is green, cats are red.\nIf a dog is green, frogs are red.\nIf a cat is green, toads are red.\nIf a frog is red, dogs are green.\nIf a toad is red, cats are green.\nIf a dog is red, frogs are green.\nIf a cat is red, toads are \n" + } + + fn assert_deterministic_generate_for_backend(backend: RateBackend, bytes: usize, label: &str) { + let prompt = continuation_prompt(); + let a = + generate_rate_backend_chain(&[prompt], bytes, &backend, GenerationConfig::default()); + let b = + generate_rate_backend_chain(&[prompt], bytes, &backend, GenerationConfig::default()); + assert_eq!( + a, b, + "{label} generation should be deterministic for identical input" + ); + assert_eq!( + a.len(), + bytes, + "{label} generation should emit requested byte count" + ); + } + + fn assert_sampled_generate_for_backend(backend: RateBackend, bytes: usize, label: &str) { + let prompt = continuation_prompt(); + let config = GenerationConfig::sampled_frozen(42); + let a = generate_rate_backend_chain(&[prompt], bytes, &backend, config); + let b = generate_rate_backend_chain(&[prompt], bytes, &backend, config); + assert_eq!( + a, b, + "{label} sampled generation should be deterministic for a fixed seed" + ); + assert_eq!( + a.len(), + bytes, + "{label} sampled generation should emit requested byte count" + ); + } + + #[cfg(feature = "backend-zpaq")] + #[test] + fn ncd_basic_identity_nonnegative() { + let x = b"abcdabcdabcd"; + let d = ncd_bytes(x, x, "5", NcdVariant::Vitanyi); + assert!(d >= -1e-9); + } + + #[test] + fn shannon_identities_empirical_aligned() { + let x = b"abracadabra"; + let y = b"abracadabra"; + + let h_x = empirical_entropy_bytes(x); + let mi = crate::api::empirical_mutual_information_bytes(x, y); + let h_xy = empirical_joint_entropy_bytes(x, y); + let h_x_given_y = (h_xy - h_x).max(0.0); + let ned = crate::api::empirical_ned_bytes(x, y); + let nte = crate::api::empirical_nte_bytes(x, y); + + assert!((h_xy - h_x).abs() < 1e-12); + assert!(h_x_given_y.abs() < 1e-12); + assert!((mi - h_x).abs() < 1e-12); + assert!(ned.abs() < 1e-12); + assert!(nte.abs() < 1e-12); + } + + #[test] + fn shannon_identities_rate_aligned_reasonable() { + let x = b"the quick brown fox jumps over the lazy dog"; + let y = b"the quick brown fox jumps over the lazy dog"; + let prev = get_default_ctx().expect("default ctx"); + set_default_ctx(ctx( + RateBackend::RosaPlus { max_order: 8 }, + default_compression_backend(), + )); + + let h_x = entropy_rate_bytes(x); + let h_xy = joint_entropy_rate_bytes(x, y); + let h_x_given_y = conditional_entropy_rate_bytes(x, y); + let mi = mutual_information_bytes(x, y); + let ned = ned_bytes(x, y); + + // Finite-sample estimators won't be exact; allow reasonable tolerance. + let tol = 0.2; + assert!((h_xy - h_x).abs() < tol); + assert!(h_x_given_y < tol); + assert!((mi - h_x).abs() < tol); + assert!(ned < tol); + set_default_ctx(prev); + } + + #[test] + fn resistance_identity_is_one() { + let x = b"some repeated repeated repeated text"; + let prev = get_default_ctx().expect("default ctx"); + set_default_ctx(ctx( + RateBackend::RosaPlus { max_order: 8 }, + default_compression_backend(), + )); + let r = resistance_to_transformation_bytes(x, x); + assert!((r - 1.0).abs() < 1e-6); + set_default_ctx(prev); + } + + #[test] + fn empirical_metrics_empty_inputs_are_zero() { + let empty: &[u8] = &[]; + let x = b"abc"; + + assert_eq!(tvd_bytes(empty, x), 0.0); + assert_eq!(tvd_bytes(x, empty), 0.0); + assert_eq!(nhd_bytes(empty, x), 0.0); + assert_eq!(nhd_bytes(x, empty), 0.0); + assert_eq!(d_kl_bytes(empty, x), 0.0); + assert_eq!(d_kl_bytes(x, empty), 0.0); + assert_eq!(js_div_bytes(empty, x), 0.0); + assert_eq!(js_div_bytes(x, empty), 0.0); + } + + #[test] + fn empirical_cross_entropy_empty_test_is_zero() { + let empty: &[u8] = &[]; + let y = b"abc"; + assert_eq!(crate::api::empirical_cross_entropy_bytes(empty, y), 0.0); + } + + #[test] + fn backend_switching_test() { + let x = b"hello world context"; + + // Default is RosaPlus + let h_rosa = entropy_rate_bytes(x); + + // Switch to CTW + set_default_ctx(ctx( + RateBackend::Ctw { depth: 16 }, + default_compression_backend(), + )); + + let h_ctw = entropy_rate_bytes(x); + + // They should generally be different, but most importantly, CTW worked + assert!(h_ctw > 0.0); + + // Reset to default + set_default_ctx(default_ctx()); + let h_rosa_back = entropy_rate_bytes(x); + assert!((h_rosa - h_rosa_back).abs() < 1e-12); + } + + #[test] + fn ctw_early_updates_work() { + // Test that CTW produces valid predictions from the very start, + // not just after `depth` symbols have been processed. + use crate::backends::ctw::ContextTree; + + let mut tree = ContextTree::new(16); + + // Even the first prediction should be valid (not NaN, not 0) + let p0 = tree.predict(false); + let p1 = tree.predict(true); + + // Initial KT estimator gives 0.5 / 1 = 0.5 for each symbol + assert!((p0 - 0.5).abs() < 1e-10, "p0 should be ~0.5, got {}", p0); + assert!((p1 - 0.5).abs() < 1e-10, "p1 should be ~0.5, got {}", p1); + assert!((p0 + p1 - 1.0).abs() < 1e-10, "p0 + p1 should = 1.0"); + + // Update with a few symbols and verify log_prob becomes negative (valid) + for _ in 0..5 { + tree.update(true); + tree.update(false); + } + + let log_prob = tree.get_log_block_probability(); + assert!( + log_prob < 0.0, + "log_prob should be negative (< log 1), got {}", + log_prob + ); + assert!(log_prob.is_finite(), "log_prob should be finite"); + } + + #[test] + fn nte_can_exceed_one() { + // Test that NTE is properly clamped to [0, 2] instead of [0, 1] + // For independent sequences with similar entropy, NTE can approach 2.0 + // + // Note: For *empirical* (order-0) NTE, due to how joint entropy works for aligned + // pairs, it's mathematically bounded differently. The fix for NTE clamping primarily + // affects *rate*-based NTE where VI can truly be 2*max(H). + // + // We test that the clamp upper bound is at least > 1.0 for cases where VI > max(H) + + // Use CTW backend for rate-based test + set_default_ctx(ctx( + RateBackend::Ctw { depth: 8 }, + default_compression_backend(), + )); + + // Generate two completely different patterns - should have high VI + let x: Vec = (0..200).map(|i| (i % 2) as u8).collect(); // 010101... + let y: Vec = (0..200).map(|i| ((i + 1) % 2) as u8).collect(); // 101010... + + let nte_rate = nte_rate_backend(&x, &y, &RateBackend::Ctw { depth: 8 }); + + // With the fix, NTE should not be clamped to 1.0 + // It may or may not exceed 1.0 depending on the specifics, but it should be allowed to + assert!( + (0.0..=2.0 + 1e-9).contains(&nte_rate), + "NTE should be in [0, 2], got {}", + nte_rate + ); + + // Reset context + set_default_ctx(default_ctx()); + } + + #[test] + fn ctw_empty_data_returns_zero() { + // Verify empty data doesn't cause division-by-zero or NaN + set_default_ctx(ctx( + RateBackend::Ctw { depth: 16 }, + default_compression_backend(), + )); + + let empty: &[u8] = &[]; + let h = entropy_rate_bytes(empty); + assert_eq!(h, 0.0, "empty data should return 0.0 entropy"); + + // Reset + set_default_ctx(default_ctx()); + } + + #[test] + fn joint_entropy_rate_aligns_inputs_and_handles_empty_cases() { + let cases = vec![ + ("ctw", RateBackend::Ctw { depth: 8 }), + ( + "fac-ctw", + RateBackend::FacCtw { + base_depth: 8, + num_percept_bits: 8, + encoding_bits: 8, + msb_first: None, + }, + ), + ("match", test_match_backend()), + ]; + + for (name, backend) in cases { + assert_eq!( + joint_entropy_rate_backend(b"", b"nonempty", &backend), + 0.0, + "{name} should return 0.0 for empty aligned pairs" + ); + assert_eq!( + joint_entropy_rate_backend(b"nonempty", b"", &backend), + 0.0, + "{name} should return 0.0 when alignment truncates to empty" + ); + + let aligned = joint_entropy_rate_backend(b"abcd", b"wxyz", &backend); + let truncated = joint_entropy_rate_backend(b"abcdextra", b"wxyz", &backend); + assert!( + (aligned - truncated).abs() < 1e-12, + "{name} should score only the aligned prefix: aligned={aligned} truncated={truncated}" + ); + } + } + + #[test] + fn biased_entropy_is_repeatable_across_backend_families() { + let data = b"ABABABAABBABABABAABB"; + let cases = vec![ + ("match", test_match_backend()), + ("ppmd", test_ppmd_backend()), + ("calibrated", test_calibrated_backend()), + ("ctw", RateBackend::Ctw { depth: 8 }), + ("mixture", test_mixture_backend()), + ("particle", test_particle_backend()), + ]; + + for (name, backend) in cases { + let h1 = biased_entropy_rate_backend(data, &backend); + let h2 = biased_entropy_rate_backend(data, &backend); + assert!(h1.is_finite(), "{name} biased entropy should be finite"); + assert!( + (h1 - h2).abs() < 1e-12, + "{name} biased entropy leaked mutable state across calls: h1={h1} h2={h2}" + ); + } + } + + #[test] + fn generate_bytes_chain_matches_flat_prompt() { + let prompt = continuation_prompt(); + let split_at = prompt.len() / 2; + let front = &prompt[..split_at]; + let back = &prompt[split_at..]; + let backend = RateBackend::Ctw { depth: 32 }; + let bytes = 8usize; + + let flat = + generate_rate_backend_chain(&[prompt], bytes, &backend, GenerationConfig::default()); + let chained = generate_rate_backend_chain( + &[front, back], + bytes, + &backend, + GenerationConfig::default(), + ); + assert_eq!( + flat, chained, + "chain conditioning should match flat prompt conditioning" + ); + } + + #[test] + fn generate_bytes_api_is_deterministic_for_ctw_rosa_match_ppmd() { + assert_deterministic_generate_for_backend(RateBackend::Ctw { depth: 32 }, 8, "ctw"); + assert_deterministic_generate_for_backend( + RateBackend::RosaPlus { max_order: -1 }, + 8, + "rosaplus", + ); + assert_deterministic_generate_for_backend(test_match_backend(), 8, "match"); + assert_deterministic_generate_for_backend(test_ppmd_backend(), 8, "ppmd"); + } + + #[cfg(feature = "backend-rwkv")] + #[test] + fn generate_bytes_api_is_deterministic_for_rwkv_method() { + let backend = RateBackend::Rwkv7Method { + method: crate::rwkvzip::parse_method_spec("cfg:hidden=64,layers=1,intermediate=64,decay_rank=8,a_rank=8,v_rank=8,g_rank=8,seed=31,train=none,lr=0.0,stride=1;policy:schedule=0..100:infer").expect("rwkv method spec"), + }; + assert_deterministic_generate_for_backend(backend, 8, "rwkv7"); + } + + #[test] + fn sampled_generation_is_deterministic_for_ctw_rosa_match_ppmd() { + assert_sampled_generate_for_backend(RateBackend::Ctw { depth: 32 }, 8, "ctw"); + assert_sampled_generate_for_backend(RateBackend::RosaPlus { max_order: -1 }, 8, "rosaplus"); + assert_sampled_generate_for_backend(test_match_backend(), 8, "match"); + assert_sampled_generate_for_backend(test_ppmd_backend(), 8, "ppmd"); + } + + #[cfg(feature = "backend-rwkv")] + #[test] + fn sampled_generation_is_deterministic_for_rwkv_method() { + let backend = RateBackend::Rwkv7Method { + method: crate::rwkvzip::parse_method_spec("cfg:hidden=64,layers=1,intermediate=64,decay_rank=8,a_rank=8,v_rank=8,g_rank=8,seed=31,train=none,lr=0.0,stride=1;policy:schedule=0..100:infer").expect("rwkv method spec"), + }; + assert_sampled_generate_for_backend(backend, 8, "rwkv7"); + } + + #[test] + fn rosaplus_sampled_generation_predicts_green_continuation() { + let out = generate_rate_backend_chain( + &[continuation_prompt()], + 8, + &RateBackend::RosaPlus { max_order: -1 }, + GenerationConfig::sampled_frozen(42), + ); + assert_eq!(out, b" green.\n"); + } + + #[test] + fn rate_backend_session_matches_ctx_generation() { + let prompt = continuation_prompt(); + let backend = RateBackend::Ppmd { + order: 12, + memory_mb: 8, + }; + let mut session = + RateBackendSession::from_spec(backend.clone(), Some((prompt.len() + 8) as u64)) + .expect("session init"); + session.observe(prompt); + let from_session = session.generate_bytes(8, GenerationConfig::sampled_frozen(42)); + session.finish().expect("session finish"); + + let ctx = ctx(backend, default_compression_backend()); + let from_ctx = ctx + .try_generate_bytes_with_config(prompt, 8, GenerationConfig::sampled_frozen(42)) + .expect("ctx generation"); + assert_eq!(from_session, from_ctx); + } + + #[test] + fn biased_entropy_ctw_uses_frozen_plugin_scoring() { + let backend = RateBackend::Ctw { depth: 8 }; + let data = b"AAAAAAAA"; + let plugin = biased_entropy_rate_backend(data, &backend); + let prequential = entropy_rate_backend(data, &backend); + assert!( + plugin + 1e-9 < prequential, + "expected plugin scoring to beat prequential scoring: plugin={plugin} prequential={prequential}" + ); + } + + #[test] + fn rosa_plugin_entropy_matches_direct_model_api() { + let data = b"abracadabra"; + let backend = RateBackend::RosaPlus { max_order: 3 }; + + let plugin = biased_entropy_rate_backend(data, &backend); + + let mut direct = RosaPlus::new(3, false, 0, 42); + direct.train_example(data); + direct.build_lm(); + let expected = direct.cross_entropy(data); + + assert!( + (plugin - expected).abs() < 1e-12, + "rosa plugin entropy must match direct model API: plugin={plugin} expected={expected}" + ); + } + + #[test] + fn rosa_plugin_cross_entropy_matches_direct_model_api() { + let train = b"alakazam"; + let test = b"abracadabra"; + let backend = RateBackend::RosaPlus { max_order: 3 }; + + let plugin = cross_entropy_rate_backend(test, train, &backend); + + let mut direct = RosaPlus::new(3, false, 0, 42); + direct.train_example(train); + direct.build_lm(); + let expected = direct.cross_entropy(test); + + assert!( + (plugin - expected).abs() < 1e-12, + "rosa plugin cross entropy must match direct model API: plugin={plugin} expected={expected}" + ); + } + + #[test] + fn rosa_conditional_chain_matches_concatenated_prefix_scoring() { + let ctx = ctx( + RateBackend::RosaPlus { max_order: -1 }, + default_compression_backend(), + ); + let prefix_parts: [&[u8]; 3] = [b"universal ", b"prior ", b"slice"]; + let data = b"query payload"; + + let chained = ctx + .try_cross_entropy_conditional_chain(&prefix_parts, data) + .expect("conditional-chain cross entropy"); + let flat_prefix: Vec = prefix_parts.concat(); + let flat = cross_entropy_rate_backend( + data, + &flat_prefix, + &RateBackend::RosaPlus { max_order: -1 }, + ); + + assert!( + (chained - flat).abs() < 1e-12, + "conditional-chain scoring drifted from concatenated-prefix scoring: chained={chained} flat={flat}" + ); + } + + #[test] + fn datagen_bernoulli_entropy_estimate() { + // Test that estimated entropy is close to theoretical for Bernoulli(0.5) + let p = 0.5; + let theoretical_h = crate::datagen::bernoulli_entropy(p); + assert!((theoretical_h - 1.0).abs() < 1e-10); + + // Generate data and check empirical entropy is close to theoretical + let data = crate::datagen::bernoulli(10000, p, 42); + let estimated_h = empirical_entropy_bytes(&data); + + // Should be close to 1.0 bit (since values are 0 or 1) + assert!( + (estimated_h - theoretical_h).abs() < 0.1, + "estimated H={} should be close to theoretical H={}", + estimated_h, + theoretical_h + ); + } + + #[cfg(feature = "backend-rwkv")] + #[test] + fn rwkv_method_entropy_is_stable_across_calls() { + let method = "cfg:hidden=64,layers=1,intermediate=64,decay_rank=8,a_rank=8,v_rank=8,g_rank=8,seed=21,train=sgd,lr=0.01,stride=1;policy:schedule=0..100:infer"; + let backend = RateBackend::Rwkv7Method { + method: crate::rwkvzip::parse_method_spec(method).expect("rwkv method spec"), + }; + let data = b"rwkv method entropy stability regression sample"; + + let h1 = entropy_rate_backend(data, &backend); + let h2 = entropy_rate_backend(data, &backend); + assert!( + (h1 - h2).abs() < 1e-12, + "rwkv method entropy leaked mutable state across calls: h1={h1}, h2={h2}" + ); + } + + #[cfg(feature = "backend-rwkv")] + #[test] + fn rwkv_method_without_policy_is_accepted_by_public_api() { + let backend = RateBackend::Rwkv7Method { + method: crate::rwkvzip::parse_method_spec("cfg:hidden=64,layers=1,intermediate=64") + .expect("rwkv method spec"), + }; + let data = b"rwkv method without policy"; + let h1 = entropy_rate_backend(data, &backend); + let h2 = biased_entropy_rate_backend(data, &backend); + assert!(h1.is_finite()); + assert!(h2.is_finite()); + } + + #[cfg(feature = "backend-rwkv")] + #[test] + fn rwkv_infer_only_plugin_collapses_to_single_pass_entropy() { + let backend = RateBackend::Rwkv7Method { + method: crate::rwkvzip::parse_method_spec("cfg:hidden=64,layers=1,intermediate=64,decay_rank=8,a_rank=8,v_rank=8,g_rank=8,seed=25,train=none,lr=0.0,stride=1;policy:schedule=0..100:infer").expect("rwkv method spec"), + }; + let data = b"rwkv infer-only plugin equality sample"; + let h = entropy_rate_backend(data, &backend); + let plugin = biased_entropy_rate_backend(data, &backend); + assert!( + (h - plugin).abs() < 1e-12, + "infer-only rwkv plugin should equal single-pass entropy: h={h}, plugin={plugin}" + ); + } + + #[cfg(feature = "backend-rwkv")] + #[test] + fn rwkv_method_biased_entropy_is_stable_across_calls_with_training_policy() { + let backend = RateBackend::Rwkv7Method { + method: crate::rwkvzip::parse_method_spec("cfg:hidden=64,layers=1,intermediate=64,decay_rank=8,a_rank=8,v_rank=8,g_rank=8,seed=23,train=sgd,lr=0.01,stride=1;policy:schedule=0..100:train(scope=head+bias,opt=sgd,lr=0.01,stride=1,bptt=1,clip=0,momentum=0.0)").expect("rwkv method spec"), + }; + let data = b"rwkv plugin stability sample"; + let h1 = biased_entropy_rate_backend(data, &backend); + let h2 = biased_entropy_rate_backend(data, &backend); + assert!( + (h1 - h2).abs() < 1e-12, + "rwkv method biased entropy leaked mutable state across calls: h1={h1}, h2={h2}" + ); + } + + #[cfg(feature = "backend-rwkv")] + #[test] + fn rwkv_method_conditional_chain_is_stable_across_calls() { + let method = "cfg:hidden=64,layers=1,intermediate=64,decay_rank=8,a_rank=8,v_rank=8,g_rank=8,seed=22,train=sgd,lr=0.01,stride=1;policy:schedule=0..100:infer"; + let ctx = ctx( + RateBackend::Rwkv7Method { + method: crate::rwkvzip::parse_method_spec(method).expect("rwkv method spec"), + }, + default_compression_backend(), + ); + + let prefix = b"universal prior slice"; + let data = b"query payload"; + let h1 = ctx + .try_cross_entropy_conditional_chain(&[prefix.as_slice()], data) + .expect("conditional-chain cross entropy"); + let h2 = ctx + .try_cross_entropy_conditional_chain(&[prefix.as_slice()], data) + .expect("conditional-chain cross entropy"); + assert!( + (h1 - h2).abs() < 1e-12, + "rwkv method conditional chain leaked mutable state across calls: h1={h1}, h2={h2}" + ); + } + + #[cfg(feature = "backend-mamba")] + #[test] + fn mamba_method_without_policy_is_accepted_by_public_api() { + let backend = RateBackend::MambaMethod { + method: crate::mambazip::parse_method_spec("cfg:hidden=64,layers=1,intermediate=96") + .expect("mamba method spec"), + }; + let data = b"mamba method without policy"; + let h1 = entropy_rate_backend(data, &backend); + let h2 = biased_entropy_rate_backend(data, &backend); + assert!(h1.is_finite()); + assert!(h2.is_finite()); + } + + #[cfg(feature = "backend-mamba")] + #[test] + fn mamba_infer_only_plugin_collapses_to_single_pass_entropy() { + let backend = RateBackend::MambaMethod { + method: crate::mambazip::parse_method_spec("cfg:hidden=64,layers=1,intermediate=96,state=16,conv=4,dt_rank=16,seed=26,train=none,lr=0.0,stride=1;policy:schedule=0..100:infer").expect("mamba method spec"), + }; + let data = b"mamba infer-only plugin equality sample"; + let h = entropy_rate_backend(data, &backend); + let plugin = biased_entropy_rate_backend(data, &backend); + assert!( + (h - plugin).abs() < 1e-12, + "infer-only mamba plugin should equal single-pass entropy: h={h}, plugin={plugin}" + ); + } + + #[cfg(feature = "backend-mamba")] + #[test] + fn mamba_method_biased_entropy_is_stable_across_calls_with_training_policy() { + let backend = RateBackend::MambaMethod { + method: crate::mambazip::parse_method_spec("cfg:hidden=64,layers=1,intermediate=96,state=16,conv=4,dt_rank=16,seed=24,train=sgd,lr=0.01,stride=1;policy:schedule=0..100:train(scope=head+bias,opt=sgd,lr=0.01,stride=1,bptt=1,clip=0,momentum=0.0)").expect("mamba method spec"), + }; + let data = b"mamba plugin stability sample"; + let h1 = biased_entropy_rate_backend(data, &backend); + let h2 = biased_entropy_rate_backend(data, &backend); + assert!( + (h1 - h2).abs() < 1e-12, + "mamba method biased entropy leaked mutable state across calls: h1={h1}, h2={h2}" + ); + } + + #[test] + fn particle_entropy_rate_in_valid_range() { + let rb = test_particle_backend(); + let data = b"hello world particle backend test"; + let rate = entropy_rate_backend(data, &rb); + assert!( + rate > 0.0 && rate < 8.0, + "particle entropy rate out of (0, 8) range: {rate}" + ); + } + + #[test] + fn particle_cross_entropy_stability() { + let rb = test_particle_backend(); + let train = b"ABCABC"; + let test = b"ABC"; + let h1 = cross_entropy_rate_backend(test, train, &rb); + let h2 = cross_entropy_rate_backend(test, train, &rb); + assert!( + (h1 - h2).abs() < 1e-12, + "particle cross entropy not deterministic: h1={h1}, h2={h2}" + ); + } + + #[test] + fn particle_empty_input() { + let rb = RateBackend::Particle { + spec: Arc::new(ParticleSpec::default()), + }; + let rate = entropy_rate_backend(b"", &rb); + assert!( + rate == 0.0, + "particle entropy rate for empty input should be 0.0, got {rate}" + ); + } + + #[test] + fn particle_joint_entropy_rate() { + let rb = test_particle_backend(); + let x = b"AAAA"; + let y = b"BBBB"; + let joint = joint_entropy_rate_backend(x, y, &rb); + assert!( + joint > 0.0 && joint < 16.0, + "particle joint entropy rate out of range: {joint}" + ); + } +} + +#[cfg(all( + test, + not(any( + feature = "all-backends", + feature = "backend-rosa", + feature = "backend-ctw", + feature = "backend-match", + feature = "backend-ppmd", + feature = "backend-sequitur", + feature = "backend-mixture", + feature = "backend-particle", + feature = "backend-calibrated", + feature = "backend-rwkv", + feature = "backend-mamba" + )) +))] +mod minimal_tests { + #[cfg(not(feature = "backend-zpaq"))] + use crate::api::CompressionBackend; + use crate::api::RateBackend; + + #[cfg(not(feature = "backend-zpaq"))] + #[test] + fn explicit_zpaq_backend_fails_to_compile_without_feature() { + let backend = CompressionBackend::zpaq("5"); + let err = backend + .compile() + .err() + .expect("zpaq backend should fail loudly at compile boundary"); + assert!( + err.to_string() + .contains("requires infotheory feature 'backend-zpaq'"), + "unexpected error: {err}" + ); + } + + #[cfg(not(feature = "backend-zpaq"))] + #[test] + fn default_rate_backend_selection_fails_when_no_rate_backends_are_enabled() { + let err = match RateBackend::try_default() { + Ok(_) => panic!("no default rate backend should exist"), + Err(err) => err, + }; + assert!( + err.to_string() + .contains("no default rate backend is available in this build"), + "unexpected error: {err}" + ); + } + + #[cfg(feature = "backend-zpaq")] + #[test] + fn default_rate_backend_is_zpaq_when_zpaq_is_enabled() { + let backend = RateBackend::try_default() + .expect("zpaq-enabled minimal build should expose a default rate backend"); + assert!( + matches!(backend, RateBackend::Zpaq { .. }), + "expected zpaq default backend variant" + ); + } + + #[cfg(not(feature = "backend-zpaq"))] + #[test] + fn default_compression_backend_selection_fails_when_no_rate_backends_are_enabled() { + let err = match CompressionBackend::try_default() { + Ok(_) => panic!("no default compression backend exists"), + Err(err) => err, + }; + assert!( + err.to_string() + .contains("no default rate backend is available in this build"), + "unexpected error: {err}" + ); + } +} diff --git a/crates/infotheory/src/main.rs b/crates/infotheory/src/main.rs new file mode 100644 index 00000000..5b6407c5 --- /dev/null +++ b/crates/infotheory/src/main.rs @@ -0,0 +1,2566 @@ +//! # InfoTheory CLI +//! +//! Command-line interface for the `infotheory` library. +//! Provides access to compression-based (NCD) and entropy-based (Shannon, ROSA, CTW) +//! estimators for files, as well as AIXI agents. +//! +//! ## Usage +//! +//! ### Single-file mode: +//! ```bash +//! infotheory +//! ``` +//! +//! ### Search mode: +//! ```bash +//! infotheory search [options] +//! ``` +//! +//! ### AIXI Agent mode: +//! ```bash +//! infotheory aixi +//! ``` +//! +//! ### Batch JSON mode (for programmatic use): +//! ```bash +//! infotheory batch < input.json > output.json +//! echo '{"op":"metrics","text":"hello world"}' | infotheory batch +//! ``` +//! +//! See `print_usage` for details on supported primitives. + +mod cli; + +#[cfg(test)] +use infotheory::aixi::common::{ActionAlphabet, ObservationKeyMode}; +#[cfg(test)] +use infotheory::aixi::environment::Environment; +#[cfg(all(test, feature = "backend-ctw"))] +use infotheory::aixi::planner_agent::{PlannerEnvironment, PlannerSchedule}; +#[cfg(test)] +use infotheory::aixi::planner_agent::{ + load_warmstart_exact_jh_teacher_dataset, validate_action_alphabet, +}; +#[cfg(all(test, feature = "vm"))] +use infotheory::aixi::vm_nyx::{ + FuzzMutator as NyxFuzzMutator, NyxActionFilter, NyxActionSource, NyxActionSpec, NyxFuzzConfig, + NyxObservationPolicy, NyxObservationStreamMode, NyxProtocolConfig, NyxRewardPolicy, + NyxRewardShaping, NyxTraceConfig, PayloadEncoding as NyxPayloadEncoding, +}; +use infotheory::api::*; +#[cfg(feature = "backend-mamba")] +use infotheory::mambazip; +#[cfg(feature = "backend-rwkv")] +use infotheory::rwkvzip; +#[cfg(feature = "backend-sequitur")] +use infotheory::sequitur::{CanonicalSymbol, SequiturModel}; +#[cfg(all(test, feature = "backend-ctw"))] +use infotheory::spec::{CompiledPlannerRunSpec, SpecDocument}; +#[cfg(all(test, feature = "vm"))] +use nyx_lite::SharedMemoryPolicy; +use std::env; +#[cfg(all(feature = "backend-ctw", feature = "research-tooling"))] +use std::fs::File; +#[cfg(all(feature = "backend-ctw", feature = "research-tooling"))] +use std::io::BufWriter; +use std::io::{self, BufRead, IsTerminal, Read, Write}; +use std::path::Path; +#[cfg(all(feature = "backend-ctw", feature = "research-tooling"))] +use std::time::Instant; + +#[cfg(all(test, feature = "all-backends"))] +use crate::cli::load_expert_spec; +#[cfg(all(test, feature = "vm"))] +use crate::cli::parse_vm_stats_backend; +use crate::cli::{ + CliBackendInvocation, CliBackendSourceFlags, build_ctx_invocation, + file_roundtrip_compiled_backend, load_mixture_spec, maybe_export_online_model, + parse_compression_backend, parse_rate_backend, read_file, read_stdin_all_for_generate, + run_batch_mode, +}; +#[cfg(feature = "backend-sequitur")] +use crate::cli::{bytes_to_hex, parse_hex_bytes}; +#[cfg(test)] +use crate::cli::{ + file_roundtrip_backend, parse_observation_key_mode, parse_observation_key_mode_for_env, + parse_observation_key_mode_for_vm, parse_observation_key_mode_str, + parse_observation_stream_len, parse_observation_stream_len_for_env, + parse_observation_stream_len_for_vm, process_json_line, validate_observation_config, +}; +#[cfg(feature = "backend-rosa")] +use infotheory::search; +#[cfg(feature = "tuner")] +use infotheory::tuner; + +#[track_caller] +fn cli_unwrap(result: Result, context: &str) -> T { + result.unwrap_or_else(|err| panic!("{context} failed: {err}")) +} + +fn ncd_bytes_backend( + x: &[u8], + y: &[u8], + backend: &CompiledCompressionBackend, + variant: NcdVariant, +) -> f64 { + cli_unwrap( + try_ncd_bytes_backend(x, y, backend, variant), + "ncd_bytes_backend", + ) +} + +fn intrinsic_dependence_bytes(data: &[u8]) -> f64 { + cli_unwrap( + try_intrinsic_dependence_bytes(data), + "intrinsic_dependence_bytes", + ) +} + +fn mutual_information_bytes(x: &[u8], y: &[u8]) -> f64 { + cli_unwrap( + try_mutual_information_bytes(x, y), + "mutual_information_bytes", + ) +} + +fn conditional_entropy_bytes(x: &[u8], y: &[u8]) -> f64 { + cli_unwrap( + try_conditional_entropy_bytes(x, y), + "conditional_entropy_bytes", + ) +} + +fn cross_entropy_bytes(test_data: &[u8], train_data: &[u8]) -> f64 { + cli_unwrap( + try_cross_entropy_bytes(test_data, train_data), + "cross_entropy_bytes", + ) +} + +fn joint_entropy_rate_bytes(x: &[u8], y: &[u8]) -> f64 { + cli_unwrap( + try_joint_entropy_rate_bytes(x, y), + "joint_entropy_rate_bytes", + ) +} + +fn resistance_to_transformation_bytes(x: &[u8], tx: &[u8]) -> f64 { + cli_unwrap( + try_resistance_to_transformation_bytes(x, tx), + "resistance_to_transformation_bytes", + ) +} + +fn ned_bytes(x: &[u8], y: &[u8]) -> f64 { + cli_unwrap(try_ned_bytes(x, y), "ned_bytes") +} + +fn ned_cons_bytes(x: &[u8], y: &[u8]) -> f64 { + cli_unwrap(try_ned_cons_bytes(x, y), "ned_cons_bytes") +} + +fn nte_bytes(x: &[u8], y: &[u8]) -> f64 { + cli_unwrap(try_nte_bytes(x, y), "nte_bytes") +} + +fn tvd_paths(x: &str, y: &str) -> f64 { + cli_unwrap(try_tvd_paths(x, y), "tvd_paths") +} + +fn nhd_paths(x: &str, y: &str) -> f64 { + cli_unwrap(try_nhd_paths(x, y), "nhd_paths") +} + +fn kl_divergence_paths(x: &str, y: &str) -> f64 { + cli_unwrap(try_kl_divergence_paths(x, y), "kl_divergence_paths") +} + +fn js_divergence_paths(x: &str, y: &str) -> f64 { + cli_unwrap(try_js_divergence_paths(x, y), "js_divergence_paths") +} + +#[cfg(all(test, feature = "all-backends"))] +fn parse_mixture_kind(kind: &str) -> anyhow::Result { + infotheory::api::parse_mixture_kind_name(kind).map_err(anyhow::Error::msg) +} + +#[cfg(all(test, feature = "all-backends"))] +fn parse_mixture_schedule(schedule: &str) -> anyhow::Result { + infotheory::api::parse_mixture_schedule_name(schedule).map_err(anyhow::Error::msg) +} + +#[cfg(all(test, feature = "all-backends"))] +fn parse_mixture_spec_value( + v: &serde_json::Value, + base_dir: &Path, + depth: usize, +) -> anyhow::Result { + infotheory::spec::parse_mixture_spec_value(v, base_dir, depth).map_err(anyhow::Error::msg) +} + +#[cfg(all( + test, + any( + feature = "all-backends", + feature = "backend-mamba", + feature = "backend-rwkv" + ) +))] +fn parse_mixture_expert_value( + v: &serde_json::Value, + base_dir: &Path, + depth: usize, +) -> anyhow::Result { + infotheory::spec::parse_mixture_expert_value(v, base_dir, depth).map_err(anyhow::Error::msg) +} + +#[cfg(feature = "tuner")] +fn run_tune_mode(args: &[String]) { + match tuner::parse_tune_command_args(args).and_then(|request| tuner::run_tune(&request)) { + Ok(()) => {} + Err(err) => { + eprintln!("Error: tune failed: {err}"); + std::process::exit(1); + } + } +} + +#[cfg(not(feature = "tuner"))] +fn run_tune_mode(_args: &[String]) { + eprintln!("Error: 'tune' requires infotheory built with feature 'tuner'"); + std::process::exit(1); +} + +#[cfg(feature = "tuner")] +fn run_tuner_eval_worker_mode() { + if let Err(err) = tuner::run_tuner_eval_worker_from_env() { + eprintln!("Error: tuner evaluator worker failed: {err}"); + std::process::exit(1); + } +} + +#[cfg(not(feature = "tuner"))] +fn run_tuner_eval_worker_mode() { + eprintln!("Error: tuner evaluator worker requires infotheory built with feature 'tuner'"); + std::process::exit(1); +} + +fn run_warmstart_mode(args: &[String]) { + match crate::cli::warmstart::parse_warmstart_command(args) + .and_then(crate::cli::warmstart::run_warmstart_command) + { + Ok(()) => {} + Err(err) => { + eprintln!("Error: warmstart failed: {err}"); + std::process::exit(1); + } + } +} + +#[cfg(feature = "backend-rosa")] +fn search_command(args: &[String]) { + if args.len() < 4 { + eprintln!("Error: 'search' requires query and target path."); + std::process::exit(1); + } + let query = &args[2]; + let target = &args[3]; + + // Preserve the legacy behavior (and avoid extra parsing work) when no flags are given. + if args.len() == 4 { + if let Err(err) = search::run_search(query, target) { + eprintln!("Error: search failed: {err}"); + std::process::exit(1); + } + return; + } + + let mut opts = match search::SearchOptions::try_default() { + Ok(opts) => opts, + Err(err) => { + eprintln!("Error: search defaults unavailable in this build: {err}"); + std::process::exit(1); + } + }; + let mut rate_backend = infotheory::search::DEFAULT_SEARCH_RATE_BACKEND_NAME.to_string(); + let compression_backend = + infotheory::search::DEFAULT_SEARCH_COMPRESSION_BACKEND_NAME.to_string(); + let mut method: Option = None; + let mut expert_spec_path: Option = None; + let mut rate_backend_json_path: Option = None; + let mut compression_backend_json_path: Option = None; + let mut explicit_rate_backend_flag: bool = false; + let explicit_compression_backend_flag: bool = false; + let mut explicit_method_flag: bool = false; + let mut stage2_prior_mode: Option = None; + let mut msb_first_flag: bool = false; + let mut lsb_first_flag: bool = false; + + let mut i = 4usize; + while i < args.len() { + match args[i].as_str() { + "--level" => { + i += 1; + let v = args + .get(i) + .unwrap_or_exit("Error: --level requires snippet|file"); + opts.granularity = if v == "snippet" { + search::SearchGranularity::Snippet + } else { + search::SearchGranularity::File + }; + } + "--prior" => { + i += 1; + opts.universal_prior = args.get(i).cloned(); + } + "--top-k" => { + i += 1; + opts.top_k = args.get(i).and_then(|s| s.parse().ok()).unwrap_or(10); + } + "--rate-backend" => { + i += 1; + let v = args + .get(i) + .unwrap_or_exit("Error: --rate-backend requires a value"); + rate_backend = parse_rate_backend_flag_or_exit(v, "--rate-backend"); + explicit_rate_backend_flag = true; + } + "--rate-backend-json" => { + i += 1; + let v = args + .get(i) + .unwrap_or_exit("Error: --rate-backend-json requires a path"); + rate_backend_json_path = Some(v.clone()); + } + "--compression-backend-json" => { + i += 1; + let v = args + .get(i) + .unwrap_or_exit("Error: --compression-backend-json requires a path"); + compression_backend_json_path = Some(v.clone()); + } + "--method" => { + i += 1; + method = args.get(i).cloned(); + explicit_method_flag = true; + } + "--expert-spec" => { + i += 1; + expert_spec_path = args.get(i).cloned(); + explicit_rate_backend_flag = true; + } + "--stage2-prior-mode" => { + i += 1; + if let Some(v) = args.get(i) { + stage2_prior_mode = match v.as_str() { + "none" | "no-prior" => Some(search::Stage2PriorMode::Disable), + "summarize" | "summarize-prior" => Some(search::Stage2PriorMode::Summarize), + "use" | "use-prior" => Some(search::Stage2PriorMode::Use), + _ => Some(search::Stage2PriorMode::Use), + }; + } + } + "--msb-first" => { + msb_first_flag = true; + } + "--lsb-first" => { + lsb_first_flag = true; + } + _ => { + i += 1; + } + } + i += 1; + } + if let Some(mode) = stage2_prior_mode { + opts.stage2_prior_mode = mode; + } + opts.ctx = build_ctx_invocation(CliBackendInvocation { + rate_backend: &rate_backend, + compression_backend: &compression_backend, + method: method.as_deref(), + expert_spec_path: expert_spec_path.as_deref(), + rate_backend_json_path: rate_backend_json_path.as_deref(), + compression_backend_json_path: compression_backend_json_path.as_deref(), + fac_ctw_msb_first: parse_fac_ctw_bit_order_flags(msb_first_flag, lsb_first_flag), + flags: CliBackendSourceFlags { + explicit_rate_backend: explicit_rate_backend_flag, + explicit_compression_backend: explicit_compression_backend_flag, + explicit_method: explicit_method_flag, + }, + }) + .ctx; + if let Err(err) = search::run_search_with_options(query, target, &opts) { + eprintln!("Error: search failed: {err}"); + std::process::exit(1); + } +} + +#[cfg(not(feature = "backend-rosa"))] +fn search_command(_args: &[String]) { + eprintln!("Error: 'search' requires infotheory built with feature 'backend-rosa'"); + std::process::exit(1); +} + +trait OptionExt { + fn unwrap_or_exit(self, msg: &str) -> T; +} +impl OptionExt for Option { + fn unwrap_or_exit(self, msg: &str) -> T { + self.unwrap_or_else(|| { + eprintln!("{}", msg); + std::process::exit(1); + }) + } +} + +fn parse_rate_backend_flag_or_exit(value: &str, flag_name: &str) -> String { + parse_rate_backend(value) + .map(std::string::ToString::to_string) + .unwrap_or_else(|| { + eprintln!( + "Error: {flag_name} expects a canonical backend name, got '{value}'. If '{value}' is a canonical RateBackend JSON document path, use --rate-backend-json. If '{value}' is a mixture spec path, use --rate-backend mixture --method ." + ); + std::process::exit(1); + }) +} + +fn parse_fac_ctw_bit_order_flags(msb_first_flag: bool, lsb_first_flag: bool) -> Option { + if msb_first_flag && lsb_first_flag { + eprintln!("Error: --msb-first and --lsb-first are mutually exclusive"); + std::process::exit(1); + } + if msb_first_flag { + Some(true) + } else if lsb_first_flag { + Some(false) + } else { + None + } +} + +fn parse_compression_backend_flag_or_exit(value: &str, flag_name: &str) -> String { + parse_compression_backend(value) + .map(std::string::ToString::to_string) + .unwrap_or_else(|| { + eprintln!( + "Error: {flag_name} expects a canonical backend name, got '{value}'. Use --compression-backend-json for a canonical JSON spec path." + ); + std::process::exit(1); + }) +} + +#[cfg(all(feature = "backend-ctw", feature = "research-tooling"))] +fn parse_ctw_profile_size(raw: &str, field: &str) -> anyhow::Result { + let trimmed = raw.trim(); + let lower = trimmed.to_ascii_lowercase(); + let (digits, multiplier): (&str, usize) = if let Some(prefix) = lower.strip_suffix("kib") { + (prefix, 1024) + } else if let Some(prefix) = lower.strip_suffix("mib") { + (prefix, 1024 * 1024) + } else if let Some(prefix) = lower.strip_suffix("gib") { + (prefix, 1024 * 1024 * 1024) + } else if let Some(prefix) = lower.strip_suffix('k') { + (prefix, 1_000) + } else if let Some(prefix) = lower.strip_suffix('m') { + (prefix, 1_000_000) + } else if let Some(prefix) = lower.strip_suffix('g') { + (prefix, 1_000_000_000) + } else { + (trimmed, 1) + }; + let value = digits + .trim() + .parse::() + .map_err(|_| anyhow::anyhow!("{field} must be a non-negative integer size, got '{raw}'"))?; + value + .checked_mul(multiplier) + .ok_or_else(|| anyhow::anyhow!("{field} overflows usize: '{raw}'")) +} + +#[cfg(all(feature = "backend-ctw", feature = "research-tooling"))] +fn parse_ctw_profile_cutpoints(raw: &str) -> anyhow::Result> { + let mut cutpoints = raw + .split(',') + .filter(|part| !part.trim().is_empty()) + .map(|part| parse_ctw_profile_size(part, "--cutpoints")) + .collect::>>()?; + cutpoints.sort_unstable(); + cutpoints.dedup(); + if cutpoints.is_empty() { + anyhow::bail!("--cutpoints must contain at least one byte count"); + } + Ok(cutpoints) +} + +#[cfg(all(feature = "backend-ctw", feature = "research-tooling"))] +fn default_ctw_profile_cutpoints(max_bytes: Option) -> Vec { + let mut cutpoints = Vec::new(); + let mut next = 1_000_000usize; + while next < 1_000_000_000usize { + cutpoints.push(next); + next = next.saturating_mul(2); + } + cutpoints.push(1_000_000_000usize); + if let Some(max_bytes) = max_bytes { + cutpoints.retain(|cutpoint| *cutpoint <= max_bytes); + if cutpoints.last().copied() != Some(max_bytes) { + cutpoints.push(max_bytes); + } + } + cutpoints +} + +#[cfg(all( + feature = "backend-ctw", + feature = "research-tooling", + target_os = "linux" +))] +fn ctw_profile_proc_memory_bytes() -> serde_json::Value { + let Ok(status) = std::fs::read_to_string("/proc/self/status") else { + return serde_json::json!(null); + }; + let mut vm_rss_bytes = None; + let mut vm_hwm_bytes = None; + for line in status.lines() { + let mut parts = line.split_whitespace(); + let Some(key) = parts.next() else { + continue; + }; + let Some(value) = parts.next() else { + continue; + }; + let Ok(kib) = value.parse::() else { + continue; + }; + match key { + "VmRSS:" => vm_rss_bytes = kib.checked_mul(1024), + "VmHWM:" => vm_hwm_bytes = kib.checked_mul(1024), + _ => {} + } + } + serde_json::json!({ + "vm_rss_bytes": vm_rss_bytes, + "vm_hwm_bytes": vm_hwm_bytes, + }) +} + +#[cfg(all( + feature = "backend-ctw", + feature = "research-tooling", + not(target_os = "linux") +))] +fn ctw_profile_proc_memory_bytes() -> serde_json::Value { + serde_json::json!(null) +} + +#[cfg(all(feature = "backend-ctw", feature = "research-tooling"))] +fn ctw_profile_tree_json(tree: &infotheory::ctw::FacContextTreeTreeTelemetry) -> serde_json::Value { + serde_json::json!({ + "bit_index": tree.bit_index, + "max_depth": tree.max_depth, + "root_visits": tree.root_visits, + "nodes_len": tree.nodes_len, + "nodes_capacity": tree.nodes_capacity, + "segments_len": tree.segments_len, + "segments_capacity": tree.segments_capacity, + "free_nodes_len": tree.free_nodes_len, + "free_nodes_capacity": tree.free_nodes_capacity, + "free_segments_len": tree.free_segments_len, + "free_segments_capacity": tree.free_segments_capacity, + "node_bytes": tree.node_bytes, + "node_payload_bytes": tree.node_payload_bytes, + "segment_bytes": tree.segment_bytes, + "segment_payload_bytes": tree.segment_payload_bytes, + "free_list_bytes": tree.free_list_bytes, + "scratch_bytes": tree.scratch_bytes, + "total_bytes": tree.total_bytes, + "arena_slack_bytes": tree.arena_slack_bytes, + "exact_segments": tree.exact_segments, + "history_segments": tree.history_segments, + "history_invert_segments": tree.history_invert_segments, + "const_segments": tree.const_segments, + "segment_bits": tree.segment_bits, + "max_segment_len": tree.max_segment_len, + }) +} + +#[cfg(all(feature = "backend-ctw", feature = "research-tooling"))] +fn ctw_profile_snapshot_json( + mode: &str, + depth: usize, + bytes_seen: usize, + log_prob: Option, + elapsed_seconds: f64, + telemetry: &infotheory::ctw::FacContextTreeTelemetry, +) -> serde_json::Value { + let bits = log_prob.map(|value| -value / std::f64::consts::LN_2); + let bits_per_byte = bits.and_then(|value| { + if bytes_seen == 0 { + None + } else { + Some(value / bytes_seen as f64) + } + }); + serde_json::json!({ + "kind": "ctw_profile_snapshot", + "mode": mode, + "depth": depth, + "bytes_seen": bytes_seen, + "elapsed_seconds": elapsed_seconds, + "log_probability": log_prob, + "bits": bits, + "bits_per_byte": bits_per_byte, + "rss": ctw_profile_proc_memory_bytes(), + "telemetry": { + "base_depth": telemetry.base_depth, + "num_bits": telemetry.num_bits, + "shared_history_len_bits": telemetry.shared_history_len_bits, + "shared_history_capacity_bits": telemetry.shared_history_capacity_bits, + "shared_history_bytes": telemetry.shared_history_bytes, + "shared_history_payload_bytes": telemetry.shared_history_payload_bytes, + "shared_history_slack_bytes": telemetry.shared_history_slack_bytes, + "shared_log_cache_bytes": telemetry.shared_log_cache_bytes, + "tree_bytes": telemetry.tree_bytes, + "tree_payload_bytes": telemetry.tree_payload_bytes, + "tree_arena_slack_bytes": telemetry.tree_arena_slack_bytes, + "total_bytes": telemetry.total_bytes, + "total_slack_bytes": telemetry.total_slack_bytes, + "nodes_len": telemetry.nodes_len, + "nodes_capacity": telemetry.nodes_capacity, + "segments_len": telemetry.segments_len, + "segments_capacity": telemetry.segments_capacity, + "free_nodes_len": telemetry.free_nodes_len, + "free_segments_len": telemetry.free_segments_len, + "exact_segments": telemetry.exact_segments, + "history_segments": telemetry.history_segments, + "history_invert_segments": telemetry.history_invert_segments, + "const_segments": telemetry.const_segments, + "segment_bits": telemetry.segment_bits, + "max_segment_len": telemetry.max_segment_len, + "trees": telemetry.trees.iter().map(ctw_profile_tree_json).collect::>(), + }, + }) +} + +#[cfg(all(feature = "backend-ctw", feature = "research-tooling"))] +fn run_ctw_profile_mode(args: &[String]) { + let result = (|| -> anyhow::Result<()> { + let mut input_path: Option = None; + let mut depth: usize = 32; + let mut max_bytes: Option = None; + let mut reserve_symbols: Option = None; + let mut cutpoints: Option> = None; + let mut update_only = false; + let mut i = 2usize; + while i < args.len() { + match args[i].as_str() { + "--update-only" => { + update_only = true; + } + "--depth" => { + i += 1; + let raw = args + .get(i) + .ok_or_else(|| anyhow::anyhow!("--depth requires a value"))?; + depth = raw + .parse::() + .map_err(|_| anyhow::anyhow!("--depth must be a non-negative integer"))?; + } + "--max-bytes" => { + i += 1; + let raw = args + .get(i) + .ok_or_else(|| anyhow::anyhow!("--max-bytes requires a value"))?; + max_bytes = Some(parse_ctw_profile_size(raw, "--max-bytes")?); + } + "--reserve-symbols" => { + i += 1; + let raw = args + .get(i) + .ok_or_else(|| anyhow::anyhow!("--reserve-symbols requires a value"))?; + reserve_symbols = Some(parse_ctw_profile_size(raw, "--reserve-symbols")?); + } + "--cutpoints" => { + i += 1; + let raw = args + .get(i) + .ok_or_else(|| anyhow::anyhow!("--cutpoints requires a value"))?; + cutpoints = Some(parse_ctw_profile_cutpoints(raw)?); + } + flag if flag.starts_with("--") => { + anyhow::bail!("unknown ctw-profile option '{flag}'"); + } + value => { + if input_path.is_some() { + anyhow::bail!("ctw-profile accepts exactly one input path or '-'"); + } + input_path = Some(value.to_string()); + } + } + i += 1; + } + + let input_path = input_path + .ok_or_else(|| anyhow::anyhow!("ctw-profile requires an input path or '-'"))?; + let mut cutpoints = cutpoints.unwrap_or_else(|| default_ctw_profile_cutpoints(max_bytes)); + cutpoints.sort_unstable(); + cutpoints.dedup(); + + let mut tree = infotheory::ctw::FacContextTree::new(depth, 8); + if let Some(symbols) = reserve_symbols { + tree.reserve_for_symbols(symbols); + } + + let stdin = io::stdin(); + let mut source: Box = if input_path == "-" { + Box::new(stdin.lock()) + } else { + Box::new(File::open(&input_path)?) + }; + let mut reader = io::BufReader::with_capacity(1 << 20, &mut source); + let mut out = BufWriter::new(io::stdout().lock()); + let started = Instant::now(); + let mut buf = [0u8; 1 << 20]; + let mut bytes_seen: usize = 0; + let mut log_prob = 0.0f64; + let mut cutpoint_index = 0usize; + let mode = if update_only { + "update_only" + } else { + "log_prob_update" + }; + + let initial = tree.telemetry(); + writeln!( + out, + "{}", + ctw_profile_snapshot_json( + mode, + depth, + bytes_seen, + (!update_only).then_some(log_prob), + 0.0, + &initial, + ) + )?; + + 'outer: loop { + let n = reader.read(&mut buf)?; + if n == 0 { + break; + } + for &byte in &buf[..n] { + if max_bytes.is_some_and(|limit| bytes_seen >= limit) { + break 'outer; + } + if update_only { + tree.update_byte_msb(byte); + } else { + log_prob += tree.log_prob_update_byte_msb(byte); + } + bytes_seen = bytes_seen.saturating_add(1); + while cutpoint_index < cutpoints.len() && bytes_seen >= cutpoints[cutpoint_index] { + let telemetry = tree.telemetry(); + writeln!( + out, + "{}", + ctw_profile_snapshot_json( + mode, + depth, + bytes_seen, + (!update_only).then_some(log_prob), + started.elapsed().as_secs_f64(), + &telemetry, + ) + )?; + out.flush()?; + cutpoint_index += 1; + } + } + } + + if cutpoints.last().copied() != Some(bytes_seen) { + let telemetry = tree.telemetry(); + writeln!( + out, + "{}", + ctw_profile_snapshot_json( + mode, + depth, + bytes_seen, + (!update_only).then_some(log_prob), + started.elapsed().as_secs_f64(), + &telemetry, + ) + )?; + } + out.flush()?; + Ok(()) + })(); + + if let Err(err) = result { + eprintln!("Error: ctw-profile failed: {err:#}"); + std::process::exit(1); + } +} + +#[cfg(not(all(feature = "backend-ctw", feature = "research-tooling")))] +fn run_ctw_profile_mode(_args: &[String]) { + eprintln!( + "Error: 'ctw-profile' requires infotheory built with features 'backend-ctw research-tooling'" + ); + std::process::exit(1); +} + +fn main() { + let args: Vec = env::args().collect(); + + if args.len() > 1 && (args[1] == "--help" || args[1] == "-h") { + print_usage(); + return; + } + + if args.len() < 2 { + print_usage(); + return; + } + + let primitive = &args[1]; + if primitive == "help" { + if let Some(topic) = args.get(2) { + print_topic_usage(topic); + } else { + print_usage(); + } + return; + } + if args + .iter() + .skip(2) + .any(|arg| arg == "--help" || arg == "-h") + { + print_topic_usage(primitive); + return; + } + if primitive == "__infotheory-tuner-eval-worker" { + run_tuner_eval_worker_mode(); + return; + } + if primitive == "batch" { + run_batch_mode(); + return; + } + if primitive == "tune" { + run_tune_mode(&args); + return; + } + if primitive == "ctw-profile" || primitive == "ctw_profile" { + run_ctw_profile_mode(&args); + return; + } + if primitive == "warmstart" { + run_warmstart_mode(&args); + return; + } + + // Common positional and flag parsing. + // Collect positionals only up to the first flag token, then parse flags separately. + let mut file1: Option = None; + let mut file2: Option = None; + let mut pos_arg3: Option = None; + let mut flags_start = 2usize; + + if primitive != "search" && primitive != "aixi" { + let mut positionals: Vec = Vec::new(); + let mut i = 2usize; + while i < args.len() { + let tok = &args[i]; + if tok.starts_with('-') { + break; + } + positionals.push(tok.clone()); + i += 1; + } + flags_start = i; + file1 = positionals.first().cloned(); + file2 = positionals.get(1).cloned(); + pos_arg3 = positionals.get(2).cloned(); + } + + let mut rate_backend_str = "rosaplus".to_string(); + let mut compression_backend_str = "zpaq".to_string(); + let mut method_str: Option = None; + let mut expert_spec_path: Option = None; + let mut rate_backend_json_path: Option = None; + let mut compression_backend_json_path: Option = None; + let mut explicit_rate_backend_flag: bool = false; + let mut explicit_compression_backend_flag: bool = false; + let mut explicit_method_flag: bool = false; + let mut model_export_path: Option = None; + let mut diagnostic_mixture_path: Option = None; + let mut diagnostic_out_prefix: Option = None; + let mut sequitur_debug_hexes: Vec = Vec::new(); + #[cfg(feature = "backend-sequitur")] + let mut sequitur_context_bytes: usize = 64; + #[cfg(not(feature = "backend-sequitur"))] + let _sequitur_context_bytes: usize = 64; + #[cfg(feature = "backend-sequitur")] + let mut sequitur_alphabet_prefix: usize = 4; + #[cfg(not(feature = "backend-sequitur"))] + let _sequitur_alphabet_prefix: usize = 4; + let mut generate_len_bytes: usize = 8; + let mut generate_config = GenerationConfig::default(); + let mut rate_backend_specified = false; + let mut msb_first_flag: bool = false; + let mut lsb_first_flag: bool = false; + + let mut i = flags_start; + while i < args.len() { + match args[i].as_str() { + "--rate-backend" => { + i += 1; + let v = args + .get(i) + .unwrap_or_exit("Error: --rate-backend requires a value"); + rate_backend_str = parse_rate_backend_flag_or_exit(v, "--rate-backend"); + rate_backend_specified = true; + explicit_rate_backend_flag = true; + } + "--rate-backend-json" => { + i += 1; + let v = args + .get(i) + .unwrap_or_exit("Error: --rate-backend-json requires a path"); + rate_backend_json_path = Some(v.clone()); + rate_backend_specified = true; + } + "--compression-backend-json" => { + i += 1; + let v = args + .get(i) + .unwrap_or_exit("Error: --compression-backend-json requires a path"); + compression_backend_json_path = Some(v.clone()); + } + "--compression-backend" => { + i += 1; + let v = args + .get(i) + .unwrap_or_exit("Error: --compression-backend requires a value"); + compression_backend_str = + parse_compression_backend_flag_or_exit(v, "--compression-backend"); + explicit_compression_backend_flag = true; + } + "--method" => { + i += 1; + method_str = args.get(i).cloned(); + explicit_method_flag = true; + } + "--expert-spec" => { + i += 1; + expert_spec_path = args.get(i).cloned(); + rate_backend_specified = true; + explicit_rate_backend_flag = true; + } + "--msb-first" => { + msb_first_flag = true; + } + "--lsb-first" => { + lsb_first_flag = true; + } + "--model-export" => { + i += 1; + model_export_path = args.get(i).cloned(); + } + "--rwkv-export" => { + eprintln!("Error: --rwkv-export has been removed; use --model-export instead"); + std::process::exit(1); + } + "--mixture" => { + i += 1; + diagnostic_mixture_path = args.get(i).cloned(); + } + "--out-prefix" => { + i += 1; + diagnostic_out_prefix = args.get(i).cloned(); + } + "--hex" => { + i += 1; + if let Some(value) = args.get(i) { + sequitur_debug_hexes.push(value.clone()); + } + } + "--context-bytes" => { + i += 1; + let raw = args + .get(i) + .unwrap_or_exit("Error: --context-bytes requires a positive integer"); + let parsed = raw.parse::().unwrap_or_else(|_| { + eprintln!("Error: --context-bytes must be a positive integer, got '{raw}'"); + std::process::exit(1); + }); + #[cfg(feature = "backend-sequitur")] + { + sequitur_context_bytes = parsed; + } + #[cfg(not(feature = "backend-sequitur"))] + { + let _ = parsed; + } + } + "--alphabet-prefix" => { + i += 1; + let raw = args + .get(i) + .unwrap_or_exit("Error: --alphabet-prefix requires a positive integer"); + let parsed = raw.parse::().unwrap_or_else(|_| { + eprintln!("Error: --alphabet-prefix must be a positive integer, got '{raw}'"); + std::process::exit(1); + }); + #[cfg(feature = "backend-sequitur")] + { + sequitur_alphabet_prefix = parsed; + } + #[cfg(not(feature = "backend-sequitur"))] + { + let _ = parsed; + } + } + "--bytes" => { + i += 1; + let raw = args + .get(i) + .unwrap_or_exit("Error: --bytes requires a non-negative integer"); + generate_len_bytes = raw.parse::().unwrap_or_else(|_| { + eprintln!("Error: --bytes must be a non-negative integer, got '{raw}'"); + std::process::exit(1); + }); + } + "--sample" => { + generate_config.strategy = GenerationStrategy::Sample; + } + "--greedy" => { + generate_config.strategy = GenerationStrategy::Greedy; + } + "--adaptive" => { + generate_config.update_mode = GenerationUpdateMode::Adaptive; + } + "--seed" => { + i += 1; + let raw = args + .get(i) + .unwrap_or_exit("Error: --seed requires an unsigned integer"); + generate_config.seed = raw.parse::().unwrap_or_else(|_| { + eprintln!("Error: --seed must be an unsigned integer, got '{raw}'"); + std::process::exit(1); + }); + generate_config.strategy = GenerationStrategy::Sample; + } + "--temperature" => { + i += 1; + let raw = args + .get(i) + .unwrap_or_exit("Error: --temperature requires a finite number"); + generate_config.temperature = raw.parse::().unwrap_or_else(|_| { + eprintln!("Error: --temperature must be a finite number, got '{raw}'"); + std::process::exit(1); + }); + if !generate_config.temperature.is_finite() || generate_config.temperature < 0.0 { + eprintln!( + "Error: --temperature must be finite and non-negative, got '{}'", + generate_config.temperature + ); + std::process::exit(1); + } + generate_config.strategy = GenerationStrategy::Sample; + } + "--top-k" => { + i += 1; + let raw = args + .get(i) + .unwrap_or_exit("Error: --top-k requires a non-negative integer"); + generate_config.top_k = raw.parse::().unwrap_or_else(|_| { + eprintln!("Error: --top-k must be a non-negative integer, got '{raw}'"); + std::process::exit(1); + }); + generate_config.strategy = GenerationStrategy::Sample; + } + "--top-p" => { + i += 1; + let raw = args + .get(i) + .unwrap_or_exit("Error: --top-p requires a number in (0, 1]"); + generate_config.top_p = raw.parse::().unwrap_or_else(|_| { + eprintln!("Error: --top-p must be a number in (0, 1], got '{raw}'"); + std::process::exit(1); + }); + if !generate_config.top_p.is_finite() + || generate_config.top_p <= 0.0 + || generate_config.top_p > 1.0 + { + eprintln!( + "Error: --top-p must be in (0, 1], got '{}'", + generate_config.top_p + ); + std::process::exit(1); + } + generate_config.strategy = GenerationStrategy::Sample; + } + _ => {} + } + i += 1; + } + + if primitive == "ac-log-loss" || primitive == "ac_log_loss" { + let input_path = file1.unwrap_or_exit( + "Error: 'ac-log-loss' requires --mixture --out-prefix ", + ); + let mixture_path = diagnostic_mixture_path + .unwrap_or_exit("Error: 'ac-log-loss' requires --mixture "); + let out_prefix = diagnostic_out_prefix + .unwrap_or_exit("Error: 'ac-log-loss' requires --out-prefix "); + let spec = load_mixture_spec(&mixture_path).unwrap_or_else(|e| { + eprintln!( + "Error: failed to load mixture spec '{}': {}", + mixture_path, e + ); + std::process::exit(1); + }); + let data = read_file(&input_path); + match infotheory::diagnostics::run_ac_log_loss_mixture_bytes(&data, &spec, &out_prefix) { + Ok(summary) => { + println!( + "wrote {} rows to {}, nodes to {}, summary to {}", + summary.positions, + summary.trace_path.display(), + summary.nodes_path.display(), + summary.summary_path.display() + ); + } + Err(err) => { + eprintln!("Error: AC log-loss diagnostic failed: {err:#}"); + std::process::exit(1); + } + } + return; + } + + if primitive == "sequitur-debug" || primitive == "sequitur_debug" { + #[cfg(not(feature = "backend-sequitur"))] + { + eprintln!( + "Error: 'sequitur-debug' requires infotheory built with feature 'backend-sequitur'" + ); + std::process::exit(1); + } + #[cfg(feature = "backend-sequitur")] + { + let inputs = if !sequitur_debug_hexes.is_empty() { + sequitur_debug_hexes + .iter() + .map(|raw_hex| { + parse_hex_bytes(raw_hex).unwrap_or_else(|e| { + eprintln!("Error: invalid --hex input for 'sequitur-debug': {e}"); + std::process::exit(1); + }) + }) + .collect::>() + } else { + let input_path = + file1.unwrap_or_exit("Error: 'sequitur-debug' requires or --hex "); + vec![read_file(&input_path)] + }; + let alphabet_prefix = sequitur_alphabet_prefix.clamp(1, 256); + let cases = inputs + .iter() + .map(|data| { + let mut model = SequiturModel::new(sequitur_context_bytes); + let trace = model.predictive_trace(data, alphabet_prefix); + let rules = model + .canonical_grammar() + .rules + .iter() + .map(|rule| { + let rhs = rule + .rhs + .iter() + .map(|sym| match sym { + CanonicalSymbol::Terminal(byte) => { + serde_json::json!(*byte as i64) + } + CanonicalSymbol::NonTerminal(rule_id) => { + serde_json::json!(-((*rule_id as i64) + 1)) + } + }) + .collect::>(); + serde_json::json!({ + "id": rule.id, + "rhs": rhs, + }) + }) + .collect::>(); + serde_json::json!({ + "input_hex": bytes_to_hex(data), + "decoded_hex": bytes_to_hex(&model.decode()), + "rules": rules, + "trace": trace, + }) + }) + .collect::>(); + let output = serde_json::json!({ + "context_bytes": sequitur_context_bytes, + "alphabet_prefix": alphabet_prefix, + "cases": cases, + }); + println!( + "{}", + serde_json::to_string(&output).expect("sequitur debug json serialization") + ); + return; + } + } + + let built_ctx = build_ctx_invocation(CliBackendInvocation { + rate_backend: &rate_backend_str, + compression_backend: &compression_backend_str, + method: method_str.as_deref(), + expert_spec_path: expert_spec_path.as_deref(), + rate_backend_json_path: rate_backend_json_path.as_deref(), + compression_backend_json_path: compression_backend_json_path.as_deref(), + fac_ctw_msb_first: parse_fac_ctw_bit_order_flags(msb_first_flag, lsb_first_flag), + flags: CliBackendSourceFlags { + explicit_rate_backend: explicit_rate_backend_flag, + explicit_compression_backend: explicit_compression_backend_flag, + explicit_method: explicit_method_flag, + }, + }); + let ctx = built_ctx.ctx; + set_default_ctx(ctx.clone()); + + match primitive.as_str() { + "aixi" => { + if let Some(p) = args.get(2) { + if let Err(e) = crate::cli::planner_run::run_aixi_mode(p) { + eprintln!("Error: {}", e); + std::process::exit(1); + } + } else { + eprintln!("Error: 'aixi' requires config.json"); + std::process::exit(1); + } + } + "search" => search_command(&args), + "compress" => { + let in_path = file1.unwrap_or_exit("Error: 'compress' requires "); + let out_path = file2.unwrap_or_exit("Error: 'compress' requires "); + let data = read_file(&in_path); + let backend = file_roundtrip_compiled_backend(&ctx.compression_backend); + let compressed = match infotheory::api::try_compress_bytes_backend(&data, &backend) { + Ok(v) => v, + Err(e) => { + eprintln!("Error: compression failed: {e}"); + std::process::exit(1); + } + }; + if let Err(e) = std::fs::write(&out_path, &compressed) { + eprintln!("Error: failed to write output '{}': {}", out_path, e); + std::process::exit(1); + } + println!( + "compressed {} bytes -> {} bytes", + data.len(), + compressed.len() + ); + if let Err(e) = maybe_export_online_model(model_export_path.as_deref(), &ctx, &[&data]) + { + eprintln!("Error exporting online model: {e}"); + std::process::exit(1); + } + } + "decompress" => { + let in_path = file1.unwrap_or_exit("Error: 'decompress' requires "); + let out_path = file2.unwrap_or_exit("Error: 'decompress' requires "); + let input = read_file(&in_path); + let backend = file_roundtrip_compiled_backend(&ctx.compression_backend); + let decoded = match infotheory::api::try_decompress_bytes_backend(&input, &backend) { + Ok(v) => v, + Err(e) => { + eprintln!("Error: decompression failed: {e}"); + std::process::exit(1); + } + }; + if let Err(e) = std::fs::write(&out_path, &decoded) { + eprintln!("Error: failed to write output '{}': {}", out_path, e); + std::process::exit(1); + } + println!( + "decompressed {} bytes -> {} bytes", + input.len(), + decoded.len() + ); + if let Err(e) = + maybe_export_online_model(model_export_path.as_deref(), &ctx, &[&decoded]) + { + eprintln!("Error exporting online model: {e}"); + std::process::exit(1); + } + } + "generate" => { + let stdin_is_piped = !io::stdin().is_terminal(); + let file_path = match (file1.as_deref(), file2.as_deref()) { + (Some(f), _) if !(stdin_is_piped && f.parse::().is_ok()) => Some(f), + _ => None, + }; + let input = if let Some(path) = file_path { + read_file(path) + } else { + read_stdin_all_for_generate() + }; + let generated = cli_unwrap( + ctx.try_generate_bytes_with_config(&input, generate_len_bytes, generate_config), + "generate_bytes_with_config", + ); + if let Err(e) = io::stdout().write_all(&generated) { + eprintln!("Error writing generated output: {e}"); + std::process::exit(1); + } + if let Err(e) = io::stdout().flush() { + eprintln!("Error flushing generated output: {e}"); + std::process::exit(1); + } + if let Err(e) = maybe_export_online_model(model_export_path.as_deref(), &ctx, &[&input]) + { + eprintln!("Error exporting online model: {e}"); + std::process::exit(1); + } + } + "ncd" | "ncd_vitanyi" | "ncd_sym" | "ncd_sym_vitanyi" | "ncd_cons" | "ncd_sym_cons" => { + let f1 = file1.unwrap_or_exit("Error: NCD requires two files"); + let f2 = file2.unwrap_or_exit("Error: NCD requires two files"); + let _method = pos_arg3.or(method_str).unwrap_or_else(|| "5".to_string()); + let variant = match primitive.as_str() { + "ncd_sym" | "ncd_sym_vitanyi" => NcdVariant::SymVitanyi, + "ncd_cons" => NcdVariant::Cons, + "ncd_sym_cons" => NcdVariant::SymCons, + _ => NcdVariant::Vitanyi, + }; + let b1 = read_file(&f1); + let b2 = read_file(&f2); + println!( + "{}", + ncd_bytes_backend(&b1, &b2, &ctx.compression_backend, variant) + ); + if let Err(e) = + maybe_export_online_model(model_export_path.as_deref(), &ctx, &[&b1, &b2]) + { + eprintln!("Error exporting online model: {e}"); + std::process::exit(1); + } + } + "entropy" | "h" | "entropy_rate" | "h_rate" => { + let f1 = file1.unwrap_or_exit("Error: 'h' requires a file"); + let data = read_file(&f1); + // `h`/`entropy` -> empirical (zero-order, IID) Shannon entropy. + // `h_rate`/`entropy_rate` -> algorithmic entropy rate via active rate backend. + // The `rate_backend_specified` flag promotes `h`/`entropy` to the + // algorithmic path so that `--rate-backend X h file` behaves intuitively. + if !primitive.contains("rate") && !rate_backend_specified { + println!("{}", empirical_entropy_bytes(&data)); + } else { + println!( + "{}", + cli_unwrap( + ctx.try_entropy_rate_bytes(&data), + "InfotheoryCtx::try_entropy_rate_bytes", + ) + ); + } + if let Err(e) = maybe_export_online_model(model_export_path.as_deref(), &ctx, &[&data]) + { + eprintln!("Error exporting online model: {e}"); + std::process::exit(1); + } + } + "id" => { + let f1 = file1.unwrap_or_exit("Error: 'id' requires a file"); + let data = read_file(&f1); + println!("{:.6}", intrinsic_dependence_bytes(&data)); + if let Err(e) = maybe_export_online_model(model_export_path.as_deref(), &ctx, &[&data]) + { + eprintln!("Error exporting online model: {e}"); + std::process::exit(1); + } + } + other => { + let f1 = file1.unwrap_or_exit("Error: requires two files"); + let f2 = file2.unwrap_or_exit("Error: requires two files"); + let b1 = read_file(&f1); + let b2 = read_file(&f2); + // For two-file primitives the `_rate_specified` flag promotes the + // empirical helpers to algorithmic rate-backend variants whenever the + // user explicitly requested a rate backend on the command line. + let res = match other { + "ned" if rate_backend_specified => ned_bytes(&b1, &b2), + "ned" => empirical_ned_bytes(&b1, &b2), + "ned_cons" if rate_backend_specified => ned_cons_bytes(&b1, &b2), + "ned_cons" => empirical_ned_cons_bytes(&b1, &b2), + "nte" if rate_backend_specified => nte_bytes(&b1, &b2), + "nte" => empirical_nte_bytes(&b1, &b2), + "mi" | "mutual_info" if rate_backend_specified => { + mutual_information_bytes(&b1, &b2) + } + "mi" | "mutual_info" => empirical_mutual_information_bytes(&b1, &b2), + "ce" | "conditional_entropy" if rate_backend_specified => { + conditional_entropy_bytes(&b1, &b2) + } + "ce" | "conditional_entropy" => { + let h_xy = empirical_joint_entropy_bytes(&b1, &b2); + let h_y = empirical_entropy_bytes(&b2); + (h_xy - h_y).max(0.0) + } + "xe" | "cross_entropy" if rate_backend_specified => cross_entropy_bytes(&b1, &b2), + "xe" | "cross_entropy" => empirical_cross_entropy_bytes(&b1, &b2), + "joint_entropy" | "h_xy" if rate_backend_specified => { + joint_entropy_rate_bytes(&b1, &b2) + } + "joint_entropy" | "h_xy" => empirical_joint_entropy_bytes(&b1, &b2), + "rt" | "resistance" if rate_backend_specified => { + resistance_to_transformation_bytes(&b1, &b2) + } + "rt" | "resistance" => empirical_resistance_to_transformation_bytes(&b1, &b2), + "tvd" => tvd_paths(&f1, &f2), + "nhd" => nhd_paths(&f1, &f2), + "kl" | "kl_divergence" => kl_divergence_paths(&f1, &f2), + "js" | "js_divergence" => js_divergence_paths(&f1, &f2), + _ => { + eprintln!("Unknown primitive: {}", other); + print_usage(); + return; + } + }; + println!("{}", res); + if let Err(e) = + maybe_export_online_model(model_export_path.as_deref(), &ctx, &[&b1, &b2]) + { + eprintln!("Error exporting online model: {e}"); + std::process::exit(1); + } + } + } +} + +fn print_usage() { + crate::cli::help::print_global_help(); +} + +fn print_topic_usage(topic: &str) { + crate::cli::help::print_topic_help(topic); +} + +#[cfg(test)] +mod tests { + use super::*; + #[cfg(feature = "backend-ctw")] + use crate::cli::planner_run::run_vm_perf_only; + use infotheory::aixi::warmstart_contract::{ + TaskFingerprint, WARMSTART_STANDALONE_OBSERVATION_ADAPTER_SPEC_REF, + WARMSTART_STANDALONE_SCALAR_REPRESENTATION, standalone_teacher_provenance_crc32_pair, + warmstart_exact_jh_planner_task_fingerprint, + }; + use serde_json::json; + use std::path::PathBuf; + use std::sync::atomic::{AtomicU64, Ordering}; + use std::time::{SystemTime, UNIX_EPOCH}; + + static TEMP_TEST_PATH_COUNTER: AtomicU64 = AtomicU64::new(0); + + fn unique_temp_path(prefix: &str, suffix: &str) -> PathBuf { + let counter = TEMP_TEST_PATH_COUNTER.fetch_add(1, Ordering::Relaxed); + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_nanos()) + .unwrap_or(0); + std::env::temp_dir().join(format!( + "{prefix}-{}-{nanos}-{counter}{suffix}", + std::process::id() + )) + } + + #[cfg(any(feature = "backend-mamba", feature = "backend-rwkv"))] + fn canonical_test_path_string(path: &std::path::Path) -> String { + path.to_string_lossy().replace('\\', "/") + } + + #[cfg(feature = "backend-ctw")] + fn action_alphabet(n: usize) -> ActionAlphabet { + ActionAlphabet::try_from_usize(n).expect("test action alphabet must be non-zero") + } + + #[cfg(feature = "backend-ctw")] + fn sample_compiled_planner_run() -> CompiledPlannerRunSpec { + let document = SpecDocument::parse_json_value( + &json!({ + "schema_version": 1, + "kind": "planner_run", + "assets": [], + "environment": { + "kind": "builtin", + "name": "coin_flip" + }, + "interface": { + "observation_bits": 2, + "observation_stream_len": 1, + "observation_key_mode": "full_stream", + "reward_bits": 4, + "agent_actions": action_alphabet(2).get() + }, + "controller": { + "kind": "aiqi_discounted", + "predictor": { + "kind": "ctw", + "depth": 4 + }, + "bit_stream_semantics": { "kind": "binary_tokens" }, + "discount_gamma": 0.5, + "return_horizon": 2, + "return_bins": 8, + "augmentation_period": 2, + "baseline_exploration": 0.1 + }, + "runtime": { + "random_seed": 7, + "learn_cycles": 3, + "eval_cycles": 1, + "terminate_lifetime": 3, + "log_every": 1, + "perf": false, + "vm_perf_only": false, + "explore_epsilon": 0.25, + "explore_gamma": 0.5 + } + }), + Path::new("."), + ) + .expect("sample planner document"); + let SpecDocument::PlannerRun(spec) = document else { + panic!("expected planner_run document"); + }; + spec.compile().expect("sample planner run should compile") + } + + #[cfg(feature = "backend-ctw")] + fn sample_warmstart_compiled_planner_run(teacher_path: &Path) -> CompiledPlannerRunSpec { + let document = SpecDocument::parse_json_value( + &json!({ + "schema_version": 1, + "kind": "planner_run", + "assets": [{ + "id": "teacher", + "path": teacher_path.to_string_lossy() + }], + "environment": { + "kind": "builtin", + "name": "coin_flip" + }, + "interface": { + "observation_bits": 2, + "observation_stream_len": 1, + "observation_key_mode": "full_stream", + "reward_bits": 2, + "agent_actions": action_alphabet(2).get() + }, + "controller": { + "kind": "aiqi_warmstart_exact_jh", + "predictor": { + "kind": "ctw", + "depth": 4 + }, + "return_horizon": 1, + "return_bins": 4, + "label_phase_period": 1, + "teacher_dataset_asset": "teacher", + "planner_simulations_per_step": 1 + }, + "runtime": { + "random_seed": 7, + "learn_cycles": 1, + "eval_cycles": 1, + "terminate_lifetime": 2, + "log_every": 1, + "perf": false, + "vm_perf_only": false, + "explore_epsilon": 0.0, + "explore_gamma": 1.0 + } + }), + Path::new("."), + ) + .expect("sample warmstart planner document"); + let SpecDocument::PlannerRun(spec) = document else { + panic!("expected planner_run document"); + }; + spec.compile() + .expect("sample warmstart planner run should compile") + } + + #[cfg(feature = "backend-ctw")] + fn write_warmstart_teacher( + path: &Path, + task_fingerprint: &TaskFingerprint, + action_alphabet_size: usize, + observation_bits: usize, + ) { + let reward_bits: usize = 2; + let observation_stream_len: usize = 1; + let (adapter_crc, reward_cert) = standalone_teacher_provenance_crc32_pair( + observation_bits, + observation_stream_len, + reward_bits, + ) + .expect("standalone teacher provenance crc pair"); + std::fs::write( + path, + serde_json::to_vec(&json!({ + "schema_version": 1, + "contract": { + "task_fingerprint": task_fingerprint.to_string(), + "action_alphabet_size": action_alphabet_size, + "observation_bits": observation_bits, + "observation_stream_len": observation_stream_len, + "observation_key_mode": "full_stream", + "observation_adapter_spec_ref": WARMSTART_STANDALONE_OBSERVATION_ADAPTER_SPEC_REF, + "observation_adapter_content_crc32": adapter_crc, + "reward_bits": reward_bits, + "return_horizon": 1, + "label_phase_period": 1, + "scalar_representation": WARMSTART_STANDALONE_SCALAR_REPRESENTATION, + "exact_reward_encoding_certificate": reward_cert + }, + "traces": [{ + "transitions": [ + {"action": 0, "observations": [1], "reward": 1} + ] + }] + })) + .expect("serialize warmstart teacher"), + ) + .expect("write warmstart teacher"); + } + + /// Mutate one `contract` string field in an on-disk warm-start teacher JSON file. + #[cfg(feature = "backend-ctw")] + fn mutate_warmstart_teacher_contract_field(path: &Path, field: &str, wrong: &str) { + let bytes = std::fs::read(path).expect("read warmstart teacher"); + let mut doc: serde_json::Value = + serde_json::from_slice(&bytes).expect("parse warmstart teacher json"); + let contract = doc + .as_object_mut() + .and_then(|root| root.get_mut("contract")) + .and_then(|c| c.as_object_mut()) + .expect("teacher.contract object"); + contract.insert(field.to_string(), json!(wrong)); + std::fs::write(path, serde_json::to_vec(&doc).expect("serialize teacher")) + .expect("write warmstart teacher"); + } + + #[cfg(feature = "backend-ctw")] + #[derive(Clone, Copy)] + struct CountingEnv { + observation: u64, + reward: i64, + reward_bits: usize, + action_bits: usize, + observation_bits: usize, + } + + #[cfg(feature = "backend-ctw")] + impl Environment for CountingEnv { + fn perform_action(&mut self, action: u64) { + self.observation = self.observation.saturating_add(action + 1); + self.reward = self.reward.saturating_add(1); + } + + fn get_observation(&self) -> u64 { + self.observation + } + + fn get_reward(&self) -> i64 { + self.reward + } + + fn is_finished(&self) -> bool { + false + } + + fn get_observation_bits(&self) -> usize { + self.observation_bits + } + + fn get_reward_bits(&self) -> usize { + self.reward_bits + } + + fn get_action_bits(&self) -> usize { + self.action_bits + } + } + + #[test] + fn file_roundtrip_backend_keeps_zpaq_unchanged() { + let b = CompressionBackend::zpaq("5"); + let out = file_roundtrip_backend(&b); + assert!(matches!(out, CompressionBackend::Zpaq { method, .. } if method.value() == "5")); + } + + #[cfg(feature = "all-backends")] + #[test] + fn file_roundtrip_backend_forces_rate_framed() { + let b = CompressionBackend::Rate { + rate_backend: RateBackend::Ctw { depth: 8 }, + coder: infotheory::coders::CoderType::AC, + framing: infotheory::compression::FramingMode::Raw, + }; + let out = file_roundtrip_backend(&b); + match out { + CompressionBackend::Rate { framing, .. } => { + assert_eq!(framing, infotheory::compression::FramingMode::Framed) + } + _ => panic!("expected rate backend"), + } + } + + #[test] + fn process_json_line_rejects_invalid_json() { + let out = process_json_line(r#"{"op":"metrics","text":"abc""#); + let parsed: serde_json::Value = serde_json::from_str(&out).expect("output should be json"); + assert!( + parsed + .get("error") + .and_then(|v| v.as_str()) + .unwrap_or("") + .contains("invalid json") + ); + } + + #[cfg(feature = "backend-ctw")] + #[test] + fn warmstart_teacher_loader_accepts_matching_compiled_planner_contract() { + let teacher_path = unique_temp_path("warmstart-teacher-matching", ".json"); + let compiled = sample_warmstart_compiled_planner_run(&teacher_path); + let task_fingerprint = + warmstart_exact_jh_planner_task_fingerprint(&compiled).expect("task fingerprint"); + write_warmstart_teacher(&teacher_path, &task_fingerprint, 2, 2); + + let teacher = load_warmstart_exact_jh_teacher_dataset(&compiled, "teacher") + .expect("matching teacher contract must load"); + assert_eq!(teacher.contract.task_fingerprint, task_fingerprint); + assert_eq!(teacher.traces.len(), 1); + + let _ = std::fs::remove_file(teacher_path); + } + + #[cfg(feature = "backend-ctw")] + #[test] + fn warmstart_teacher_loader_rejects_mismatched_task_fingerprint() { + let teacher_path = unique_temp_path("warmstart-teacher-task-mismatch", ".json"); + let compiled = sample_warmstart_compiled_planner_run(&teacher_path); + write_warmstart_teacher( + &teacher_path, + &TaskFingerprint::parse_hex( + "deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef", + ) + .expect("valid mismatch fingerprint"), + 2, + 2, + ); + + let err = load_warmstart_exact_jh_teacher_dataset(&compiled, "teacher") + .expect_err("mismatched teacher task must fail"); + assert!(err.to_string().contains("task_fingerprint"), "{err}"); + + let _ = std::fs::remove_file(teacher_path); + } + + #[cfg(feature = "backend-ctw")] + #[test] + fn warmstart_teacher_loader_rejects_action_alphabet_mismatch() { + let teacher_path = unique_temp_path("warmstart-teacher-interface-mismatch", ".json"); + let compiled = sample_warmstart_compiled_planner_run(&teacher_path); + let task_fingerprint = + warmstart_exact_jh_planner_task_fingerprint(&compiled).expect("task fingerprint"); + write_warmstart_teacher(&teacher_path, &task_fingerprint, 3, 2); + + let err = load_warmstart_exact_jh_teacher_dataset(&compiled, "teacher") + .expect_err("mismatched action alphabet must fail"); + assert!(err.to_string().contains("action_alphabet_size"), "{err}"); + + let _ = std::fs::remove_file(teacher_path); + } + + #[cfg(feature = "backend-ctw")] + #[test] + fn warmstart_teacher_loader_rejects_observation_adapter_spec_ref_mismatch() { + let teacher_path = unique_temp_path("warmstart-teacher-adapter-ref", ".json"); + let compiled = sample_warmstart_compiled_planner_run(&teacher_path); + let task_fingerprint = + warmstart_exact_jh_planner_task_fingerprint(&compiled).expect("task fingerprint"); + write_warmstart_teacher(&teacher_path, &task_fingerprint, 2, 2); + mutate_warmstart_teacher_contract_field( + &teacher_path, + "observation_adapter_spec_ref", + "wrong-adapter-ref", + ); + let err = load_warmstart_exact_jh_teacher_dataset(&compiled, "teacher") + .expect_err("adapter spec ref mismatch must fail"); + assert!( + err.to_string().contains("observation_adapter_spec_ref"), + "{err}" + ); + let _ = std::fs::remove_file(teacher_path); + } + + #[cfg(feature = "backend-ctw")] + #[test] + fn warmstart_teacher_loader_rejects_observation_adapter_content_crc32_mismatch() { + let teacher_path = unique_temp_path("warmstart-teacher-adapter-crc", ".json"); + let compiled = sample_warmstart_compiled_planner_run(&teacher_path); + let task_fingerprint = + warmstart_exact_jh_planner_task_fingerprint(&compiled).expect("task fingerprint"); + write_warmstart_teacher(&teacher_path, &task_fingerprint, 2, 2); + mutate_warmstart_teacher_contract_field( + &teacher_path, + "observation_adapter_content_crc32", + "deadbeef", + ); + let err = load_warmstart_exact_jh_teacher_dataset(&compiled, "teacher") + .expect_err("adapter crc mismatch must fail"); + assert!( + err.to_string() + .contains("observation_adapter_content_crc32"), + "{err}" + ); + let _ = std::fs::remove_file(teacher_path); + } + + #[cfg(feature = "backend-ctw")] + #[test] + fn warmstart_teacher_loader_rejects_scalar_representation_mismatch() { + let teacher_path = unique_temp_path("warmstart-teacher-scalar", ".json"); + let compiled = sample_warmstart_compiled_planner_run(&teacher_path); + let task_fingerprint = + warmstart_exact_jh_planner_task_fingerprint(&compiled).expect("task fingerprint"); + write_warmstart_teacher(&teacher_path, &task_fingerprint, 2, 2); + mutate_warmstart_teacher_contract_field( + &teacher_path, + "scalar_representation", + "wrong-scalar", + ); + let err = load_warmstart_exact_jh_teacher_dataset(&compiled, "teacher") + .expect_err("scalar representation mismatch must fail"); + assert!(err.to_string().contains("scalar_representation"), "{err}"); + let _ = std::fs::remove_file(teacher_path); + } + + #[cfg(feature = "backend-ctw")] + #[test] + fn warmstart_teacher_loader_rejects_exact_reward_encoding_certificate_mismatch() { + let teacher_path = unique_temp_path("warmstart-teacher-cert", ".json"); + let compiled = sample_warmstart_compiled_planner_run(&teacher_path); + let task_fingerprint = + warmstart_exact_jh_planner_task_fingerprint(&compiled).expect("task fingerprint"); + write_warmstart_teacher(&teacher_path, &task_fingerprint, 2, 2); + mutate_warmstart_teacher_contract_field( + &teacher_path, + "exact_reward_encoding_certificate", + "wrong-cert", + ); + let err = load_warmstart_exact_jh_teacher_dataset(&compiled, "teacher") + .expect_err("reward certificate mismatch must fail"); + assert!( + err.to_string() + .contains("exact_reward_encoding_certificate"), + "{err}" + ); + let _ = std::fs::remove_file(teacher_path); + } + + #[cfg(feature = "backend-ctw")] + #[test] + fn byte_and_path_metric_wrappers_match_basic_identities() { + let x = b"banana bandana"; + let y = b"banana bandana"; + let z = b"entropy coding"; + let backend = CompressionBackend::Rate { + rate_backend: RateBackend::Ctw { depth: 8 }, + coder: infotheory::coders::CoderType::AC, + framing: infotheory::compression::FramingMode::Raw, + } + .compile() + .expect("ctw compression backend should compile"); + + let ncd_same = ncd_bytes_backend(x, y, &backend, NcdVariant::Vitanyi); + let ncd_diff = ncd_bytes_backend(x, z, &backend, NcdVariant::Vitanyi); + assert!(ncd_same.is_finite() && ncd_diff.is_finite()); + assert!( + ncd_same <= ncd_diff, + "identical inputs should not rank farther apart" + ); + assert!( + ncd_same < 0.6, + "identical inputs should remain relatively close; got {ncd_same}" + ); + + let id = intrinsic_dependence_bytes(x); + assert!(id.is_finite()); + assert!(id >= 0.0); + + let mi = mutual_information_bytes(x, y); + assert!(mi.is_finite()); + assert!(mi >= 0.0); + + let h_y_given_x = conditional_entropy_bytes(y, x); + assert!(h_y_given_x.is_finite()); + assert!(h_y_given_x >= 0.0); + + let cross = cross_entropy_bytes(z, x); + assert!(cross.is_finite()); + assert!(cross >= 0.0); + + let joint = joint_entropy_rate_bytes(x, y); + assert!(joint.is_finite()); + assert!(joint >= 0.0); + + let resistance = resistance_to_transformation_bytes(x, y); + assert!(resistance.is_finite()); + assert!(resistance >= 0.0); + + let ned = ned_bytes(x, y); + let ned_cons = ned_cons_bytes(x, y); + let nte = nte_bytes(x, y); + assert!(ned.is_finite() && ned >= 0.0); + assert!(ned_cons.is_finite() && ned_cons >= 0.0); + assert!(nte.is_finite() && nte >= 0.0); + + let left = unique_temp_path("metric-left", ".txt"); + let right = unique_temp_path("metric-right", ".txt"); + let different = unique_temp_path("metric-different", ".txt"); + std::fs::write(&left, x).expect("write left file"); + std::fs::write(&right, y).expect("write right file"); + std::fs::write(&different, z).expect("write different file"); + + let tvd_same = tvd_paths(left.to_str().expect("utf8"), right.to_str().expect("utf8")); + let tvd_diff = tvd_paths( + left.to_str().expect("utf8"), + different.to_str().expect("utf8"), + ); + assert!(tvd_same.is_finite() && tvd_diff.is_finite()); + assert!(tvd_same <= tvd_diff); + + let nhd_same = nhd_paths(left.to_str().expect("utf8"), right.to_str().expect("utf8")); + let nhd_diff = nhd_paths( + left.to_str().expect("utf8"), + different.to_str().expect("utf8"), + ); + assert!(nhd_same.is_finite() && nhd_diff.is_finite()); + assert!(nhd_same <= nhd_diff); + + let kl_same = + kl_divergence_paths(left.to_str().expect("utf8"), right.to_str().expect("utf8")); + let js_same = + js_divergence_paths(left.to_str().expect("utf8"), right.to_str().expect("utf8")); + assert!(kl_same.is_finite() && kl_same >= 0.0); + assert!(js_same.is_finite() && js_same >= 0.0); + + let _ = std::fs::remove_file(left); + let _ = std::fs::remove_file(right); + let _ = std::fs::remove_file(different); + } + + #[cfg(feature = "backend-ctw")] + #[test] + fn planner_run_schedule_derives_cycles_and_extra_exploration() { + let compiled = sample_compiled_planner_run(); + let mut runtime = compiled.runtime().clone(); + runtime.learn_cycles = None; + runtime.eval_cycles = Some(3); + runtime.terminate_lifetime = 5; + runtime.log_every = 2; + runtime.perf = true; + runtime.explore_epsilon = 0.4; + runtime.explore_gamma = 0.5; + let schedule = PlannerSchedule::from_runtime(&runtime); + assert_eq!(schedule.learn_cycles, 5); + assert_eq!(schedule.eval_cycles, 3); + assert_eq!(runtime.log_every, 2); + assert!(runtime.perf); + assert!((schedule.extra_exploration(0) - 0.4).abs() < 1e-12); + assert!((schedule.extra_exploration(2) - 0.1).abs() < 1e-12); + + let mut no_explore_runtime = runtime.clone(); + no_explore_runtime.learn_cycles = Some(1); + no_explore_runtime.eval_cycles = None; + no_explore_runtime.terminate_lifetime = 1; + no_explore_runtime.explore_epsilon = 0.0; + no_explore_runtime.explore_gamma = 0.25; + let no_explore = PlannerSchedule::from_runtime(&no_explore_runtime); + assert_eq!(no_explore.extra_exploration(99), 0.0); + } + + #[cfg(feature = "backend-ctw")] + #[test] + fn planner_environment_and_vm_perf_only_update_environment_state() { + let compiled = sample_compiled_planner_run(); + let env = Box::new(CountingEnv { + observation: 1, + reward: -1, + reward_bits: 4, + action_bits: 1, + observation_bits: 2, + }); + validate_action_alphabet(&compiled, env.as_ref()).expect("matching action alphabet"); + + let mut planner_env = PlannerEnvironment::new(&compiled, env).expect("planner env"); + assert_eq!(planner_env.observations(), &[1]); + assert_eq!(planner_env.reward(), -1); + + let reward = planner_env.perform_action(0).expect("perform action"); + assert_eq!(reward, 0); + assert_eq!(planner_env.observations(), &[2]); + assert_eq!(planner_env.reward(), 0); + + let schedule = PlannerSchedule::new(2, 0); + run_vm_perf_only(&schedule, 0, false, &mut planner_env).expect("vm perf only run"); + assert_eq!(planner_env.observations(), &[4]); + assert_eq!(planner_env.reward(), 2); + } + + #[cfg(feature = "backend-ctw")] + #[test] + fn validate_action_alphabet_reports_mismatch() { + let compiled = sample_compiled_planner_run(); + let env = CountingEnv { + observation: 0, + reward: 0, + reward_bits: 4, + action_bits: 2, + observation_bits: 2, + }; + let err = validate_action_alphabet(&compiled, &env) + .expect_err("mismatched action bits must fail"); + assert!(err.to_string().contains("action_alphabet_mismatch")); + assert!(err.to_string().contains("2 actions")); + assert!(err.to_string().contains("4")); + } + + #[test] + fn process_json_line_parses_escaped_and_nested_json_correctly() { + let line = r#"{ + "op":"metrics", + "text":"hello\n\"json\"", + "meta":{"op":"ncd"}, + "max_order":-1 + }"#; + let out = process_json_line(line); + let parsed: serde_json::Value = serde_json::from_str(&out).expect("output should be json"); + if infotheory::api::RateBackend::try_default().is_ok() { + assert!(parsed.get("h0").and_then(|v| v.as_f64()).unwrap_or(-1.0) >= 0.0); + assert_eq!(parsed.get("len").and_then(|v| v.as_u64()), Some(12)); + } else { + let err = parsed + .get("error") + .and_then(|v| v.as_str()) + .expect("backend-free build should return structured batch error"); + assert!( + err.contains("metrics failed") + && err.contains("no default rate backend is available in this build"), + "unexpected error: {err}" + ); + } + } + + #[cfg(feature = "backend-rwkv")] + #[test] + fn build_ctx_rwkv7_compression_accepts_cfg_method() { + let ctx = crate::cli::build_ctx( + "rosaplus", + "rwkv7", + Some( + "cfg:hidden=64,intermediate=64,layers=1,train=sgd,lr=0.01;policy:schedule=0..100:infer", + ), + None, + ) + .ctx; + + match ctx.compression_backend.canonical_spec() { + CompressionBackend::Rate { + rate_backend, + coder, + framing, + } => { + assert!(matches!(rate_backend, RateBackend::Rwkv7Method { .. })); + assert_eq!(*coder, infotheory::coders::CoderType::AC); + assert_eq!(*framing, infotheory::compression::FramingMode::Raw); + } + _ => panic!("expected rate-coded RWKV backend for cfg: method"), + } + } + + #[test] + fn parse_backend_aliases_and_unknowns() { + #[cfg(feature = "backend-rosa")] + assert_eq!(parse_rate_backend("rosa"), Some("rosaplus")); + #[cfg(feature = "backend-ctw")] + assert_eq!(parse_rate_backend("fac-ctw"), Some("fac-ctw")); + #[cfg(feature = "backend-match")] + assert_eq!(parse_rate_backend("sparse-match"), Some("sparse-match")); + #[cfg(feature = "backend-ppmd")] + assert_eq!(parse_rate_backend("ppmd"), Some("ppmd")); + #[cfg(feature = "backend-calibrated")] + assert_eq!(parse_rate_backend("calibrated"), Some("calibrated")); + assert_eq!(parse_rate_backend("facctw"), None); + assert_eq!(parse_rate_backend("sparsematch"), None); + assert_eq!(parse_rate_backend("ppm"), None); + assert_eq!(parse_rate_backend("cal"), None); + assert_eq!(parse_rate_backend("unknown"), None); + + assert_eq!(parse_compression_backend("unknown"), None); + #[cfg(feature = "backend-zpaq")] + assert_eq!(parse_compression_backend("zpaq"), Some("zpaq")); + assert_eq!(parse_compression_backend("rate-ac"), Some("rate-ac")); + assert_eq!(parse_compression_backend("rate-rans"), Some("rate-rans")); + assert_eq!(parse_compression_backend("rate_ac"), None); + assert_eq!(parse_compression_backend("raterans"), None); + assert_eq!(parse_compression_backend("rate_rans"), None); + #[cfg(feature = "backend-rwkv")] + { + assert_eq!(parse_compression_backend("rwkv7"), Some("rwkv7")); + } + assert_eq!(parse_compression_backend("rwkv"), None); + #[cfg(feature = "backend-mamba")] + { + assert_eq!(parse_rate_backend("mamba"), Some("mamba")); + } + assert_eq!(parse_rate_backend("mamba1"), None); + } + + #[cfg(feature = "all-backends")] + #[test] + fn parse_mixture_expert_supports_calibrated_and_match_backends() { + let base_dir = Path::new("."); + let expert = json!({ + "name": "cal-ctw", + "kind": "calibrated", + "context": "text", + "bins": 33, + "learning_rate": 0.02, + "bias_clip": 4.0, + "base": { + "kind": "match" + } + }); + let parsed = parse_mixture_expert_value(&expert, base_dir, 4).expect("expert should parse"); + match parsed.backend { + RateBackend::Calibrated { spec } => match spec.base { + RateBackend::Match { .. } => {} + _ => panic!("unexpected calibrated base"), + }, + _ => panic!("expected calibrated backend"), + } + } + + #[cfg(feature = "all-backends")] + #[test] + fn parse_mixture_expert_supports_sequitur_backend() { + let base_dir = Path::new("."); + let expert = json!({ + "name": "sequitur", + "kind": "sequitur", + "context_bytes": 96 + }); + let parsed = parse_mixture_expert_value(&expert, base_dir, 4).expect("expert should parse"); + match parsed.backend { + RateBackend::Sequitur { context_bytes } => assert_eq!(context_bytes, 96), + _ => panic!("expected sequitur backend"), + } + } + + #[cfg(feature = "backend-mamba")] + #[test] + fn parse_mixture_expert_resolves_mamba_model_path_relative_to_base_dir() { + let base_dir = unique_temp_path("infotheory-mamba-relpath", ""); + std::fs::create_dir_all(base_dir.join("weights")).expect("create temp dir"); + let rel_path = "weights/model;v1.safetensors"; + let expected = canonical_test_path_string(&base_dir.join(rel_path)); + let expert = json!({ + "name": "mamba-relative", + "kind": "mamba", + "model_path": rel_path + }); + let err = match parse_mixture_expert_value(&expert, &base_dir, 4) { + Ok(_) => panic!("missing model should return an error"), + Err(err) => err, + }; + let msg = err.to_string(); + assert!( + msg.contains(&expected), + "error should mention resolved absolute model path. expected substring: {expected}, got: {msg}" + ); + let _ = std::fs::remove_dir_all(&base_dir); + } + + #[cfg(feature = "backend-rwkv")] + #[test] + fn parse_mixture_expert_resolves_rwkv_model_path_relative_to_base_dir() { + let base_dir = unique_temp_path("infotheory-rwkv-relpath", ""); + std::fs::create_dir_all(base_dir.join("weights")).expect("create temp dir"); + let rel_path = "weights/model;v1.safetensors"; + let expected = canonical_test_path_string(&base_dir.join(rel_path)); + let expert = json!({ + "name": "rwkv-relative", + "kind": "rwkv7", + "model_path": rel_path + }); + let err = match parse_mixture_expert_value(&expert, &base_dir, 4) { + Ok(_) => panic!("missing model should return an error"), + Err(err) => err, + }; + let msg = err.to_string(); + assert!( + msg.contains(&expected), + "error should mention resolved absolute model path. expected substring: {expected}, got: {msg}" + ); + let _ = std::fs::remove_dir_all(&base_dir); + } + + #[cfg(feature = "all-backends")] + #[test] + fn load_expert_spec_preserves_exact_ppmd_settings() { + let expert_path = unique_temp_path("infotheory-expert-spec", ".json"); + std::fs::write( + &expert_path, + serde_json::to_vec(&json!({ + "name": "ppmd", + "kind": "ppmd", + "order": 12, + "memory_mb": 256 + })) + .expect("expert json"), + ) + .expect("write temp expert spec"); + + let parsed = load_expert_spec(expert_path.to_str().expect("utf8 path")) + .expect("ppmd expert should load"); + match parsed.backend { + RateBackend::Ppmd { order, memory_mb } => { + assert_eq!(order, 12); + assert_eq!(memory_mb, 256); + } + _ => panic!("expected ppmd backend"), + } + + let _ = std::fs::remove_file(&expert_path); + } + + #[cfg(feature = "all-backends")] + #[test] + fn build_ctx_loads_expert_spec_with_rosa_max_order() { + let expert_path = unique_temp_path("infotheory-expert-spec-rosa", ".json"); + std::fs::write( + &expert_path, + serde_json::to_vec(&json!({ + "name": "rosa", + "kind": "rosaplus", + "max_order": 32 + })) + .expect("expert json"), + ) + .expect("write temp expert spec"); + + let built = crate::cli::build_ctx( + "rosaplus", + "zpaq", + None, + Some(expert_path.to_string_lossy().as_ref()), + ); + assert!(matches!( + built.ctx.rate_backend.canonical_spec(), + RateBackend::RosaPlus { max_order: 32 } + )); + + let _ = std::fs::remove_file(&expert_path); + } + + #[test] + fn parse_observation_helpers_cover_vm_and_non_vm_cases() { + let base = json!({ + "observation_stream_len": 3, + "observation_key_mode": "stream_hash" + }); + assert_eq!(parse_observation_stream_len(&base), 3); + assert_eq!( + parse_observation_key_mode(&base).expect("parse stream_hash mode"), + ObservationKeyMode::StreamHash + ); + assert_eq!( + parse_observation_key_mode_str("full_stream").expect("parse full_stream"), + ObservationKeyMode::FullStream + ); + assert_eq!( + parse_observation_key_mode_str("last").expect("parse last"), + ObservationKeyMode::Last + ); + assert!(parse_observation_key_mode_str("unknown").is_err()); + + let vm = json!({ + "observation_stream_len": 2, + "observation_key_mode": "full_stream", + "vm_observation": { + "stream_len": 2, + "key_mode": "last" + } + }); + assert_eq!( + parse_observation_stream_len_for_vm(&vm["vm_observation"]), + 2 + ); + assert_eq!( + parse_observation_key_mode_for_vm(&vm["vm_observation"]).expect("parse vm mode"), + ObservationKeyMode::Last + ); + assert_eq!(parse_observation_stream_len_for_env(&vm, "vm"), 2); + assert_eq!( + parse_observation_key_mode_for_env(&vm, "vm").expect("parse vm env mode"), + ObservationKeyMode::Last + ); + assert_eq!( + parse_observation_key_mode_for_env(&vm, "coin").expect("parse non-vm mode"), + ObservationKeyMode::FullStream + ); + + let mismatch = json!({ + "observation_stream_len": 2, + "vm_observation": { + "stream_len": 3 + } + }); + let err = validate_observation_config("vm", &mismatch, 2, ObservationKeyMode::FullStream) + .expect_err("mismatched vm stream_len should fail"); + assert!(err.to_string().contains("conflicts")); + + let mismatch_mode = json!({ + "observation_key_mode": "full_stream", + "vm_observation": { + "key_mode": "last" + } + }); + let err = validate_observation_config("vm", &mismatch_mode, 1, ObservationKeyMode::Last) + .expect_err("mismatched vm key mode should fail"); + assert!(err.to_string().contains("conflicts")); + } + + #[cfg(feature = "all-backends")] + #[test] + fn parse_mixture_kind_and_spec_validation() { + assert_eq!( + parse_mixture_kind("bayes").expect("bayes kind"), + MixtureKind::Bayes + ); + assert_eq!( + parse_mixture_kind("switching").expect("switching kind"), + MixtureKind::Switching + ); + assert_eq!( + parse_mixture_kind("convex").expect("convex kind"), + MixtureKind::Convex + ); + assert_eq!( + parse_mixture_kind("neural").expect("neural kind"), + MixtureKind::Neural + ); + assert!(parse_mixture_kind("bayes-mix").is_err()); + assert!(parse_mixture_kind("switch").is_err()); + assert!(parse_mixture_kind("nonsense").is_err()); + assert_eq!( + parse_mixture_schedule("theorem").expect("theorem schedule"), + MixtureScheduleMode::Theorem + ); + assert!(parse_mixture_schedule("nonsense").is_err()); + + let base_dir = Path::new("."); + let missing_experts = json!({ + "kind": "bayes", + "experts": [] + }); + assert!(parse_mixture_spec_value(&missing_experts, base_dir, 8).is_err()); + + let fading_without_decay = json!({ + "kind": "fading", + "experts": [ + {"name": "ctw-e", "kind": "ctw", "depth": 4} + ] + }); + assert!(parse_mixture_spec_value(&fading_without_decay, base_dir, 8).is_err()); + + let valid = json!({ + "kind": "convex", + "schedule": "theorem", + "experts": [ + {"name": "ctw-e", "kind": "ctw", "depth": 8}, + {"name": "fac-e", "kind": "fac-ctw", "base_depth": 8, "encoding_bits": 8} + ] + }); + let spec = parse_mixture_spec_value(&valid, base_dir, 8).expect("valid mixture"); + assert_eq!(spec.schedule, MixtureScheduleMode::Theorem); + assert_eq!(spec.experts.len(), 2); + assert!(matches!(spec.kind, MixtureKind::Convex)); + + let nested = json!({ + "kind": "convex", + "alpha": 1.25, + "experts": [ + { + "name": "nested", + "kind": "mixture", + "spec": { + "kind": "bayes", + "experts": [ + {"name": "ctw-e", "kind": "ctw", "depth": 4} + ] + } + }, + {"name": "match-e", "kind": "match", "hash_bits": 18} + ] + }); + let nested_spec = parse_mixture_spec_value(&nested, base_dir, 8).expect("nested mixture"); + assert!(matches!(nested_spec.kind, MixtureKind::Convex)); + assert_eq!(nested_spec.experts.len(), 2); + match &nested_spec.experts[0].backend { + RateBackend::Mixture { spec } => { + assert!(matches!(spec.kind, MixtureKind::Bayes)); + assert_eq!(spec.experts.len(), 1); + } + _ => panic!("expected nested mixture backend"), + } + } + + #[cfg(all(feature = "vm", feature = "all-backends"))] + #[test] + fn parse_vm_stats_backend_supports_new_backends_and_rejects_unknowns() { + let root = json!({ + "algorithm": "ctw", + "ct_depth": 8, + "observation_bits": 8, + "reward_bits": 8 + }); + let base_dir = Path::new("."); + + let matched = + parse_vm_stats_backend(&json!({"kind":"match","hash_bits":18}), &root, base_dir) + .expect("match backend should parse"); + assert!(matches!(matched, RateBackend::Match { hash_bits: 18, .. })); + + let sparse = parse_vm_stats_backend( + &json!({"kind":"sparse-match","gap_min":2,"gap_max":4}), + &root, + base_dir, + ) + .expect("sparse-match backend should parse"); + assert!(matches!( + sparse, + RateBackend::SparseMatch { + gap_min: 2, + gap_max: 4, + .. + } + )); + + let ppmd = parse_vm_stats_backend(&json!({"kind":"ppmd","order":12}), &root, base_dir) + .expect("ppmd backend should parse"); + assert!(matches!(ppmd, RateBackend::Ppmd { order: 12, .. })); + + let sequitur = parse_vm_stats_backend( + &json!({"kind":"sequitur","context_bytes":72}), + &root, + base_dir, + ) + .expect("sequitur backend should parse"); + assert!(matches!( + sequitur, + RateBackend::Sequitur { context_bytes: 72 } + )); + + let particle = parse_vm_stats_backend( + &json!({ + "kind":"particle", + "spec":{"num_particles":4,"num_cells":4,"cell_dim":8} + }), + &root, + base_dir, + ) + .expect("particle backend should parse"); + assert!(matches!(particle, RateBackend::Particle { .. })); + + let mixture = parse_vm_stats_backend( + &json!({ + "kind":"mixture", + "spec":{"kind":"bayes","experts":[{"kind":"match"}]} + }), + &root, + base_dir, + ) + .expect("mixture backend should parse"); + assert!(matches!(mixture, RateBackend::Mixture { .. })); + + let calibrated = parse_vm_stats_backend( + &json!({ + "kind":"calibrated", + "base":{"kind":"ctw","depth":8}, + "context":"text", + "bins":17, + "learning_rate":0.05, + "bias_clip":3.0 + }), + &root, + base_dir, + ) + .expect("calibrated backend should parse"); + assert!(matches!(calibrated, RateBackend::Calibrated { .. })); + + let err = match parse_vm_stats_backend(&json!("unknown-backend"), &root, base_dir) { + Ok(_) => panic!("unknown backend should not silently fall back"), + Err(err) => err, + }; + assert!( + err.to_string().contains("unknown vm stats backend"), + "unexpected error: {err}" + ); + } + + #[cfg(all(feature = "vm", feature = "backend-ctw"))] + #[test] + fn parse_vm_stats_backend_preserves_fac_ctw_vm_defaults() { + let root = json!({ + "algorithm": "fac-ctw", + "ct_depth": 11, + "observation_bits": 13, + "reward_bits": 5 + }); + let parsed = parse_vm_stats_backend(&json!({"kind":"fac-ctw"}), &root, Path::new(".")) + .expect("fac-ctw backend should parse"); + match parsed { + RateBackend::FacCtw { + base_depth, + num_percept_bits, + encoding_bits, + msb_first, + } => { + assert_eq!(base_depth, 32); + assert_eq!(encoding_bits, 8); + assert_eq!(num_percept_bits, 18); + assert_eq!(msb_first, None); + } + _ => panic!("expected fac-ctw backend"), + } + } +} diff --git a/crates/infotheory/src/mixture.rs b/crates/infotheory/src/mixture.rs new file mode 100644 index 00000000..8dd98cc9 --- /dev/null +++ b/crates/infotheory/src/mixture.rs @@ -0,0 +1,8067 @@ +//! Online mixtures of probabilistic predictors (log-loss Hedge / Bayes, switching, MDL). +//! +//! This module provides a small, rigorously correct toolkit for sequential model mixing. +//! Predictors expose per-symbol log-probabilities, which allows principled Bayesian +//! mixture updates and clean information-theoretic accounting. +//! +//! ## Rate-Backend Mixtures +//! +//! The mixture primitives here power `RateBackend::Mixture`, enabling Bayes, fading Bayes, +//! switching, and MDL-style selectors to be used anywhere a rate backend is accepted. +#![cfg_attr( + not(feature = "all-backends"), + allow( + dead_code, + unused_imports, + unused_variables, + unused_mut, + unreachable_code + ) +)] + +#[cfg(test)] +use crate::api::MixtureSpec; +use crate::api::{MixtureKind, MixtureScheduleMode, RateBackend}; +#[cfg(feature = "backend-calibrated")] +use crate::backends::calibration::CalibratorCore; +#[cfg(feature = "backend-ctw")] +use crate::backends::ctw::{ + ContextTree, ContextTreeLifecycleSnapshot, FacContextTree, FacContextTreeLifecycleSnapshot, +}; +#[cfg(feature = "backend-match")] +use crate::backends::match_model::{MatchModel, MatchModelLifecycleSnapshot}; +#[cfg(feature = "backend-ppmd")] +use crate::backends::ppmd::{PpmdLifecycleSnapshot, PpmdModel}; +#[cfg(feature = "backend-rosa")] +use crate::backends::rosaplus::{RosaPlus, RosaTx}; +#[cfg(feature = "backend-sequitur")] +use crate::backends::sequitur::{SequiturCheckpoint, SequiturLifecycleSnapshot, SequiturModel}; +#[cfg(feature = "backend-match")] +use crate::backends::sparse_match::SparseMatchModel; +use crate::backends::text_context::TextContextAnalyzer; +#[cfg(feature = "backend-zpaq")] +use crate::backends::zpaq_rate::ZpaqRateModel; +#[cfg(feature = "backend-mamba")] +use crate::mambazip; +use crate::neural_mix::{NeuralHistoryState, NeuralMixCore}; +#[cfg(feature = "backend-rwkv")] +use crate::rwkvzip; +use crate::spec::CompiledRateBackend; +use std::sync::Arc; + +/// Default minimum probability floor to avoid log(0). +pub const DEFAULT_MIN_PROB: f64 = 5.960_464_477_539_063e-8; + +#[inline] +fn clamp_prob(p: f64, min_prob: f64) -> f64 { + if p.is_finite() { + p.max(min_prob) + } else { + min_prob + } +} + +#[inline] +fn clamp_unit_prob(p: f64, min_prob: f64) -> f64 { + clamp_prob(p, min_prob).min(1.0 - min_prob) +} + +#[inline] +fn logsumexp(xs: &[f64]) -> f64 { + let mut max_v = f64::NEG_INFINITY; + for &v in xs { + if v > max_v { + max_v = v; + } + } + if !max_v.is_finite() { + return max_v; + } + let mut sum = 0.0; + for &v in xs { + sum += (v - max_v).exp(); + } + max_v + sum.ln() +} + +#[inline] +fn logsumexp2(a: f64, b: f64) -> f64 { + let m = if a > b { a } else { b }; + if !m.is_finite() { + return m; + } + m + ((a - m).exp() + (b - m).exp()).ln() +} + +#[inline] +fn logsumexp_weights(experts: &[ExpertState]) -> f64 { + let mut max_v = f64::NEG_INFINITY; + for e in experts { + if e.log_weight > max_v { + max_v = e.log_weight; + } + } + if !max_v.is_finite() { + return max_v; + } + let mut sum = 0.0; + for e in experts { + sum += (e.log_weight - max_v).exp(); + } + max_v + sum.ln() +} + +fn normalize_simplex_weights(weights: &mut [f64]) { + if weights.is_empty() { + return; + } + let mut sum = 0.0; + for weight in weights.iter_mut() { + if !weight.is_finite() || *weight < 0.0 { + *weight = 0.0; + } + sum += *weight; + } + if !sum.is_finite() || sum <= 0.0 { + let uniform = 1.0 / (weights.len() as f64); + weights.fill(uniform); + return; + } + for weight in weights.iter_mut() { + *weight /= sum; + } +} + +pub(crate) fn project_simplex_with_scratch(weights: &mut [f64], scratch: &mut Vec) { + if weights.is_empty() { + return; + } + + scratch.clear(); + scratch.extend( + weights + .iter() + .map(|&weight| if weight.is_finite() { weight } else { 0.0 }), + ); + let sorted = scratch.as_mut_slice(); + sorted.sort_by(|a, b| b.total_cmp(a)); + + let mut cumulative = 0.0; + let mut rho = None; + for (index, value) in sorted.iter().enumerate() { + cumulative += *value; + let theta = (cumulative - 1.0) / ((index + 1) as f64); + if *value > theta { + rho = Some(index); + } + } + + let Some(rho_index) = rho else { + let uniform = 1.0 / (weights.len() as f64); + weights.fill(uniform); + return; + }; + + let theta = (sorted.iter().take(rho_index + 1).sum::() - 1.0) / ((rho_index + 1) as f64); + for weight in weights.iter_mut() { + *weight = (*weight - theta).max(0.0); + } + normalize_simplex_weights(weights); +} + +#[inline] +pub(crate) fn switching_alpha_for_update( + schedule: MixtureScheduleMode, + alpha: f64, + processed_symbols: u64, +) -> f64 { + match schedule { + MixtureScheduleMode::Default => alpha.clamp(0.0, 1.0), + MixtureScheduleMode::Theorem => 1.0 / ((processed_symbols + 2) as f64), + } +} + +#[inline] +pub(crate) fn convex_step_size_for_update( + schedule: MixtureScheduleMode, + alpha: f64, + update_index: u64, +) -> f64 { + let t = update_index.max(1) as f64; + match schedule { + MixtureScheduleMode::Default => alpha.max(1e-12) / t.sqrt(), + MixtureScheduleMode::Theorem => DEFAULT_MIN_PROB / t.sqrt(), + } +} + +fn normalized_log_weights(log_weights: impl IntoIterator) -> Vec { + let mut weights: Vec = log_weights.into_iter().collect(); + if weights.is_empty() { + return Vec::new(); + } + let max_log = weights.iter().copied().fold(f64::NEG_INFINITY, f64::max); + for w in &mut weights { + *w = if max_log.is_finite() { + (*w - max_log).exp() + } else { + 0.0 + }; + } + normalize_simplex_weights(&mut weights); + weights +} + +fn normalized_prior_weights(configs: &[ExpertConfig]) -> Vec { + normalized_log_weights(configs.iter().map(|cfg| cfg.log_prior)) +} + +fn normalized_expert_prior_weights(experts: &[ExpertState]) -> Vec { + normalized_log_weights(experts.iter().map(|expert| expert.log_prior)) +} + +#[cfg(feature = "backend-calibrated")] +#[inline] +fn reset_calibrated_wrapper_state( + core: &mut CalibratorCore, + pdf: &mut [f64; 256], + valid: &mut bool, +) { + core.reset_context(); + pdf.fill(1.0 / 256.0); + *valid = false; +} + +fn set_log_weights_from_linear(experts: &mut [ExpertState], weights: &[f64]) { + for (expert, &weight) in experts.iter_mut().zip(weights.iter()) { + expert.log_weight = if weight > 0.0 { + weight.ln() + } else { + f64::NEG_INFINITY + }; + } +} + +/// Trait for online byte-level predictors that expose per-symbol log-probabilities. +pub trait OnlineBytePredictorClone { + /// Clone this predictor as a trait object. + /// + /// This supports `Clone` for `Box` via type erasure, + /// so mixture experts can be duplicated without knowing their concrete type. + fn clone_box(&self) -> Box; +} + +impl OnlineBytePredictorClone for T +where + T: 'static + OnlineBytePredictor + Clone, +{ + fn clone_box(&self) -> Box { + Box::new(self.clone()) + } +} + +impl Clone for Box { + fn clone(&self) -> Self { + self.clone_box() + } +} + +/// Trait for online byte-level predictors that expose per-symbol log-probabilities. +pub trait OnlineBytePredictor: Send + OnlineBytePredictorClone { + /// Whether this predictor supports frozen-conditioning resets. + /// + /// Predictors that return `false` may still support ordinary stream lifecycle + /// hooks (`begin_stream`/`finish_stream`), but cannot provide plugin-entropy + /// style frozen reset semantics. + fn supports_frozen_reset(&self) -> bool { + true + } + + /// Optional stream-start hook. + /// + /// Predictors that require total symbol count (for example percent-based + /// policy schedules) can initialize runtime state here. + fn begin_stream(&mut self, _total_symbols: Option) -> Result<(), String> { + Ok(()) + } + + /// Optional stream-finalization hook. + fn finish_stream(&mut self) -> Result<(), String> { + Ok(()) + } + + /// Capture a structural checkpoint when the concrete predictor supports it. + fn checkpoint_if_supported(&mut self) -> Option { + None + } + + /// Restore a structural checkpoint created by [`Self::checkpoint_if_supported`]. + fn restore_checkpoint_if_supported( + &mut self, + _checkpoint: &OnlineBytePredictorCheckpoint, + ) -> bool { + false + } + + /// Release a structural checkpoint that will never be restored. + /// + /// Predictors with compact rollback journals can use this to retire + /// temporary checkpoints without clearing older checkpoints that may still + /// be live elsewhere. Plain drop-based checkpoints can accept the default + /// behavior: taking ownership of `_checkpoint` is already a successful + /// discard. + fn discard_checkpoint_if_supported( + &mut self, + _checkpoint: OnlineBytePredictorCheckpoint, + ) -> bool { + true + } + + /// Clear compact checkpoint journals after all structural checkpoints expire. + fn clear_checkpoints_if_supported(&mut self) {} + + /// Log-probability (natural log) of `symbol` given the current history. + fn log_prob(&mut self, symbol: u8) -> f64; + + /// Bulk 256-way log-probabilities for the next byte. + fn fill_log_probs(&mut self, out: &mut [f64; 256]) { + for (sym, slot) in out.iter_mut().enumerate() { + *slot = self.log_prob(sym as u8); + } + } + + /// Whether this predictor can expose an MSB-first byte prefix natively. + /// + /// Native prefix stepping lets bitwise consumers query and condition on the + /// bits of the next byte without first materializing all 256 byte + /// probabilities. Predictors that return `false` remain fully supported + /// through the generic byte-PDF prefix fallback. + fn has_native_msb_byte_prefix(&self) -> bool { + false + } + + /// Prepare a native MSB-first byte-prefix step. + /// + /// Returns `true` when the predictor entered a native prefix state. Callers + /// must then query bits in order, call [`Self::observe_native_msb_prefix_bit`] + /// after each observed prefix bit, and finish with + /// [`Self::finish_native_msb_byte_prefix`]. Implementations must leave the + /// predictor unchanged when they return `Ok(false)` or `Err(_)`. + fn begin_native_msb_byte_prefix(&mut self) -> Result { + Ok(false) + } + + /// Abort an active native MSB-first byte-prefix step before any bits have + /// been observed. + /// + /// This is used only when a caller restores to a checkpoint that was taken + /// immediately after `begin_native_msb_byte_prefix`. Implementations must + /// return an error rather than dropping observed prefix bits. + fn abort_empty_native_msb_byte_prefix(&mut self) -> Result { + Ok(false) + } + + /// Predict `P(bit = 1)` for the next MSB-first prefix bit. + /// + /// `bit_idx` must match the next unobserved bit in the active native + /// prefix, counted MSB-first in `0..8`. Re-querying the current bit index + /// before observing it is allowed. + fn native_msb_prefix_prob_one(&mut self, _bit_idx: usize) -> Result { + Err("native MSB-first byte-prefix prediction is unavailable".to_string()) + } + + /// Observe one MSB-first prefix bit inside an active native byte-prefix step. + /// + /// `bit_idx` must match the next unobserved bit in the active native + /// prefix, counted MSB-first in `0..8`. + fn observe_native_msb_prefix_bit(&mut self, _bit_idx: usize, _bit: bool) -> Result<(), String> { + Err("native MSB-first byte-prefix stepping is unavailable".to_string()) + } + + /// Finish an active native byte-prefix step after all eight bits are known. + fn finish_native_msb_byte_prefix(&mut self, _symbol: u8) -> Result<(), String> { + Err("native MSB-first byte-prefix stepping is unavailable".to_string()) + } + + /// Log-probability (natural log) of `symbol`, then update the predictor. + fn log_prob_update(&mut self, symbol: u8) -> f64 { + let logp = self.log_prob(symbol); + self.update(symbol); + logp + } + + /// Update the predictor with the observed `symbol`. + fn update(&mut self, symbol: u8); + + /// Reset only dynamic conditioning state while preserving fitted parameters/statistics. + /// + /// Predictors with latent/posterior state may also preserve their learned + /// parameter posterior here; "frozen" means no new parameter fitting during + /// the score pass, not necessarily a static hidden-state belief. + fn reset_frozen(&mut self, total_symbols: Option) -> Result<(), String> { + self.finish_stream()?; + self.begin_stream(total_symbols) + } + + /// Start a new stream in a way that preserves each predictor's semantic contract. + /// + /// This uses frozen-reset semantics when supported, and falls back to ordinary + /// begin/finish stream hooks otherwise. + fn begin_fresh_stream(&mut self, total_symbols: Option) -> Result<(), String> { + if self.supports_frozen_reset() { + self.reset_frozen(total_symbols) + } else { + self.begin_stream(total_symbols) + } + } + + /// Advance conditioning state without fitting or adapting parameters. + /// + /// For state-space or latent-variable models this may still update internal + /// filtering/posterior state needed for correct sequential predictions. + fn update_frozen(&mut self, symbol: u8) { + self.update(symbol); + } +} + +#[doc(hidden)] +#[derive(Clone)] +pub struct OnlineBytePredictorCheckpoint(OnlineBytePredictorCheckpointKind); + +#[derive(Clone)] +enum OnlineBytePredictorCheckpointKind { + RateBackend(RateBackendPredictorCheckpoint), +} + +impl OnlineBytePredictorCheckpoint { + fn rate_backend(checkpoint: RateBackendPredictorCheckpoint) -> Self { + Self(OnlineBytePredictorCheckpointKind::RateBackend(checkpoint)) + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum OnlineBytePredictorLifecycleOp { + /// Start a possibly continuing stream. + BeginStream, + /// Start a fresh stream, preserving fitted predictor state where supported. + BeginFreshStream, + /// Reset transient conditioning state while preserving fitted state. + ResetFrozen, + /// Finish the current stream. + FinishStream, +} + +#[cfg(feature = "backend-ctw")] +fn validate_native_msb_prefix_bit_idx(next_bit_idx: usize, bit_idx: usize) -> Result<(), String> { + if bit_idx >= 8 { + return Err(format!( + "native MSB-first byte-prefix bit index {bit_idx} is out of range; expected 0..8" + )); + } + if bit_idx != next_bit_idx { + return Err(format!( + "native MSB-first byte-prefix bit index {bit_idx} violated sequential stepping; expected {next_bit_idx}" + )); + } + Ok(()) +} + +#[cfg(feature = "backend-ctw")] +fn validate_native_msb_prefix_finish(next_bit_idx: usize) -> Result<(), String> { + if next_bit_idx != 8 { + return Err(format!( + "native MSB-first byte-prefix finish requires 8 observed bits, got {next_bit_idx}" + )); + } + Ok(()) +} + +#[derive(Clone)] +enum BytePrefixStepState { + Native, + PdfPrefix { + cdf: Box<[f64; 257]>, + lo: usize, + hi: usize, + }, +} + +impl Default for BytePrefixStepState { + fn default() -> Self { + Self::PdfPrefix { + cdf: Box::new([0.0; 257]), + lo: 0, + hi: 256, + } + } +} + +impl BytePrefixStepState { + fn prepare(&mut self, predictor: &mut dyn OnlineBytePredictor) -> Result<(), String> { + if predictor.begin_native_msb_byte_prefix()? { + *self = Self::Native; + return Ok(()); + } + + let mut cdf = match std::mem::take(self) { + Self::PdfPrefix { cdf, .. } => cdf, + Self::Native => Box::new([0.0; 257]), + }; + let mut logps = [0.0f64; 256]; + predictor.fill_log_probs(&mut logps); + cdf[0] = 0.0; + for (idx, &lp) in logps.iter().enumerate() { + cdf[idx + 1] = cdf[idx] + clamp_prob(lp.exp(), DEFAULT_MIN_PROB); + } + if !cdf[256].is_finite() || cdf[256] <= 0.0 { + for (idx, slot) in cdf.iter_mut().enumerate() { + *slot = (idx as f64) / 256.0; + } + } + *self = Self::PdfPrefix { + cdf, + lo: 0, + hi: 256, + }; + Ok(()) + } + + fn prob_one( + &mut self, + predictor: &mut dyn OnlineBytePredictor, + bit_idx: usize, + ) -> Result { + match self { + Self::Native => predictor.native_msb_prefix_prob_one(bit_idx), + Self::PdfPrefix { cdf, lo, hi } => { + let mid: usize = (*lo + *hi) >> 1; + let total: f64 = (cdf[*hi] - cdf[*lo]).max(DEFAULT_MIN_PROB); + let one: f64 = (cdf[*hi] - cdf[mid]).max(0.0); + Ok((one / total).clamp(DEFAULT_MIN_PROB, 1.0 - DEFAULT_MIN_PROB)) + } + } + } + + fn observe( + &mut self, + predictor: &mut dyn OnlineBytePredictor, + bit_idx: usize, + bit: bool, + ) -> Result<(), String> { + match self { + Self::Native => predictor.observe_native_msb_prefix_bit(bit_idx, bit), + Self::PdfPrefix { lo, hi, .. } => { + let mid: usize = (*lo + *hi) >> 1; + if bit { + *lo = mid; + } else { + *hi = mid; + } + Ok(()) + } + } + } + + fn abort_empty(&mut self, predictor: &mut dyn OnlineBytePredictor) -> Result<(), String> { + match self { + Self::Native => { + predictor.abort_empty_native_msb_byte_prefix()?; + *self = Self::default(); + Ok(()) + } + Self::PdfPrefix { .. } => { + *self = Self::default(); + Ok(()) + } + } + } + + fn finish( + &mut self, + predictor: &mut dyn OnlineBytePredictor, + symbol: u8, + ) -> Result<(), String> { + match self { + Self::Native => predictor.finish_native_msb_byte_prefix(symbol), + Self::PdfPrefix { .. } => { + predictor.update(symbol); + Ok(()) + } + } + } +} + +#[derive(Clone, Default)] +struct MixtureBitPrefixState { + states: Vec, + weights: Vec, + likelihoods: Vec, + bit_probs: Vec, + logps: Vec, + active: bool, + primed_bit_idx: Option, + expected_bit_idx: usize, +} + +impl MixtureBitPrefixState { + fn reset_inactive(&mut self) { + self.active = false; + self.primed_bit_idx = None; + self.expected_bit_idx = 0; + } + + fn validate_bit_idx(&self, bit_idx: usize) -> Result<(), String> { + if bit_idx >= 8 { + return Err(format!( + "native MSB-first byte-prefix bit index {bit_idx} is out of range; expected 0..8" + )); + } + if bit_idx != self.expected_bit_idx { + return Err(format!( + "native MSB-first byte-prefix bit index {bit_idx} violated sequential stepping; expected {}", + self.expected_bit_idx + )); + } + Ok(()) + } + + fn begin(&mut self, experts: &mut [ExpertState], weights: &[f64]) -> Result { + if !experts + .iter() + .any(|expert| expert.predictor.has_native_msb_byte_prefix()) + { + self.reset_inactive(); + return Ok(false); + } + + let n: usize = experts.len(); + let mut native_checkpoints: Vec = Vec::new(); + for (idx, expert) in experts.iter_mut().enumerate() { + if !expert.predictor.has_native_msb_byte_prefix() { + continue; + } + let Some(checkpoint) = expert.predictor.checkpoint_if_supported() else { + for checkpoint in native_checkpoints.drain(..) { + assert!( + experts[checkpoint.index] + .predictor + .discard_checkpoint_if_supported(checkpoint.checkpoint), + "native-prefix expert checkpoint could not be discarded", + ); + } + self.reset_inactive(); + return Ok(false); + }; + native_checkpoints.push(ExpertTempCheckpoint { + index: idx, + checkpoint, + }); + } + + self.states.resize_with(n, BytePrefixStepState::default); + self.weights.clear(); + self.weights.extend(weights.iter().copied()); + normalize_simplex_weights(&mut self.weights); + self.likelihoods.resize(n, 1.0); + self.likelihoods.fill(1.0); + self.bit_probs.resize(n, 0.5); + self.logps.resize(n, 0.0); + self.primed_bit_idx = None; + self.expected_bit_idx = 0; + for (state, expert) in self.states.iter_mut().zip(experts.iter_mut()) { + if let Err(err) = state.prepare(expert.predictor.as_mut()) { + for checkpoint in native_checkpoints.drain(..) { + assert!( + experts[checkpoint.index] + .predictor + .restore_checkpoint_if_supported(&checkpoint.checkpoint), + "native-prefix expert checkpoint could not be restored", + ); + assert!( + experts[checkpoint.index] + .predictor + .discard_checkpoint_if_supported(checkpoint.checkpoint), + "native-prefix expert checkpoint could not be discarded", + ); + } + self.reset_inactive(); + return Err(err); + } + } + for checkpoint in native_checkpoints.drain(..) { + assert!( + experts[checkpoint.index] + .predictor + .discard_checkpoint_if_supported(checkpoint.checkpoint), + "native-prefix expert checkpoint could not be discarded", + ); + } + self.active = true; + Ok(true) + } + + fn abort_empty(&mut self, experts: &mut [ExpertState]) -> Result { + if !self.active { + return Ok(false); + } + if self.expected_bit_idx != 0 { + return Err(format!( + "native MSB-first byte-prefix abort requires zero observed bits, got {}", + self.expected_bit_idx + )); + } + for (state, expert) in self.states.iter_mut().zip(experts.iter_mut()) { + state.abort_empty(expert.predictor.as_mut())?; + } + self.reset_inactive(); + Ok(true) + } + + fn prime_bit_probs_if_needed( + &mut self, + experts: &mut [ExpertState], + bit_idx: usize, + ) -> Result<(), String> { + self.validate_bit_idx(bit_idx)?; + if self.primed_bit_idx == Some(bit_idx) { + return Ok(()); + } + // Index form required for coordinated access to per-expert state + scratch buffers + // (same rationale as the allows in prob_one/observe below). + #[allow(clippy::needless_range_loop)] + for idx in 0..experts.len() { + let p1: f64 = self.states[idx].prob_one(experts[idx].predictor.as_mut(), bit_idx)?; + self.bit_probs[idx] = p1; + } + self.primed_bit_idx = Some(bit_idx); + Ok(()) + } + + fn prob_one(&mut self, experts: &mut [ExpertState], bit_idx: usize) -> Result { + debug_assert!(self.active); + self.prime_bit_probs_if_needed(experts, bit_idx)?; + let mut denom: f64 = 0.0; + let mut numer: f64 = 0.0; + // Index form required for parallel mutable access to multiple scratch buffers + // alongside experts; iterators would require zip + tuple mutation which is less clear here. + #[allow(clippy::needless_range_loop)] + for idx in 0..experts.len() { + let p1: f64 = self.bit_probs[idx]; + let weighted_prefix: f64 = self.weights[idx] * self.likelihoods[idx]; + denom += weighted_prefix; + numer += weighted_prefix * p1; + } + Ok(if denom.is_finite() && denom > 0.0 { + (numer / denom).clamp(DEFAULT_MIN_PROB, 1.0 - DEFAULT_MIN_PROB) + } else { + // Invariant failure (introduced in bitwiseness bit-prefix state; tightened): + // non-positive/NaN denom means internal expert weighting or priming + // produced invalid state. Panic with context per AGENTS (contract violation). + panic!( + "MixtureBitPrefixState::prob_one: invalid weighted denom (must be finite > 0); \ + this indicates a bug in prime_bit_probs_if_needed or expert likelihoods" + ) + }) + } + + fn observe( + &mut self, + experts: &mut [ExpertState], + bit_idx: usize, + bit: bool, + ) -> Result<(), String> { + debug_assert!(self.active); + self.prime_bit_probs_if_needed(experts, bit_idx)?; + // Index form clearest for coordinated mutation of likelihoods/states + experts[idx]. + #[allow(clippy::needless_range_loop)] + for idx in 0..experts.len() { + let p1: f64 = self.bit_probs[idx]; + let pb: f64 = if bit { p1 } else { 1.0 - p1 }; + self.likelihoods[idx] = (self.likelihoods[idx] * pb).max(DEFAULT_MIN_PROB); + self.states[idx].observe(experts[idx].predictor.as_mut(), bit_idx, bit)?; + } + self.expected_bit_idx += 1; + self.primed_bit_idx = None; + Ok(()) + } + + fn finish_adaptive(&mut self, experts: &mut [ExpertState], symbol: u8) -> Result<(), String> { + debug_assert!(self.active); + if self.expected_bit_idx != 8 { + return Err(format!( + "native MSB-first byte-prefix finish requires 8 observed bits, got {}", + self.expected_bit_idx + )); + } + // Index form clearest for coordinated mutation of likelihoods/logps/states + experts[idx]. + #[allow(clippy::needless_range_loop)] + for idx in 0..experts.len() { + let lp: f64 = self.likelihoods[idx].max(DEFAULT_MIN_PROB).ln(); + self.logps[idx] = lp; + self.states[idx].finish(experts[idx].predictor.as_mut(), symbol)?; + } + self.reset_inactive(); + Ok(()) + } +} + +struct ExpertTempCheckpoint { + index: usize, + checkpoint: OnlineBytePredictorCheckpoint, +} + +#[cfg(feature = "backend-rwkv")] +#[inline] +fn ensure_rwkv_primed(compressor: &mut rwkvzip::Compressor, primed: &mut bool) { + if !*primed { + compressor.reset_and_prime(); + *primed = true; + } +} + +#[cfg(feature = "backend-ctw")] +use crate::backends::ctw::{ + ctw_log_prob_msb, ctw_log_prob_update_lsb, ctw_log_prob_update_msb, ctw_symbol_bit_msb, + fill_ctw_tree_log_probs, fill_fac_tree_log_probs, +}; +/// A concrete online predictor backed by a `RateBackend` configuration. +#[allow(clippy::large_enum_variant)] +#[derive(Clone)] +pub enum RateBackendPredictor { + /// ROSA-Plus online suffix automaton. + #[cfg(feature = "backend-rosa")] + Rosa { + /// ROSA model state. + model: RosaPlus, + /// Probability floor for numeric stability. + min_prob: f64, + /// Undo log for checkpointed updates and frozen-conditioning moves. + checkpoint_journal: Vec, + /// Number of active checkpoints currently recording into `checkpoint_journal`. + checkpoint_depth: usize, + }, + /// Local contiguous match predictor. + #[cfg(feature = "backend-match")] + Match { + /// Match model state. + model: MatchModel, + /// Probability floor for numeric stability. + min_prob: f64, + }, + /// Sparse/gapped local match predictor. + #[cfg(feature = "backend-match")] + SparseMatch { + /// Sparse-match model state. + model: SparseMatchModel, + /// Probability floor for numeric stability. + min_prob: f64, + }, + /// Bounded-memory PPMD-style predictor. + #[cfg(feature = "backend-ppmd")] + Ppmd { + /// PPMD model state. + model: PpmdModel, + /// Probability floor for numeric stability. + min_prob: f64, + }, + /// Exact online Sequitur grammar backend with predictive suffix contexts. + #[cfg(feature = "backend-sequitur")] + Sequitur { + /// Sequitur model state. + model: SequiturModel, + /// Probability floor for numeric stability. + min_prob: f64, + }, + /// AC-CTW with consumer-chosen symbol width interpreted MSB-first. + #[cfg(feature = "backend-ctw")] + Ctw { + /// Single binary context tree. + tree: ContextTree, + /// Active bit-width per observed symbol. + bits_per_symbol: usize, + /// Probability floor for numeric stability. + min_prob: f64, + /// Compact rollback journal used while checkpoint scopes are active. + checkpoint_journal: Vec, + /// Number of active checkpoints that require journaling. + checkpoint_depth: usize, + /// In-flight native byte-prefix progress when stepping MSB-first bits. + native_prefix_progress: Option, + }, + /// Factorized CTW with width-dependent bit order. + #[cfg(feature = "backend-ctw")] + FacCtw { + /// FAC-CTW tree stack for configured bit width. + tree: FacContextTree, + /// Active bit-width per symbol. + bits_per_symbol: usize, + /// Effective symbol bit order (`true` => MSB-first). + /// + /// FAC-CTW keeps legacy LSB-first behavior for non-byte symbol widths, + /// but 8-bit symbols run MSB-first so byte-packed sessions can use the + /// native prefix path consistently. + msb_first: bool, + /// Probability floor for numeric stability. + min_prob: f64, + /// Compact rollback journal used while checkpoint scopes are active. + checkpoint_journal: Vec, + /// Number of active checkpoints that require journaling. + checkpoint_depth: usize, + /// In-flight native byte-prefix progress when stepping MSB-first bits. + native_prefix_progress: Option, + }, + /// RWKV-7 neural predictor. + #[cfg(feature = "backend-rwkv")] + Rwkv7 { + /// RWKV compressor/runtime state. + compressor: rwkvzip::Compressor, + /// Whether the first-token distribution has been primed. + primed: bool, + /// Scratch copy used for update API that borrows immutable PDF. + pdf_scratch: Vec, + /// Probability floor for numeric stability. + min_prob: f64, + }, + /// Mamba-1 neural predictor. + #[cfg(feature = "backend-mamba")] + Mamba { + /// Mamba compressor/runtime state. + compressor: mambazip::Compressor, + /// Whether the first-token distribution has been primed. + primed: bool, + /// Scratch copy used for update API that borrows immutable PDF. + pdf_scratch: Vec, + /// Probability floor for numeric stability. + min_prob: f64, + }, + /// ZPAQ streaming rate model. + #[cfg(feature = "backend-zpaq")] + Zpaq { + /// ZPAQ rate model state. + model: ZpaqRateModel, + }, + /// Online mixture over experts (Bayes, fading Bayes, switching, MDL). + #[cfg(feature = "backend-mixture")] + Mixture { + /// Active mixture runtime. + runtime: MixtureRuntime, + }, + /// Particle-latent filter ensemble. + #[cfg(feature = "backend-particle")] + Particle { + /// Particle runtime. + runtime: crate::backends::particle::ParticleRuntime, + }, + /// Calibrated wrapper around another predictor. + #[cfg(feature = "backend-calibrated")] + Calibrated { + /// Wrapped predictor whose PDF is calibrated. + base: Box, + /// Online calibrator state and context features. + core: CalibratorCore, + /// Cached calibrated PDF. + pdf: [f64; 256], + /// Whether `pdf` currently matches wrapped state. + valid: bool, + /// Probability floor used for numerical stability. + min_prob: f64, + }, + /// Internal fallback variant used in ultra-minimal builds. + Disabled { + /// Human-readable failure reason. + reason: String, + }, +} + +#[derive(Clone)] +/// Checkpoint snapshot used for temporary predictor rollback. +/// +/// Most backends use a full cloned predictor snapshot. Sequitur, CTW/FAC-CTW, +/// and ROSA use compact rollback markers to avoid cloning hot-path runtime +/// state. +pub enum RateBackendPredictorCheckpoint { + /// Full predictor clone for backends without specialized checkpointing. + /// + /// Boxed so the enum stays pointer-sized: compact variants (`Ctw`, `Rosa`, + /// `Sequitur`, etc.) are not forced to move ~4 KiB on the stack when passed + /// through this type. The clone+heap cost is negligible versus `self.clone()`. + /// Byte-prefix session buffering still boxes checkpoints out-of-line when + /// stored in long-lived session state. + Full(Box), + /// Compact ROSA journal marker for [`RateBackendPredictor::Rosa`]. + #[cfg(feature = "backend-rosa")] + Rosa { + /// Length of the rollback journal to restore when unwinding the checkpoint. + journal_len: usize, + }, + /// Compact Sequitur undo marker for [`RateBackendPredictor::Sequitur`]. + #[cfg(feature = "backend-sequitur")] + Sequitur(SequiturCheckpoint), + /// Compact CTW journal marker for [`RateBackendPredictor::Ctw`]. + #[cfg(feature = "backend-ctw")] + Ctw { + /// Length of the rollback journal to restore when unwinding the checkpoint. + journal_len: usize, + /// In-flight native prefix progress captured with the checkpoint. + native_prefix_progress: Option, + }, + /// Compact FAC-CTW journal marker for [`RateBackendPredictor::FacCtw`]. + #[cfg(feature = "backend-ctw")] + FacCtw { + /// Length of the rollback journal to restore when unwinding the checkpoint. + journal_len: usize, + /// In-flight native prefix progress captured with the checkpoint. + native_prefix_progress: Option, + }, + /// Composite checkpoint for calibrated predictors. + #[cfg(feature = "backend-calibrated")] + Calibrated(Box), + /// Composite checkpoint for mixture predictors. + #[cfg(feature = "backend-mixture")] + Mixture(Box), +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +/// Internal AC-CTW journal event used to restore predictor state from checkpoints. +#[doc(hidden)] +pub enum CtwUndoOp { + /// Symbol update applied in learning mode. + LearnedSymbol, + /// One native prefix bit applied in learning mode. + LearnedBit, + /// Symbol update applied in frozen/scoring mode. + FrozenSymbol, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +/// Internal FAC-CTW journal event used to restore predictor state from checkpoints. +#[doc(hidden)] +pub enum FacCtwUndoOp { + /// Symbol update applied in learning mode. + LearnedSymbol, + /// One native prefix bit applied in learning mode. + LearnedBit { bit_idx: usize }, + /// Symbol update applied in frozen/scoring mode. + FrozenSymbol, +} + +#[derive(Clone)] +#[cfg(feature = "backend-rosa")] +/// Internal ROSA journal event used to restore predictor state from checkpoints. +#[doc(hidden)] +pub enum RosaPredictorUndo { + Learned(Box), + FrozenCursor { previous_last: i32 }, +} + +#[derive(Clone)] +#[cfg(feature = "backend-calibrated")] +/// Internal checkpoint payload for [`RateBackendPredictor::Calibrated`]. +/// +/// This stores wrapped predictor state plus calibrator caches so temporary +/// lookahead scoring can rollback without rebuilding runtime objects. +pub struct CalibratedPredictorCheckpoint { + base: Box, + core: CalibratorCore, + pdf: [f64; 256], + valid: bool, +} + +enum RateBackendPredictorLifecycleCheckpoint { + NotNeeded, + Full(Box), + #[cfg(feature = "backend-ctw")] + Ctw { + tree: Option, + native_prefix_progress: Option, + }, + #[cfg(feature = "backend-ctw")] + FacCtw { + tree: Option, + native_prefix_progress: Option, + }, + #[cfg(feature = "backend-ppmd")] + Ppmd(PpmdLifecycleSnapshot), + #[cfg(feature = "backend-match")] + Match(MatchModelLifecycleSnapshot), + #[cfg(feature = "backend-match")] + SparseMatch(MatchModelLifecycleSnapshot), + #[cfg(feature = "backend-sequitur")] + Sequitur(SequiturLifecycleSnapshot), + #[cfg(feature = "backend-calibrated")] + Calibrated(Box), + #[cfg(feature = "backend-mixture")] + Mixture(Box), +} + +#[cfg(feature = "backend-calibrated")] +struct CalibratedPredictorLifecycleCheckpoint { + base: Box, + core: CalibratorCore, + pdf: [f64; 256], + valid: bool, +} + +#[cfg(feature = "backend-ctw")] +fn restore_ctw_checkpoint( + tree: &mut ContextTree, + bits_per_symbol: usize, + checkpoint_journal: &mut Vec, + target_len: usize, +) { + let bits = bits_per_symbol.clamp(1, 8); + while checkpoint_journal.len() > target_len { + match checkpoint_journal + .pop() + .expect("ctw checkpoint journal underflow") + { + CtwUndoOp::LearnedSymbol => { + for _ in 0..bits { + tree.revert(); + } + } + CtwUndoOp::LearnedBit => { + tree.revert(); + } + CtwUndoOp::FrozenSymbol => { + for _ in 0..bits { + tree.revert_history(); + } + } + } + } +} + +#[cfg(feature = "backend-ctw")] +fn restore_fac_ctw_checkpoint( + tree: &mut FacContextTree, + bits_per_symbol: usize, + checkpoint_journal: &mut Vec, + target_len: usize, +) { + let bits = bits_per_symbol.clamp(1, 8); + while checkpoint_journal.len() > target_len { + match checkpoint_journal + .pop() + .expect("ctw checkpoint journal underflow") + { + FacCtwUndoOp::LearnedSymbol => { + for bit_idx in (0..bits).rev() { + tree.revert(bit_idx); + } + } + FacCtwUndoOp::LearnedBit { bit_idx } => { + tree.revert(bit_idx); + } + FacCtwUndoOp::FrozenSymbol => { + tree.revert_history(bits); + } + } + } +} + +impl RateBackendPredictor { + /// Create a new online predictor from a compiled rate backend plan. + pub fn try_from_compiled(backend: &CompiledRateBackend, min_prob: f64) -> Result { + crate::runtime::build_rate_backend_predictor(backend, min_prob) + } + + /// Create a new online predictor from a rate backend configuration. + pub fn try_from_backend(backend: RateBackend, min_prob: f64) -> Result { + let compiled = backend.compile().map_err(|err| err.to_string())?; + Self::try_from_compiled(&compiled, min_prob) + } + + /// Create a new online predictor from a rate backend configuration. + pub fn from_backend(backend: RateBackend, min_prob: f64) -> Self { + Self::try_from_backend(backend, min_prob) + .unwrap_or_else(|err| panic!("failed to build RateBackendPredictor: {err}")) + } + + /// Create a new online predictor from a compiled rate backend plan. + pub fn from_compiled(backend: &CompiledRateBackend, min_prob: f64) -> Self { + Self::try_from_compiled(backend, min_prob) + .unwrap_or_else(|err| panic!("failed to build RateBackendPredictor: {err}")) + } + + /// Human-readable default name for a backend. + pub fn default_name(backend: &RateBackend) -> String { + backend + .compile() + .map(|compiled| compiled.default_name()) + .unwrap_or_else(|_| { + backend + .descriptor() + .map(|descriptor| format!("{}(invalid)", descriptor.canonical)) + .unwrap_or_else(|_| "backend(invalid)".to_string()) + }) + } + + fn lifecycle_checkpoint( + &mut self, + op: OnlineBytePredictorLifecycleOp, + ) -> RateBackendPredictorLifecycleCheckpoint { + #[cfg(feature = "backend-ctw")] + if matches!( + self, + RateBackendPredictor::Ctw { + native_prefix_progress: Some(bits), + .. + } | RateBackendPredictor::FacCtw { + native_prefix_progress: Some(bits), + .. + } if *bits > 0 + ) { + return RateBackendPredictorLifecycleCheckpoint::Full(Box::new(self.clone())); + } + + match self { + #[cfg(feature = "backend-rosa")] + RateBackendPredictor::Rosa { .. } => match op { + OnlineBytePredictorLifecycleOp::FinishStream + | OnlineBytePredictorLifecycleOp::BeginStream => { + RateBackendPredictorLifecycleCheckpoint::NotNeeded + } + OnlineBytePredictorLifecycleOp::ResetFrozen + | OnlineBytePredictorLifecycleOp::BeginFreshStream => { + RateBackendPredictorLifecycleCheckpoint::Full(Box::new(self.clone())) + } + }, + #[cfg(feature = "backend-match")] + RateBackendPredictor::Match { model, .. } => match op { + OnlineBytePredictorLifecycleOp::FinishStream + | OnlineBytePredictorLifecycleOp::BeginStream => { + RateBackendPredictorLifecycleCheckpoint::NotNeeded + } + OnlineBytePredictorLifecycleOp::ResetFrozen + | OnlineBytePredictorLifecycleOp::BeginFreshStream => { + RateBackendPredictorLifecycleCheckpoint::Match(model.lifecycle_snapshot()) + } + }, + #[cfg(feature = "backend-match")] + RateBackendPredictor::SparseMatch { model, .. } => match op { + OnlineBytePredictorLifecycleOp::FinishStream + | OnlineBytePredictorLifecycleOp::BeginStream => { + RateBackendPredictorLifecycleCheckpoint::NotNeeded + } + OnlineBytePredictorLifecycleOp::ResetFrozen + | OnlineBytePredictorLifecycleOp::BeginFreshStream => { + RateBackendPredictorLifecycleCheckpoint::SparseMatch(model.lifecycle_snapshot()) + } + }, + #[cfg(feature = "backend-ppmd")] + RateBackendPredictor::Ppmd { model, .. } => match op { + OnlineBytePredictorLifecycleOp::FinishStream + | OnlineBytePredictorLifecycleOp::BeginStream => { + RateBackendPredictorLifecycleCheckpoint::NotNeeded + } + OnlineBytePredictorLifecycleOp::ResetFrozen + | OnlineBytePredictorLifecycleOp::BeginFreshStream => { + RateBackendPredictorLifecycleCheckpoint::Ppmd(model.lifecycle_snapshot()) + } + }, + #[cfg(feature = "backend-sequitur")] + RateBackendPredictor::Sequitur { model, .. } => match op { + OnlineBytePredictorLifecycleOp::FinishStream => { + RateBackendPredictorLifecycleCheckpoint::NotNeeded + } + OnlineBytePredictorLifecycleOp::BeginStream + | OnlineBytePredictorLifecycleOp::ResetFrozen + | OnlineBytePredictorLifecycleOp::BeginFreshStream => { + RateBackendPredictorLifecycleCheckpoint::Sequitur(model.lifecycle_snapshot()) + } + }, + #[cfg(feature = "backend-ctw")] + RateBackendPredictor::Ctw { + tree, + native_prefix_progress, + .. + } => match op { + OnlineBytePredictorLifecycleOp::ResetFrozen + | OnlineBytePredictorLifecycleOp::BeginFreshStream => { + RateBackendPredictorLifecycleCheckpoint::Ctw { + tree: Some(tree.lifecycle_snapshot()), + native_prefix_progress: *native_prefix_progress, + } + } + OnlineBytePredictorLifecycleOp::FinishStream + | OnlineBytePredictorLifecycleOp::BeginStream => { + if native_prefix_progress.is_some() { + RateBackendPredictorLifecycleCheckpoint::Ctw { + tree: None, + native_prefix_progress: *native_prefix_progress, + } + } else { + RateBackendPredictorLifecycleCheckpoint::NotNeeded + } + } + }, + #[cfg(feature = "backend-ctw")] + RateBackendPredictor::FacCtw { + tree, + native_prefix_progress, + .. + } => match op { + OnlineBytePredictorLifecycleOp::ResetFrozen + | OnlineBytePredictorLifecycleOp::BeginFreshStream => { + RateBackendPredictorLifecycleCheckpoint::FacCtw { + tree: Some(tree.lifecycle_snapshot()), + native_prefix_progress: *native_prefix_progress, + } + } + OnlineBytePredictorLifecycleOp::FinishStream + | OnlineBytePredictorLifecycleOp::BeginStream => { + if native_prefix_progress.is_some() { + RateBackendPredictorLifecycleCheckpoint::FacCtw { + tree: None, + native_prefix_progress: *native_prefix_progress, + } + } else { + RateBackendPredictorLifecycleCheckpoint::NotNeeded + } + } + }, + #[cfg(feature = "backend-zpaq")] + RateBackendPredictor::Zpaq { .. } => match op { + OnlineBytePredictorLifecycleOp::FinishStream + | OnlineBytePredictorLifecycleOp::ResetFrozen => { + RateBackendPredictorLifecycleCheckpoint::NotNeeded + } + OnlineBytePredictorLifecycleOp::BeginStream + | OnlineBytePredictorLifecycleOp::BeginFreshStream => { + RateBackendPredictorLifecycleCheckpoint::Full(Box::new(self.clone())) + } + }, + #[cfg(feature = "backend-particle")] + RateBackendPredictor::Particle { .. } => match op { + OnlineBytePredictorLifecycleOp::FinishStream + | OnlineBytePredictorLifecycleOp::BeginStream => { + RateBackendPredictorLifecycleCheckpoint::NotNeeded + } + OnlineBytePredictorLifecycleOp::ResetFrozen + | OnlineBytePredictorLifecycleOp::BeginFreshStream => { + RateBackendPredictorLifecycleCheckpoint::Full(Box::new(self.clone())) + } + }, + #[cfg(feature = "backend-rwkv")] + RateBackendPredictor::Rwkv7 { .. } => { + RateBackendPredictorLifecycleCheckpoint::Full(Box::new(self.clone())) + } + #[cfg(feature = "backend-mamba")] + RateBackendPredictor::Mamba { .. } => { + RateBackendPredictorLifecycleCheckpoint::Full(Box::new(self.clone())) + } + #[cfg(feature = "backend-calibrated")] + RateBackendPredictor::Calibrated { + base, + core, + pdf, + valid, + .. + } => RateBackendPredictorLifecycleCheckpoint::Calibrated(Box::new( + CalibratedPredictorLifecycleCheckpoint { + base: Box::new(base.lifecycle_checkpoint(op)), + core: core.clone(), + pdf: *pdf, + valid: *valid, + }, + )), + #[cfg(feature = "backend-mixture")] + RateBackendPredictor::Mixture { runtime } => { + RateBackendPredictorLifecycleCheckpoint::Mixture(Box::new( + runtime.lifecycle_checkpoint(op), + )) + } + RateBackendPredictor::Disabled { .. } => { + RateBackendPredictorLifecycleCheckpoint::NotNeeded + } + } + } + + fn restore_lifecycle_checkpoint( + &mut self, + op: OnlineBytePredictorLifecycleOp, + checkpoint: RateBackendPredictorLifecycleCheckpoint, + ) { + match (self, checkpoint) { + (_, RateBackendPredictorLifecycleCheckpoint::NotNeeded) => {} + (slot, RateBackendPredictorLifecycleCheckpoint::Full(state)) => { + *slot = *state; + } + #[cfg(feature = "backend-ctw")] + ( + RateBackendPredictor::Ctw { + tree, + native_prefix_progress, + .. + }, + RateBackendPredictorLifecycleCheckpoint::Ctw { + tree: tree_snapshot, + native_prefix_progress: prefix, + }, + ) => { + if let Some(snapshot) = tree_snapshot { + tree.restore_lifecycle_snapshot(snapshot); + } + *native_prefix_progress = prefix; + } + #[cfg(feature = "backend-ctw")] + ( + RateBackendPredictor::FacCtw { + tree, + native_prefix_progress, + .. + }, + RateBackendPredictorLifecycleCheckpoint::FacCtw { + tree: tree_snapshot, + native_prefix_progress: prefix, + }, + ) => { + if let Some(snapshot) = tree_snapshot { + tree.restore_lifecycle_snapshot(snapshot); + } + *native_prefix_progress = prefix; + } + #[cfg(feature = "backend-ppmd")] + ( + RateBackendPredictor::Ppmd { model, .. }, + RateBackendPredictorLifecycleCheckpoint::Ppmd(snapshot), + ) => model.restore_lifecycle_snapshot(snapshot), + #[cfg(feature = "backend-match")] + ( + RateBackendPredictor::Match { model, .. }, + RateBackendPredictorLifecycleCheckpoint::Match(snapshot), + ) => model.restore_lifecycle_snapshot(snapshot), + #[cfg(feature = "backend-match")] + ( + RateBackendPredictor::SparseMatch { model, .. }, + RateBackendPredictorLifecycleCheckpoint::SparseMatch(snapshot), + ) => model.restore_lifecycle_snapshot(snapshot), + #[cfg(feature = "backend-sequitur")] + ( + RateBackendPredictor::Sequitur { model, .. }, + RateBackendPredictorLifecycleCheckpoint::Sequitur(snapshot), + ) => model.restore_lifecycle_snapshot(snapshot), + #[cfg(feature = "backend-calibrated")] + ( + RateBackendPredictor::Calibrated { + base, + core, + pdf, + valid, + .. + }, + RateBackendPredictorLifecycleCheckpoint::Calibrated(checkpoint), + ) => { + base.restore_lifecycle_checkpoint(op, *checkpoint.base); + *core = checkpoint.core; + *pdf = checkpoint.pdf; + *valid = checkpoint.valid; + } + #[cfg(feature = "backend-mixture")] + ( + RateBackendPredictor::Mixture { runtime }, + RateBackendPredictorLifecycleCheckpoint::Mixture(checkpoint), + ) => runtime.restore_lifecycle_checkpoint(op, *checkpoint), + #[cfg(feature = "backend-ctw")] + (_, RateBackendPredictorLifecycleCheckpoint::Ctw { .. }) => { + panic!("mismatched RateBackendPredictor lifecycle checkpoint variant") + } + #[cfg(feature = "backend-ctw")] + (_, RateBackendPredictorLifecycleCheckpoint::FacCtw { .. }) => { + panic!("mismatched RateBackendPredictor lifecycle checkpoint variant") + } + #[cfg(feature = "backend-ppmd")] + (_, RateBackendPredictorLifecycleCheckpoint::Ppmd(_)) => { + panic!("mismatched RateBackendPredictor lifecycle checkpoint variant") + } + #[cfg(feature = "backend-match")] + (_, RateBackendPredictorLifecycleCheckpoint::Match(_)) => { + panic!("mismatched RateBackendPredictor lifecycle checkpoint variant") + } + #[cfg(feature = "backend-match")] + (_, RateBackendPredictorLifecycleCheckpoint::SparseMatch(_)) => { + panic!("mismatched RateBackendPredictor lifecycle checkpoint variant") + } + #[cfg(feature = "backend-sequitur")] + (_, RateBackendPredictorLifecycleCheckpoint::Sequitur(_)) => { + panic!("mismatched RateBackendPredictor lifecycle checkpoint variant") + } + #[cfg(feature = "backend-calibrated")] + (_, RateBackendPredictorLifecycleCheckpoint::Calibrated(_)) => { + panic!("mismatched RateBackendPredictor lifecycle checkpoint variant") + } + #[cfg(feature = "backend-mixture")] + (_, RateBackendPredictorLifecycleCheckpoint::Mixture(_)) => { + panic!("mismatched RateBackendPredictor lifecycle checkpoint variant") + } + } + } + + fn discard_lifecycle_checkpoint( + &mut self, + op: OnlineBytePredictorLifecycleOp, + checkpoint: RateBackendPredictorLifecycleCheckpoint, + ) { + match (self, checkpoint) { + (_, RateBackendPredictorLifecycleCheckpoint::NotNeeded) + | (_, RateBackendPredictorLifecycleCheckpoint::Full(_)) => {} + #[cfg(feature = "backend-calibrated")] + ( + RateBackendPredictor::Calibrated { base, .. }, + RateBackendPredictorLifecycleCheckpoint::Calibrated(checkpoint), + ) => base.discard_lifecycle_checkpoint(op, *checkpoint.base), + #[cfg(feature = "backend-mixture")] + ( + RateBackendPredictor::Mixture { runtime }, + RateBackendPredictorLifecycleCheckpoint::Mixture(checkpoint), + ) => runtime.discard_lifecycle_checkpoint(op, *checkpoint), + #[cfg(feature = "backend-ctw")] + ( + RateBackendPredictor::Ctw { .. }, + RateBackendPredictorLifecycleCheckpoint::Ctw { .. }, + ) + | ( + RateBackendPredictor::FacCtw { .. }, + RateBackendPredictorLifecycleCheckpoint::FacCtw { .. }, + ) => {} + #[cfg(feature = "backend-ppmd")] + ( + RateBackendPredictor::Ppmd { .. }, + RateBackendPredictorLifecycleCheckpoint::Ppmd(_), + ) => {} + #[cfg(feature = "backend-match")] + ( + RateBackendPredictor::Match { .. }, + RateBackendPredictorLifecycleCheckpoint::Match(_), + ) + | ( + RateBackendPredictor::SparseMatch { .. }, + RateBackendPredictorLifecycleCheckpoint::SparseMatch(_), + ) => {} + #[cfg(feature = "backend-sequitur")] + ( + RateBackendPredictor::Sequitur { .. }, + RateBackendPredictorLifecycleCheckpoint::Sequitur(_), + ) => {} + #[cfg(feature = "backend-ctw")] + (_, RateBackendPredictorLifecycleCheckpoint::Ctw { .. }) + | (_, RateBackendPredictorLifecycleCheckpoint::FacCtw { .. }) => { + panic!("mismatched RateBackendPredictor lifecycle checkpoint variant") + } + #[cfg(feature = "backend-ppmd")] + (_, RateBackendPredictorLifecycleCheckpoint::Ppmd(_)) => { + panic!("mismatched RateBackendPredictor lifecycle checkpoint variant") + } + #[cfg(feature = "backend-match")] + (_, RateBackendPredictorLifecycleCheckpoint::Match(_)) + | (_, RateBackendPredictorLifecycleCheckpoint::SparseMatch(_)) => { + panic!("mismatched RateBackendPredictor lifecycle checkpoint variant") + } + #[cfg(feature = "backend-sequitur")] + (_, RateBackendPredictorLifecycleCheckpoint::Sequitur(_)) => { + panic!("mismatched RateBackendPredictor lifecycle checkpoint variant") + } + #[cfg(feature = "backend-calibrated")] + (_, RateBackendPredictorLifecycleCheckpoint::Calibrated(_)) => { + panic!("mismatched RateBackendPredictor lifecycle checkpoint variant") + } + #[cfg(feature = "backend-mixture")] + (_, RateBackendPredictorLifecycleCheckpoint::Mixture(_)) => { + panic!("mismatched RateBackendPredictor lifecycle checkpoint variant") + } + } + } + + pub(crate) fn checkpoint(&mut self) -> RateBackendPredictorCheckpoint { + match self { + #[cfg(feature = "backend-rosa")] + RateBackendPredictor::Rosa { + checkpoint_journal, + checkpoint_depth, + .. + } => { + *checkpoint_depth = checkpoint_depth.saturating_add(1); + RateBackendPredictorCheckpoint::Rosa { + journal_len: checkpoint_journal.len(), + } + } + #[cfg(feature = "backend-sequitur")] + RateBackendPredictor::Sequitur { model, .. } => { + RateBackendPredictorCheckpoint::Sequitur(model.checkpoint()) + } + #[cfg(feature = "backend-ctw")] + RateBackendPredictor::Ctw { + checkpoint_journal, + checkpoint_depth, + native_prefix_progress, + .. + } => { + *checkpoint_depth = checkpoint_depth.saturating_add(1); + RateBackendPredictorCheckpoint::Ctw { + journal_len: checkpoint_journal.len(), + native_prefix_progress: *native_prefix_progress, + } + } + #[cfg(feature = "backend-ctw")] + RateBackendPredictor::FacCtw { + checkpoint_journal, + checkpoint_depth, + native_prefix_progress, + .. + } => { + *checkpoint_depth = checkpoint_depth.saturating_add(1); + RateBackendPredictorCheckpoint::FacCtw { + journal_len: checkpoint_journal.len(), + native_prefix_progress: *native_prefix_progress, + } + } + #[cfg(feature = "backend-calibrated")] + RateBackendPredictor::Calibrated { + base, + core, + pdf, + valid, + .. + } => RateBackendPredictorCheckpoint::Calibrated(Box::new( + CalibratedPredictorCheckpoint { + base: Box::new(base.checkpoint()), + core: core.clone(), + pdf: *pdf, + valid: *valid, + }, + )), + #[cfg(feature = "backend-mixture")] + RateBackendPredictor::Mixture { runtime } => runtime + .checkpoint() + .map(|checkpoint| RateBackendPredictorCheckpoint::Mixture(Box::new(checkpoint))) + .unwrap_or_else(|| RateBackendPredictorCheckpoint::Full(Box::new(self.clone()))), + _ => RateBackendPredictorCheckpoint::Full(Box::new(self.clone())), + } + } + + pub(crate) fn abort_empty_native_msb_byte_prefix(&mut self) -> Result { + match self { + #[cfg(feature = "backend-ctw")] + RateBackendPredictor::Ctw { + native_prefix_progress, + .. + } => match *native_prefix_progress { + Some(0) => { + *native_prefix_progress = None; + Ok(true) + } + Some(bits) => Err(format!( + "native MSB-first byte-prefix abort requires zero observed bits, got {bits}" + )), + None => Ok(false), + }, + #[cfg(feature = "backend-ctw")] + RateBackendPredictor::FacCtw { + native_prefix_progress, + .. + } => match *native_prefix_progress { + Some(0) => { + *native_prefix_progress = None; + Ok(true) + } + Some(bits) => Err(format!( + "native MSB-first byte-prefix abort requires zero observed bits, got {bits}" + )), + None => Ok(false), + }, + #[cfg(feature = "backend-mixture")] + RateBackendPredictor::Mixture { runtime } => { + runtime.abort_empty_native_msb_byte_prefix() + } + _ => Ok(false), + } + } + + pub(crate) fn restore_checkpoint(&mut self, checkpoint: &RateBackendPredictorCheckpoint) { + match (self, checkpoint) { + #[cfg(feature = "backend-rosa")] + ( + RateBackendPredictor::Rosa { + model, + checkpoint_journal, + .. + }, + RateBackendPredictorCheckpoint::Rosa { journal_len }, + ) => { + while checkpoint_journal.len() > *journal_len { + match checkpoint_journal + .pop() + .expect("rosa checkpoint journal underflow") + { + RosaPredictorUndo::Learned(tx) => model.rollback_tx(*tx), + RosaPredictorUndo::FrozenCursor { previous_last } => { + model.restore_conditioning_cursor(previous_last) + } + } + } + } + #[cfg(feature = "backend-sequitur")] + ( + RateBackendPredictor::Sequitur { model, .. }, + RateBackendPredictorCheckpoint::Sequitur(ck), + ) => { + model.restore(ck); + } + #[cfg(feature = "backend-ctw")] + ( + RateBackendPredictor::Ctw { + tree, + bits_per_symbol, + checkpoint_journal, + native_prefix_progress, + .. + }, + RateBackendPredictorCheckpoint::Ctw { + journal_len, + native_prefix_progress: checkpoint_progress, + }, + ) => { + restore_ctw_checkpoint(tree, *bits_per_symbol, checkpoint_journal, *journal_len); + *native_prefix_progress = *checkpoint_progress; + } + #[cfg(feature = "backend-ctw")] + ( + RateBackendPredictor::FacCtw { + tree, + bits_per_symbol, + checkpoint_journal, + native_prefix_progress, + .. + }, + RateBackendPredictorCheckpoint::FacCtw { + journal_len, + native_prefix_progress: checkpoint_progress, + }, + ) => { + restore_fac_ctw_checkpoint( + tree, + *bits_per_symbol, + checkpoint_journal, + *journal_len, + ); + *native_prefix_progress = *checkpoint_progress; + } + #[cfg(feature = "backend-calibrated")] + ( + RateBackendPredictor::Calibrated { + base, + core, + pdf, + valid, + .. + }, + RateBackendPredictorCheckpoint::Calibrated(ck), + ) => { + base.restore_checkpoint(&ck.base); + *core = ck.core.clone(); + *pdf = ck.pdf; + *valid = ck.valid; + } + #[cfg(feature = "backend-mixture")] + ( + RateBackendPredictor::Mixture { runtime }, + RateBackendPredictorCheckpoint::Mixture(ck), + ) => { + runtime.restore_checkpoint(ck); + } + (slot, RateBackendPredictorCheckpoint::Full(state)) => { + *slot = state.as_ref().clone(); + } + #[cfg(feature = "backend-rosa")] + (_, RateBackendPredictorCheckpoint::Rosa { .. }) => { + panic!("mismatched RateBackendPredictor checkpoint variant") + } + #[cfg(feature = "backend-ctw")] + (_, RateBackendPredictorCheckpoint::Ctw { .. }) => { + panic!("mismatched RateBackendPredictor checkpoint variant") + } + #[cfg(feature = "backend-ctw")] + (_, RateBackendPredictorCheckpoint::FacCtw { .. }) => { + panic!("mismatched RateBackendPredictor checkpoint variant") + } + #[cfg(feature = "backend-calibrated")] + (_, RateBackendPredictorCheckpoint::Calibrated(_)) => { + panic!("mismatched RateBackendPredictor checkpoint variant") + } + #[cfg(feature = "backend-mixture")] + (_, RateBackendPredictorCheckpoint::Mixture(_)) => { + panic!("mismatched RateBackendPredictor checkpoint variant") + } + #[cfg(feature = "backend-sequitur")] + (_, RateBackendPredictorCheckpoint::Sequitur(_)) => { + panic!("mismatched RateBackendPredictor checkpoint variant") + } + } + } + + pub(crate) fn clear_checkpoints_if_supported(&mut self) { + match self { + #[cfg(feature = "backend-rosa")] + RateBackendPredictor::Rosa { + checkpoint_journal, + checkpoint_depth, + .. + } => { + checkpoint_journal.clear(); + *checkpoint_depth = 0; + } + #[cfg(feature = "backend-sequitur")] + RateBackendPredictor::Sequitur { model, .. } => model.clear_checkpoints(), + #[cfg(feature = "backend-ctw")] + RateBackendPredictor::Ctw { + checkpoint_journal, + checkpoint_depth, + native_prefix_progress, + .. + } => { + checkpoint_journal.clear(); + *checkpoint_depth = 0; + *native_prefix_progress = None; + } + #[cfg(feature = "backend-ctw")] + RateBackendPredictor::FacCtw { + checkpoint_journal, + checkpoint_depth, + native_prefix_progress, + .. + } => { + checkpoint_journal.clear(); + *checkpoint_depth = 0; + *native_prefix_progress = None; + } + #[cfg(feature = "backend-calibrated")] + RateBackendPredictor::Calibrated { base, .. } => { + base.clear_checkpoints_if_supported(); + } + #[cfg(feature = "backend-mixture")] + RateBackendPredictor::Mixture { runtime } => { + runtime.clear_checkpoints_if_supported(); + } + _ => {} + } + } + + pub(crate) fn discard_checkpoint(&mut self, checkpoint: RateBackendPredictorCheckpoint) { + match (self, checkpoint) { + #[cfg(feature = "backend-rosa")] + ( + RateBackendPredictor::Rosa { + checkpoint_depth, .. + }, + RateBackendPredictorCheckpoint::Rosa { .. }, + ) => { + *checkpoint_depth = checkpoint_depth.saturating_sub(1); + } + #[cfg(feature = "backend-sequitur")] + ( + RateBackendPredictor::Sequitur { .. }, + RateBackendPredictorCheckpoint::Sequitur(_), + ) => {} + #[cfg(feature = "backend-ctw")] + ( + RateBackendPredictor::Ctw { + checkpoint_depth, .. + }, + RateBackendPredictorCheckpoint::Ctw { .. }, + ) => { + *checkpoint_depth = checkpoint_depth.saturating_sub(1); + } + #[cfg(feature = "backend-ctw")] + ( + RateBackendPredictor::FacCtw { + checkpoint_depth, .. + }, + RateBackendPredictorCheckpoint::FacCtw { .. }, + ) => { + *checkpoint_depth = checkpoint_depth.saturating_sub(1); + } + #[cfg(feature = "backend-calibrated")] + ( + RateBackendPredictor::Calibrated { base, .. }, + RateBackendPredictorCheckpoint::Calibrated(checkpoint), + ) => { + base.discard_checkpoint(*checkpoint.base); + } + #[cfg(feature = "backend-mixture")] + ( + RateBackendPredictor::Mixture { runtime }, + RateBackendPredictorCheckpoint::Mixture(checkpoint), + ) => { + runtime.discard_checkpoint(*checkpoint); + } + (_, RateBackendPredictorCheckpoint::Full(_)) => {} + #[cfg(feature = "backend-rosa")] + (_, RateBackendPredictorCheckpoint::Rosa { .. }) => { + panic!("mismatched RateBackendPredictor checkpoint variant") + } + #[cfg(feature = "backend-ctw")] + (_, RateBackendPredictorCheckpoint::Ctw { .. }) => { + panic!("mismatched RateBackendPredictor checkpoint variant") + } + #[cfg(feature = "backend-ctw")] + (_, RateBackendPredictorCheckpoint::FacCtw { .. }) => { + panic!("mismatched RateBackendPredictor checkpoint variant") + } + #[cfg(feature = "backend-calibrated")] + (_, RateBackendPredictorCheckpoint::Calibrated(_)) => { + panic!("mismatched RateBackendPredictor checkpoint variant") + } + #[cfg(feature = "backend-mixture")] + (_, RateBackendPredictorCheckpoint::Mixture(_)) => { + panic!("mismatched RateBackendPredictor checkpoint variant") + } + #[cfg(feature = "backend-sequitur")] + (_, RateBackendPredictorCheckpoint::Sequitur(_)) => { + panic!("mismatched RateBackendPredictor checkpoint variant") + } + } + } +} + +impl OnlineBytePredictor for RateBackendPredictor { + fn supports_frozen_reset(&self) -> bool { + match self { + #[cfg(feature = "backend-zpaq")] + RateBackendPredictor::Zpaq { .. } => false, + #[cfg(feature = "backend-mixture")] + RateBackendPredictor::Mixture { runtime } => runtime.supports_frozen_reset(), + #[cfg(feature = "backend-calibrated")] + RateBackendPredictor::Calibrated { base, .. } => base.supports_frozen_reset(), + _ => true, + } + } + + fn begin_fresh_stream(&mut self, total_symbols: Option) -> Result<(), String> { + match self { + #[cfg(feature = "backend-mixture")] + RateBackendPredictor::Mixture { runtime } => runtime.begin_fresh_stream(total_symbols), + #[cfg(feature = "backend-calibrated")] + RateBackendPredictor::Calibrated { + base, + core, + pdf, + valid, + .. + } => { + base.begin_fresh_stream(total_symbols)?; + reset_calibrated_wrapper_state(core, pdf, valid); + Ok(()) + } + _ => { + if self.supports_frozen_reset() { + self.reset_frozen(total_symbols) + } else { + self.begin_stream(total_symbols) + } + } + } + } + + fn begin_stream(&mut self, total_symbols: Option) -> Result<(), String> { + self.finish_stream()?; + match self { + #[cfg(feature = "backend-rosa")] + RateBackendPredictor::Rosa { + model, + checkpoint_depth, + .. + } => { + if *checkpoint_depth > 0 { + return Err( + "rosa lifecycle reset cannot run while prediction checkpoints are active" + .to_string(), + ); + } + if let Some(total) = total_symbols { + let reserve = usize::try_from(total).unwrap_or(usize::MAX / 4); + model.reserve_for_stream(reserve); + } + Ok(()) + } + #[cfg(feature = "backend-match")] + RateBackendPredictor::Match { .. } => Ok(()), + #[cfg(feature = "backend-match")] + RateBackendPredictor::SparseMatch { .. } => Ok(()), + #[cfg(feature = "backend-ppmd")] + RateBackendPredictor::Ppmd { .. } => Ok(()), + #[cfg(feature = "backend-sequitur")] + RateBackendPredictor::Sequitur { model, .. } => { + if model.checkpoints_active() { + return Err( + "sequitur lifecycle begin_stream cannot run while prediction checkpoints are active" + .to_string(), + ); + } + model.begin_stream(total_symbols); + Ok(()) + } + #[cfg(feature = "backend-ctw")] + RateBackendPredictor::Ctw { + native_prefix_progress, + .. + } => { + *native_prefix_progress = None; + Ok(()) + } + #[cfg(feature = "backend-ctw")] + RateBackendPredictor::FacCtw { + native_prefix_progress, + .. + } => { + *native_prefix_progress = None; + Ok(()) + } + #[cfg(feature = "backend-zpaq")] + RateBackendPredictor::Zpaq { model } => { + model.begin_stream(); + Ok(()) + } + #[cfg(feature = "backend-particle")] + RateBackendPredictor::Particle { .. } => Ok(()), + #[cfg(feature = "backend-rwkv")] + RateBackendPredictor::Rwkv7 { compressor, .. } => compressor + .begin_online_policy_stream(total_symbols) + .map_err(|e| e.to_string()), + #[cfg(feature = "backend-mamba")] + RateBackendPredictor::Mamba { compressor, .. } => compressor + .begin_online_policy_stream(total_symbols) + .map_err(|e| e.to_string()), + #[cfg(feature = "backend-mixture")] + RateBackendPredictor::Mixture { runtime } => runtime.begin_stream(total_symbols), + #[cfg(feature = "backend-calibrated")] + RateBackendPredictor::Calibrated { base, .. } => base.begin_stream(total_symbols), + RateBackendPredictor::Disabled { reason } => Err(reason.clone()), + } + } + + fn finish_stream(&mut self) -> Result<(), String> { + match self { + #[cfg(feature = "backend-rosa")] + RateBackendPredictor::Rosa { .. } => Ok(()), + #[cfg(feature = "backend-match")] + RateBackendPredictor::Match { .. } => Ok(()), + #[cfg(feature = "backend-match")] + RateBackendPredictor::SparseMatch { .. } => Ok(()), + #[cfg(feature = "backend-ppmd")] + RateBackendPredictor::Ppmd { .. } => Ok(()), + #[cfg(feature = "backend-ctw")] + RateBackendPredictor::Ctw { + native_prefix_progress, + .. + } => { + *native_prefix_progress = None; + Ok(()) + } + #[cfg(feature = "backend-ctw")] + RateBackendPredictor::FacCtw { + native_prefix_progress, + .. + } => { + *native_prefix_progress = None; + Ok(()) + } + #[cfg(feature = "backend-zpaq")] + RateBackendPredictor::Zpaq { .. } => Ok(()), + #[cfg(feature = "backend-particle")] + RateBackendPredictor::Particle { .. } => Ok(()), + #[cfg(feature = "backend-sequitur")] + RateBackendPredictor::Sequitur { model, .. } => { + model.finish_stream(); + Ok(()) + } + #[cfg(feature = "backend-rwkv")] + RateBackendPredictor::Rwkv7 { compressor, .. } => compressor + .finish_online_policy_stream() + .map_err(|e| e.to_string()), + #[cfg(feature = "backend-mamba")] + RateBackendPredictor::Mamba { compressor, .. } => compressor + .finish_online_policy_stream() + .map_err(|e| e.to_string()), + #[cfg(feature = "backend-mixture")] + RateBackendPredictor::Mixture { runtime } => runtime.finish_stream(), + #[cfg(feature = "backend-calibrated")] + RateBackendPredictor::Calibrated { base, .. } => base.finish_stream(), + RateBackendPredictor::Disabled { .. } => Ok(()), + } + } + + fn checkpoint_if_supported(&mut self) -> Option { + Some(OnlineBytePredictorCheckpoint::rate_backend( + self.checkpoint(), + )) + } + + fn restore_checkpoint_if_supported( + &mut self, + checkpoint: &OnlineBytePredictorCheckpoint, + ) -> bool { + match &checkpoint.0 { + OnlineBytePredictorCheckpointKind::RateBackend(checkpoint) => { + self.restore_checkpoint(checkpoint); + true + } + } + } + + fn discard_checkpoint_if_supported( + &mut self, + checkpoint: OnlineBytePredictorCheckpoint, + ) -> bool { + match checkpoint.0 { + OnlineBytePredictorCheckpointKind::RateBackend(checkpoint) => { + self.discard_checkpoint(checkpoint); + true + } + } + } + + fn clear_checkpoints_if_supported(&mut self) { + RateBackendPredictor::clear_checkpoints_if_supported(self); + } + + fn log_prob(&mut self, symbol: u8) -> f64 { + match self { + #[cfg(feature = "backend-rosa")] + RateBackendPredictor::Rosa { + model, min_prob, .. + } => { + let p = clamp_prob(model.prob_for_last(symbol as u32), *min_prob); + p.ln() + } + #[cfg(feature = "backend-match")] + RateBackendPredictor::Match { model, min_prob } => model.log_prob(symbol, *min_prob), + #[cfg(feature = "backend-match")] + RateBackendPredictor::SparseMatch { model, min_prob } => { + model.log_prob(symbol, *min_prob) + } + #[cfg(feature = "backend-ppmd")] + RateBackendPredictor::Ppmd { model, min_prob } => model.log_prob(symbol, *min_prob), + #[cfg(feature = "backend-sequitur")] + RateBackendPredictor::Sequitur { model, min_prob } => model.log_prob(symbol, *min_prob), + #[cfg(feature = "backend-ctw")] + RateBackendPredictor::Ctw { + tree, + bits_per_symbol, + min_prob, + .. + } => ctw_log_prob_msb(tree, symbol, *bits_per_symbol, *min_prob), + #[cfg(feature = "backend-ctw")] + RateBackendPredictor::FacCtw { + tree, + bits_per_symbol, + msb_first, + min_prob, + .. + } => { + let log_before = tree.get_log_block_probability(); + for i in 0..*bits_per_symbol { + let bit = if *msb_first { + ctw_symbol_bit_msb(symbol, *bits_per_symbol, i) + } else { + ((symbol >> i) & 1) == 1 + }; + tree.update(bit, i); + } + let log_after = tree.get_log_block_probability(); + for i in (0..*bits_per_symbol).rev() { + tree.revert(i); + } + let logp = log_after - log_before; + if logp.is_finite() { + logp.max(min_prob.ln()) + } else { + min_prob.ln() + } + } + #[cfg(feature = "backend-rwkv")] + RateBackendPredictor::Rwkv7 { + compressor, + primed, + min_prob, + .. + } => { + ensure_rwkv_primed(compressor, primed); + let p = clamp_prob(compressor.pdf_buffer[symbol as usize], *min_prob); + p.ln() + } + #[cfg(feature = "backend-mamba")] + RateBackendPredictor::Mamba { + compressor, + primed, + min_prob, + .. + } => { + if !*primed { + let bias = compressor.online_bias_snapshot(); + let logits = + compressor + .model + .forward(&mut compressor.scratch, 0, &mut compressor.state); + mambazip::Compressor::logits_to_pdf( + logits, + bias.as_deref(), + &mut compressor.pdf_buffer, + ); + *primed = true; + } + let p = clamp_prob(compressor.pdf_buffer[symbol as usize], *min_prob); + p.ln() + } + #[cfg(feature = "backend-zpaq")] + RateBackendPredictor::Zpaq { model } => model.log_prob(symbol), + #[cfg(feature = "backend-mixture")] + RateBackendPredictor::Mixture { runtime } => runtime.peek_log_prob(symbol), + #[cfg(feature = "backend-particle")] + RateBackendPredictor::Particle { runtime } => runtime.peek_log_prob(symbol), + #[cfg(feature = "backend-calibrated")] + RateBackendPredictor::Calibrated { + base, + core, + pdf, + valid, + min_prob, + } => { + if !*valid { + let mut base_logps = [0.0; 256]; + base.fill_log_probs(&mut base_logps); + let mut base_pdf = [0.0; 256]; + for (dst, &lp) in base_pdf.iter_mut().zip(base_logps.iter()) { + *dst = clamp_prob(lp.exp(), *min_prob); + } + core.apply_pdf(&base_pdf, pdf); + *valid = true; + } + pdf[symbol as usize].max(*min_prob).ln() + } + RateBackendPredictor::Disabled { .. } => f64::NEG_INFINITY, + } + } + + fn fill_log_probs(&mut self, out: &mut [f64; 256]) { + match self { + #[cfg(feature = "backend-rosa")] + RateBackendPredictor::Rosa { + model, min_prob, .. + } => { + model.fill_probs_for_last_bytes(out); + for slot in out.iter_mut() { + *slot = clamp_prob(*slot, *min_prob).ln(); + } + } + #[cfg(feature = "backend-match")] + RateBackendPredictor::Match { model, min_prob } => { + let mut pdf = [0.0; 256]; + model.fill_pdf(&mut pdf); + for (slot, &p) in out.iter_mut().zip(pdf.iter()) { + *slot = clamp_prob(p, *min_prob).ln(); + } + } + #[cfg(feature = "backend-match")] + RateBackendPredictor::SparseMatch { model, min_prob } => { + let mut pdf = [0.0; 256]; + model.fill_pdf(&mut pdf); + for (slot, &p) in out.iter_mut().zip(pdf.iter()) { + *slot = clamp_prob(p, *min_prob).ln(); + } + } + #[cfg(feature = "backend-ppmd")] + RateBackendPredictor::Ppmd { model, min_prob } => { + let mut pdf = [0.0; 256]; + model.fill_pdf(&mut pdf); + for (slot, &p) in out.iter_mut().zip(pdf.iter()) { + *slot = clamp_prob(p, *min_prob).ln(); + } + } + #[cfg(feature = "backend-sequitur")] + RateBackendPredictor::Sequitur { model, min_prob } => { + let mut pdf = [0.0; 256]; + model.fill_pdf(&mut pdf); + for (slot, &p) in out.iter_mut().zip(pdf.iter()) { + *slot = clamp_prob(p, *min_prob).ln(); + } + } + #[cfg(feature = "backend-ctw")] + RateBackendPredictor::Ctw { + tree, + bits_per_symbol, + min_prob, + .. + } => fill_ctw_tree_log_probs(tree, *bits_per_symbol, min_prob.ln(), out), + #[cfg(feature = "backend-ctw")] + RateBackendPredictor::FacCtw { + tree, + bits_per_symbol, + msb_first, + min_prob, + .. + } => { + fill_fac_tree_log_probs(tree, *bits_per_symbol, *msb_first, min_prob.ln(), out); + } + #[cfg(feature = "backend-rwkv")] + RateBackendPredictor::Rwkv7 { + compressor, + primed, + min_prob, + .. + } => { + ensure_rwkv_primed(compressor, primed); + for (slot, &p_raw) in out + .iter_mut() + .take(256) + .zip(compressor.pdf_buffer.iter().take(256)) + { + let p = clamp_prob(p_raw, *min_prob); + *slot = p.ln(); + } + } + #[cfg(feature = "backend-mamba")] + RateBackendPredictor::Mamba { + compressor, + primed, + min_prob, + .. + } => { + if !*primed { + let bias = compressor.online_bias_snapshot(); + let logits = + compressor + .model + .forward(&mut compressor.scratch, 0, &mut compressor.state); + mambazip::Compressor::logits_to_pdf( + logits, + bias.as_deref(), + &mut compressor.pdf_buffer, + ); + *primed = true; + } + for (slot, &p_raw) in out + .iter_mut() + .take(256) + .zip(compressor.pdf_buffer.iter().take(256)) + { + let p = clamp_prob(p_raw, *min_prob); + *slot = p.ln(); + } + } + #[cfg(feature = "backend-zpaq")] + RateBackendPredictor::Zpaq { model } => { + model.fill_log_probs(out); + } + #[cfg(feature = "backend-mixture")] + RateBackendPredictor::Mixture { runtime } => { + runtime.fill_log_probs(out); + } + #[cfg(feature = "backend-particle")] + RateBackendPredictor::Particle { runtime } => { + runtime.fill_log_probs_cached(out); + } + #[cfg(feature = "backend-calibrated")] + RateBackendPredictor::Calibrated { + base, + core, + pdf, + valid, + min_prob, + } => { + if !*valid { + let mut base_logps = [0.0; 256]; + base.fill_log_probs(&mut base_logps); + let mut base_pdf = [0.0; 256]; + for (dst, &lp) in base_pdf.iter_mut().zip(base_logps.iter()) { + *dst = clamp_prob(lp.exp(), *min_prob); + } + core.apply_pdf(&base_pdf, pdf); + *valid = true; + } + for (slot, &p) in out.iter_mut().zip(pdf.iter()) { + *slot = clamp_prob(p, *min_prob).ln(); + } + } + RateBackendPredictor::Disabled { .. } => out.fill(-(256.0f64).ln()), + } + } + + fn has_native_msb_byte_prefix(&self) -> bool { + match self { + #[cfg(feature = "backend-ctw")] + RateBackendPredictor::Ctw { + bits_per_symbol, .. + } => *bits_per_symbol == 8, + #[cfg(feature = "backend-ctw")] + RateBackendPredictor::FacCtw { + bits_per_symbol, + msb_first, + .. + } => *bits_per_symbol == 8 && *msb_first, + #[cfg(feature = "backend-mixture")] + RateBackendPredictor::Mixture { runtime } => runtime.has_native_msb_byte_prefix(), + _ => false, + } + } + + fn begin_native_msb_byte_prefix(&mut self) -> Result { + match self { + #[cfg(feature = "backend-ctw")] + RateBackendPredictor::Ctw { + bits_per_symbol, .. + } if *bits_per_symbol != 8 => Ok(false), + #[cfg(feature = "backend-ctw")] + RateBackendPredictor::FacCtw { + bits_per_symbol, + msb_first, + .. + } if *bits_per_symbol != 8 || !*msb_first => Ok(false), + #[cfg(feature = "backend-ctw")] + RateBackendPredictor::Ctw { + native_prefix_progress, + .. + } => { + if native_prefix_progress.is_some() { + return Err( + "native MSB-first byte-prefix step is already active for this predictor" + .to_string(), + ); + } + *native_prefix_progress = Some(0); + Ok(true) + } + #[cfg(feature = "backend-ctw")] + RateBackendPredictor::FacCtw { + native_prefix_progress, + .. + } => { + if native_prefix_progress.is_some() { + return Err( + "native MSB-first byte-prefix step is already active for this predictor" + .to_string(), + ); + } + *native_prefix_progress = Some(0); + Ok(true) + } + #[cfg(feature = "backend-mixture")] + RateBackendPredictor::Mixture { runtime } => runtime.begin_native_msb_byte_prefix(), + _ => Ok(false), + } + } + + fn abort_empty_native_msb_byte_prefix(&mut self) -> Result { + RateBackendPredictor::abort_empty_native_msb_byte_prefix(self) + } + + fn native_msb_prefix_prob_one(&mut self, bit_idx: usize) -> Result { + match self { + #[cfg(feature = "backend-ctw")] + RateBackendPredictor::Ctw { + tree, + min_prob, + native_prefix_progress, + .. + } => { + let next_bit_idx = native_prefix_progress + .as_ref() + .ok_or_else(|| "native MSB-first byte-prefix step is not active".to_string())?; + validate_native_msb_prefix_bit_idx(*next_bit_idx, bit_idx)?; + let p: f64 = tree.predict(true); + Ok(p.clamp(*min_prob, 1.0 - *min_prob)) + } + #[cfg(feature = "backend-ctw")] + RateBackendPredictor::FacCtw { + tree, + min_prob, + native_prefix_progress, + .. + } => { + let next_bit_idx = native_prefix_progress + .as_ref() + .ok_or_else(|| "native MSB-first byte-prefix step is not active".to_string())?; + validate_native_msb_prefix_bit_idx(*next_bit_idx, bit_idx)?; + let p: f64 = tree.predict_one(bit_idx); + Ok(p.clamp(*min_prob, 1.0 - *min_prob)) + } + #[cfg(feature = "backend-mixture")] + RateBackendPredictor::Mixture { runtime } => { + runtime.native_msb_prefix_prob_one(bit_idx) + } + _ => Err("native MSB-first byte-prefix prediction is unavailable".to_string()), + } + } + + fn observe_native_msb_prefix_bit(&mut self, bit_idx: usize, bit: bool) -> Result<(), String> { + match self { + #[cfg(feature = "backend-ctw")] + RateBackendPredictor::Ctw { + tree, + checkpoint_journal, + checkpoint_depth, + native_prefix_progress, + .. + } => { + let next_bit_idx = native_prefix_progress + .as_mut() + .ok_or_else(|| "native MSB-first byte-prefix step is not active".to_string())?; + validate_native_msb_prefix_bit_idx(*next_bit_idx, bit_idx)?; + *next_bit_idx += 1; + tree.update(bit); + if *checkpoint_depth > 0 { + checkpoint_journal.push(CtwUndoOp::LearnedBit); + } + Ok(()) + } + #[cfg(feature = "backend-ctw")] + RateBackendPredictor::FacCtw { + tree, + checkpoint_journal, + checkpoint_depth, + native_prefix_progress, + .. + } => { + let next_bit_idx = native_prefix_progress + .as_mut() + .ok_or_else(|| "native MSB-first byte-prefix step is not active".to_string())?; + validate_native_msb_prefix_bit_idx(*next_bit_idx, bit_idx)?; + *next_bit_idx += 1; + tree.update_predicted(bit, bit_idx); + if *checkpoint_depth > 0 { + checkpoint_journal.push(FacCtwUndoOp::LearnedBit { bit_idx }); + } + Ok(()) + } + #[cfg(feature = "backend-mixture")] + RateBackendPredictor::Mixture { runtime } => { + runtime.observe_native_msb_prefix_bit(bit_idx, bit) + } + _ => Err("native MSB-first byte-prefix stepping is unavailable".to_string()), + } + } + + fn finish_native_msb_byte_prefix(&mut self, symbol: u8) -> Result<(), String> { + match self { + #[cfg(feature = "backend-ctw")] + RateBackendPredictor::Ctw { + native_prefix_progress, + .. + } => { + let next_bit_idx = native_prefix_progress + .as_ref() + .ok_or_else(|| "native MSB-first byte-prefix step is not active".to_string())?; + validate_native_msb_prefix_finish(*next_bit_idx)?; + *native_prefix_progress = None; + let _ = symbol; + Ok(()) + } + #[cfg(feature = "backend-ctw")] + RateBackendPredictor::FacCtw { + native_prefix_progress, + .. + } => { + let next_bit_idx = native_prefix_progress + .as_ref() + .ok_or_else(|| "native MSB-first byte-prefix step is not active".to_string())?; + validate_native_msb_prefix_finish(*next_bit_idx)?; + *native_prefix_progress = None; + let _ = symbol; + Ok(()) + } + #[cfg(feature = "backend-mixture")] + RateBackendPredictor::Mixture { runtime } => { + runtime.finish_native_msb_byte_prefix(symbol) + } + _ => Err("native MSB-first byte-prefix stepping is unavailable".to_string()), + } + } + + fn update(&mut self, symbol: u8) { + match self { + #[cfg(feature = "backend-rosa")] + RateBackendPredictor::Rosa { + model, + checkpoint_journal, + checkpoint_depth, + .. + } => { + if *checkpoint_depth > 0 { + let mut tx = model.begin_tx(); + model.train_sequence_tx(&mut tx, &[symbol]); + checkpoint_journal.push(RosaPredictorUndo::Learned(Box::new(tx))); + } else { + model.train_byte(symbol); + } + } + #[cfg(feature = "backend-match")] + RateBackendPredictor::Match { model, .. } => { + model.update(symbol); + } + #[cfg(feature = "backend-match")] + RateBackendPredictor::SparseMatch { model, .. } => { + model.update(symbol); + } + #[cfg(feature = "backend-ppmd")] + RateBackendPredictor::Ppmd { model, .. } => { + model.update(symbol); + } + #[cfg(feature = "backend-sequitur")] + RateBackendPredictor::Sequitur { model, .. } => { + model.update(symbol); + } + #[cfg(feature = "backend-ctw")] + RateBackendPredictor::Ctw { + tree, + bits_per_symbol, + checkpoint_journal, + checkpoint_depth, + native_prefix_progress, + .. + } => { + debug_assert!( + native_prefix_progress.is_none(), + "ctw symbol update while native byte-prefix step is active" + ); + *native_prefix_progress = None; + for bit_idx in 0..(*bits_per_symbol).clamp(1, 8) { + tree.update(ctw_symbol_bit_msb(symbol, *bits_per_symbol, bit_idx)); + } + if *checkpoint_depth > 0 { + checkpoint_journal.push(CtwUndoOp::LearnedSymbol); + } + } + #[cfg(feature = "backend-ctw")] + RateBackendPredictor::FacCtw { + tree, + bits_per_symbol, + msb_first, + checkpoint_journal, + checkpoint_depth, + native_prefix_progress, + .. + } => { + debug_assert!( + native_prefix_progress.is_none(), + "fac-ctw symbol update while native byte-prefix step is active" + ); + *native_prefix_progress = None; + for i in 0..*bits_per_symbol { + let bit = if *msb_first { + ctw_symbol_bit_msb(symbol, *bits_per_symbol, i) + } else { + ((symbol >> i) & 1) == 1 + }; + tree.update(bit, i); + } + if *checkpoint_depth > 0 { + checkpoint_journal.push(FacCtwUndoOp::LearnedSymbol); + } + } + #[cfg(feature = "backend-rwkv")] + RateBackendPredictor::Rwkv7 { + compressor, primed, .. + } => { + ensure_rwkv_primed(compressor, primed); + compressor + .observe_symbol_from_current_pdf(symbol) + .unwrap_or_else(|e| panic!("rwkv online update failed: {e}")); + } + #[cfg(feature = "backend-mamba")] + RateBackendPredictor::Mamba { + compressor, + primed, + pdf_scratch, + .. + } => { + if !*primed { + let bias = compressor.online_bias_snapshot(); + let logits = + compressor + .model + .forward(&mut compressor.scratch, 0, &mut compressor.state); + mambazip::Compressor::logits_to_pdf( + logits, + bias.as_deref(), + &mut compressor.pdf_buffer, + ); + *primed = true; + } + if pdf_scratch.len() != compressor.pdf_buffer.len() { + pdf_scratch.resize(compressor.pdf_buffer.len(), 0.0); + } + pdf_scratch.copy_from_slice(&compressor.pdf_buffer); + compressor + .online_update_from_pdf(symbol, pdf_scratch) + .unwrap_or_else(|e| panic!("mamba online update failed: {e}")); + let bias = compressor.online_bias_snapshot(); + let logits = compressor.model.forward( + &mut compressor.scratch, + symbol as u32, + &mut compressor.state, + ); + mambazip::Compressor::logits_to_pdf( + logits, + bias.as_deref(), + &mut compressor.pdf_buffer, + ); + } + #[cfg(feature = "backend-zpaq")] + RateBackendPredictor::Zpaq { model } => { + model.update(symbol); + } + #[cfg(feature = "backend-mixture")] + RateBackendPredictor::Mixture { runtime } => { + let _ = runtime.step(symbol); + } + #[cfg(feature = "backend-particle")] + RateBackendPredictor::Particle { runtime } => { + runtime.step(symbol); + } + #[cfg(feature = "backend-calibrated")] + RateBackendPredictor::Calibrated { + base, + core, + pdf, + valid, + .. + } => { + if !*valid { + let mut base_logps = [0.0; 256]; + base.fill_log_probs(&mut base_logps); + let mut base_pdf = [0.0; 256]; + for (dst, &lp) in base_pdf.iter_mut().zip(base_logps.iter()) { + *dst = clamp_prob(lp.exp(), DEFAULT_MIN_PROB); + } + core.apply_pdf(&base_pdf, pdf); + } + core.update(symbol, pdf); + base.update(symbol); + *valid = false; + } + RateBackendPredictor::Disabled { .. } => {} + } + } + + fn log_prob_update(&mut self, symbol: u8) -> f64 { + match self { + #[cfg(feature = "backend-rosa")] + RateBackendPredictor::Rosa { + model, + min_prob, + checkpoint_journal, + checkpoint_depth, + } => { + let p = clamp_prob(model.prob_for_last(symbol as u32), *min_prob); + if *checkpoint_depth > 0 { + let mut tx = model.begin_tx(); + model.train_sequence_tx(&mut tx, &[symbol]); + checkpoint_journal.push(RosaPredictorUndo::Learned(Box::new(tx))); + } else { + model.train_byte(symbol); + } + p.ln() + } + #[cfg(feature = "backend-ctw")] + RateBackendPredictor::Ctw { + tree, + bits_per_symbol, + min_prob, + checkpoint_journal, + checkpoint_depth, + native_prefix_progress, + } => { + debug_assert!( + native_prefix_progress.is_none(), + "ctw symbol log_prob_update while native byte-prefix step is active" + ); + *native_prefix_progress = None; + let logp = ctw_log_prob_update_msb(tree, symbol, *bits_per_symbol, *min_prob); + if *checkpoint_depth > 0 { + checkpoint_journal.push(CtwUndoOp::LearnedSymbol); + } + logp + } + #[cfg(feature = "backend-ctw")] + RateBackendPredictor::FacCtw { + tree, + bits_per_symbol, + msb_first, + min_prob, + checkpoint_journal, + checkpoint_depth, + native_prefix_progress, + } => { + debug_assert!( + native_prefix_progress.is_none(), + "fac-ctw symbol log_prob_update while native byte-prefix step is active" + ); + *native_prefix_progress = None; + let logp = if *msb_first { + let bits = (*bits_per_symbol).clamp(1, 8); + let mut acc = 0.0f64; + for bit_idx in 0..bits { + let bit = ctw_symbol_bit_msb(symbol, bits, bit_idx); + let p = tree.predict(bit, bit_idx).clamp(*min_prob, 1.0 - *min_prob); + acc += p.ln(); + tree.update_predicted(bit, bit_idx); + } + acc + } else { + ctw_log_prob_update_lsb(tree, symbol, *bits_per_symbol, *min_prob) + }; + if *checkpoint_depth > 0 { + checkpoint_journal.push(FacCtwUndoOp::LearnedSymbol); + } + logp + } + _ => { + let logp = self.log_prob(symbol); + self.update(symbol); + logp + } + } + } + + fn reset_frozen(&mut self, total_symbols: Option) -> Result<(), String> { + #[cfg(feature = "backend-zpaq")] + if matches!(self, RateBackendPredictor::Zpaq { .. }) { + return Err("plugin entropy is not supported for zpaq rate backends".to_string()); + } + #[cfg(feature = "backend-mixture")] + if let RateBackendPredictor::Mixture { runtime } = self + && !runtime.supports_frozen_reset() + { + return Err( + "plugin entropy is not supported for mixture rate backends with non-resettable experts" + .to_string(), + ); + } + #[cfg(feature = "backend-calibrated")] + if let RateBackendPredictor::Calibrated { base, .. } = self + && !base.supports_frozen_reset() + { + return base.reset_frozen(total_symbols); + } + + self.finish_stream()?; + match self { + #[cfg(feature = "backend-rosa")] + RateBackendPredictor::Rosa { model, .. } => { + if let Some(total) = total_symbols { + let reserve = usize::try_from(total).unwrap_or(usize::MAX / 4); + model.reserve_for_stream(reserve); + } + model.build_lm_full_bytes_no_finalize_endpos(); + model.reset_conditioning_cursor(); + Ok(()) + } + #[cfg(feature = "backend-match")] + RateBackendPredictor::Match { model, .. } => { + model.reset_history(); + Ok(()) + } + #[cfg(feature = "backend-match")] + RateBackendPredictor::SparseMatch { model, .. } => { + model.reset_history(); + Ok(()) + } + #[cfg(feature = "backend-ppmd")] + RateBackendPredictor::Ppmd { model, .. } => { + model.reset_history(); + Ok(()) + } + #[cfg(feature = "backend-sequitur")] + RateBackendPredictor::Sequitur { model, .. } => { + if model.checkpoints_active() { + return Err( + "sequitur lifecycle reset cannot run while prediction checkpoints are active" + .to_string(), + ); + } + model.reset_frozen(); + Ok(()) + } + #[cfg(feature = "backend-ctw")] + RateBackendPredictor::Ctw { + tree, + checkpoint_depth, + native_prefix_progress, + .. + } => { + if *checkpoint_depth > 0 { + return Err( + "ctw lifecycle reset cannot run while prediction checkpoints are active" + .to_string(), + ); + } + *native_prefix_progress = None; + tree.truncate_history(0); + Ok(()) + } + #[cfg(feature = "backend-ctw")] + RateBackendPredictor::FacCtw { + tree, + checkpoint_depth, + native_prefix_progress, + .. + } => { + if *checkpoint_depth > 0 { + return Err( + "fac-ctw lifecycle reset cannot run while prediction checkpoints are active" + .to_string(), + ); + } + *native_prefix_progress = None; + tree.reset_history_only(); + Ok(()) + } + #[cfg(feature = "backend-rwkv")] + RateBackendPredictor::Rwkv7 { + compressor, primed, .. + } => { + compressor.reset_and_prime(); + *primed = true; + Ok(()) + } + #[cfg(feature = "backend-mamba")] + RateBackendPredictor::Mamba { + compressor, primed, .. + } => { + compressor.reset_and_prime(); + *primed = true; + Ok(()) + } + #[cfg(feature = "backend-zpaq")] + RateBackendPredictor::Zpaq { .. } => { + Err("plugin entropy is not supported for zpaq rate backends".to_string()) + } + #[cfg(feature = "backend-mixture")] + RateBackendPredictor::Mixture { runtime } => runtime.reset_frozen(total_symbols), + #[cfg(feature = "backend-particle")] + RateBackendPredictor::Particle { runtime } => { + runtime.reset_frozen_state(); + Ok(()) + } + #[cfg(feature = "backend-calibrated")] + RateBackendPredictor::Calibrated { + base, + core, + pdf, + valid, + .. + } => { + base.reset_frozen(total_symbols)?; + reset_calibrated_wrapper_state(core, pdf, valid); + Ok(()) + } + RateBackendPredictor::Disabled { reason } => Err(reason.clone()), + } + } + + fn update_frozen(&mut self, symbol: u8) { + match self { + #[cfg(feature = "backend-rosa")] + RateBackendPredictor::Rosa { + model, + checkpoint_journal, + checkpoint_depth, + .. + } => { + if *checkpoint_depth > 0 { + checkpoint_journal.push(RosaPredictorUndo::FrozenCursor { + previous_last: model.conditioning_cursor(), + }); + } + model.advance_conditioning_byte(symbol); + } + #[cfg(feature = "backend-match")] + RateBackendPredictor::Match { model, .. } => { + model.update_history_only(symbol); + } + #[cfg(feature = "backend-match")] + RateBackendPredictor::SparseMatch { model, .. } => { + model.update_history_only(symbol); + } + #[cfg(feature = "backend-ppmd")] + RateBackendPredictor::Ppmd { model, .. } => { + model.update_history_only(symbol); + } + #[cfg(feature = "backend-sequitur")] + RateBackendPredictor::Sequitur { model, .. } => { + model.update_frozen(symbol); + } + #[cfg(feature = "backend-ctw")] + RateBackendPredictor::Ctw { + tree, + bits_per_symbol, + checkpoint_journal, + checkpoint_depth, + native_prefix_progress, + .. + } => { + debug_assert!( + native_prefix_progress.is_none(), + "ctw frozen symbol update while native byte-prefix step is active" + ); + *native_prefix_progress = None; + let bits = (*bits_per_symbol).clamp(1, 8); + let mut history_bits = [false; 8]; + for (bit_idx, slot) in history_bits.iter_mut().enumerate().take(bits) { + *slot = ctw_symbol_bit_msb(symbol, bits, bit_idx); + } + tree.update_history(&history_bits[..bits]); + if *checkpoint_depth > 0 { + checkpoint_journal.push(CtwUndoOp::FrozenSymbol); + } + } + #[cfg(feature = "backend-ctw")] + RateBackendPredictor::FacCtw { + tree, + bits_per_symbol, + msb_first, + checkpoint_journal, + checkpoint_depth, + native_prefix_progress, + .. + } => { + debug_assert!( + native_prefix_progress.is_none(), + "fac-ctw frozen symbol update while native byte-prefix step is active" + ); + *native_prefix_progress = None; + let bits = (*bits_per_symbol).clamp(1, 8); + let mut history_bits = [false; 8]; + for (idx, slot) in history_bits.iter_mut().enumerate().take(bits) { + *slot = if *msb_first { + ctw_symbol_bit_msb(symbol, bits, idx) + } else { + ((symbol >> idx) & 1) == 1 + }; + } + tree.update_history(&history_bits[..bits]); + if *checkpoint_depth > 0 { + checkpoint_journal.push(FacCtwUndoOp::FrozenSymbol); + } + } + #[cfg(feature = "backend-rwkv")] + RateBackendPredictor::Rwkv7 { + compressor, primed, .. + } => { + if !*primed { + compressor.reset_and_prime(); + *primed = true; + } + compressor.forward_to_internal_pdf(symbol as u32); + } + #[cfg(feature = "backend-mamba")] + RateBackendPredictor::Mamba { + compressor, primed, .. + } => { + if !*primed { + compressor.reset_and_prime(); + *primed = true; + } + let bias = compressor.online_bias_snapshot(); + let logits = compressor.model.forward( + &mut compressor.scratch, + symbol as u32, + &mut compressor.state, + ); + mambazip::Compressor::logits_to_pdf( + logits, + bias.as_deref(), + &mut compressor.pdf_buffer, + ); + } + #[cfg(feature = "backend-zpaq")] + RateBackendPredictor::Zpaq { model } => { + model.update(symbol); + } + #[cfg(feature = "backend-mixture")] + RateBackendPredictor::Mixture { runtime } => { + runtime.update_frozen(symbol); + } + #[cfg(feature = "backend-particle")] + RateBackendPredictor::Particle { runtime } => { + runtime.update_frozen(symbol); + } + #[cfg(feature = "backend-calibrated")] + RateBackendPredictor::Calibrated { + base, + core, + pdf, + valid, + .. + } => { + if !*valid { + let mut base_logps = [0.0; 256]; + base.fill_log_probs(&mut base_logps); + let mut base_pdf = [0.0; 256]; + for (dst, &lp) in base_pdf.iter_mut().zip(base_logps.iter()) { + *dst = clamp_prob(lp.exp(), DEFAULT_MIN_PROB); + } + core.apply_pdf(&base_pdf, pdf); + *valid = true; + } + base.update_frozen(symbol); + core.update_context_only(symbol); + *valid = false; + } + RateBackendPredictor::Disabled { .. } => {} + } + } +} + +#[derive(Clone)] +enum ExpertPredictor { + Generic(Box), + RateBackend(Box), +} + +impl ExpertPredictor { + fn generic(predictor: Box) -> Self { + Self::Generic(predictor) + } + + fn rate_backend(predictor: RateBackendPredictor) -> Self { + Self::RateBackend(Box::new(predictor)) + } + + fn as_mut(&mut self) -> &mut (dyn OnlineBytePredictor + 'static) { + match self { + Self::Generic(predictor) => predictor.as_mut(), + Self::RateBackend(predictor) => predictor.as_mut(), + } + } + + fn into_box(self) -> Box { + match self { + Self::Generic(predictor) => predictor, + Self::RateBackend(predictor) => predictor, + } + } + + fn lifecycle_checkpoint(&mut self, op: OnlineBytePredictorLifecycleOp) -> ExpertLifecycleToken { + match self { + Self::Generic(_) => ExpertLifecycleToken::Full(self.clone()), + Self::RateBackend(predictor) => { + ExpertLifecycleToken::Compact(Box::new(predictor.lifecycle_checkpoint(op))) + } + } + } + + fn restore_lifecycle( + &mut self, + op: OnlineBytePredictorLifecycleOp, + token: ExpertLifecycleToken, + ) { + match (self, token) { + (slot, ExpertLifecycleToken::Full(predictor)) => { + *slot = predictor; + } + (Self::RateBackend(predictor), ExpertLifecycleToken::Compact(checkpoint)) => { + predictor.restore_lifecycle_checkpoint(op, *checkpoint); + } + (Self::Generic(_), ExpertLifecycleToken::Compact(_)) => { + panic!("generic expert received a compact rate-backend lifecycle checkpoint") + } + } + } + + fn discard_lifecycle( + &mut self, + op: OnlineBytePredictorLifecycleOp, + token: ExpertLifecycleToken, + ) { + match (self, token) { + (_, ExpertLifecycleToken::Full(_)) => {} + (Self::RateBackend(predictor), ExpertLifecycleToken::Compact(checkpoint)) => { + predictor.discard_lifecycle_checkpoint(op, *checkpoint); + } + (Self::Generic(_), ExpertLifecycleToken::Compact(_)) => { + panic!("generic expert received a compact rate-backend lifecycle checkpoint") + } + } + } +} + +impl std::ops::Deref for ExpertPredictor { + type Target = dyn OnlineBytePredictor + 'static; + + fn deref(&self) -> &Self::Target { + match self { + Self::Generic(predictor) => predictor.as_ref(), + Self::RateBackend(predictor) => predictor.as_ref(), + } + } +} + +impl std::ops::DerefMut for ExpertPredictor { + fn deref_mut(&mut self) -> &mut Self::Target { + self.as_mut() + } +} + +/// Configuration for a mixture expert. +#[derive(Clone)] +pub struct ExpertConfig { + /// Human-readable expert identifier. + pub name: String, + /// Log prior weight (natural log). Uniform priors can be `0.0`. + pub log_prior: f64, + builder: Arc ExpertPredictor + Send + Sync>, +} + +impl ExpertConfig { + /// Create a new expert config from a builder closure. + pub fn new( + name: impl Into, + log_prior: f64, + builder: impl Fn() -> Box + Send + Sync + 'static, + ) -> Self { + Self::new_with_predictor(name, log_prior, move || ExpertPredictor::generic(builder())) + } + + fn new_with_predictor( + name: impl Into, + log_prior: f64, + builder: impl Fn() -> ExpertPredictor + Send + Sync + 'static, + ) -> Self { + Self { + name: name.into(), + log_prior, + builder: Arc::new(builder), + } + } + + fn new_rate_backend( + name: impl Into, + log_prior: f64, + builder: impl Fn() -> RateBackendPredictor + Send + Sync + 'static, + ) -> Self { + Self::new_with_predictor(name, log_prior, move || { + ExpertPredictor::rate_backend(builder()) + }) + } + + /// Uniform prior helper. + pub fn uniform( + name: impl Into, + builder: impl Fn() -> Box + Send + Sync + 'static, + ) -> Self { + Self::new(name, 0.0, builder) + } + + /// Expert from a `RateBackend` configuration. ROSA's `max_order` lives inside + /// the [`RateBackend::RosaPlus`] variant. + pub fn from_rate_backend(name: Option, log_prior: f64, backend: RateBackend) -> Self { + let name = name.unwrap_or_else(|| RateBackendPredictor::default_name(&backend)); + Self::new_rate_backend(name, log_prior, move || { + RateBackendPredictor::from_backend(backend.clone(), DEFAULT_MIN_PROB) + }) + } + + /// Expert from a compiled rate backend plan. + pub fn from_compiled_rate_backend( + name: Option, + log_prior: f64, + backend: CompiledRateBackend, + ) -> Self { + let name = name.unwrap_or_else(|| backend.display_label()); + Self::new_rate_backend(name, log_prior, move || { + RateBackendPredictor::from_compiled(&backend, DEFAULT_MIN_PROB) + }) + } + + /// ROSA expert (uniform prior) with explicit `max_order`. + pub fn rosa(name: impl Into, max_order: i64) -> Self { + let name = name.into(); + Self::new_rate_backend(name, 0.0, move || { + RateBackendPredictor::from_backend( + RateBackend::RosaPlus { max_order }, + DEFAULT_MIN_PROB, + ) + }) + } + + /// CTW expert (uniform prior). + pub fn ctw(name: impl Into, depth: usize) -> Self { + let name = name.into(); + Self::new_rate_backend(name, 0.0, move || { + RateBackendPredictor::from_backend(RateBackend::Ctw { depth }, DEFAULT_MIN_PROB) + }) + } + + /// FAC-CTW expert (uniform prior). + pub fn fac_ctw(name: impl Into, base_depth: usize, encoding_bits: usize) -> Self { + let name = name.into(); + Self::new_rate_backend(name, 0.0, move || { + RateBackendPredictor::from_backend( + RateBackend::FacCtw { + base_depth, + num_percept_bits: encoding_bits, + encoding_bits, + msb_first: None, + }, + DEFAULT_MIN_PROB, + ) + }) + } + + /// RWKV-7 expert (uniform prior). + #[cfg(feature = "backend-rwkv")] + pub fn rwkv(name: impl Into, method: impl Into) -> Self { + let name = name.into(); + let method = crate::rwkvzip::parse_method_spec(&method.into()) + .expect("rwkv expert method must be a valid RWKV method spec"); + Self::new_rate_backend(name, 0.0, move || { + RateBackendPredictor::from_backend( + RateBackend::Rwkv7Method { + method: method.clone(), + }, + DEFAULT_MIN_PROB, + ) + }) + } + + /// Mamba expert (uniform prior). + #[cfg(feature = "backend-mamba")] + pub fn mamba(name: impl Into, method: impl Into) -> Self { + let name = name.into(); + let method = crate::mambazip::parse_method_spec(&method.into()) + .expect("mamba expert method must be a valid Mamba method spec"); + Self::new_rate_backend(name, 0.0, move || { + RateBackendPredictor::from_backend( + RateBackend::MambaMethod { + method: method.clone(), + }, + DEFAULT_MIN_PROB, + ) + }) + } + + /// ZPAQ expert (uniform prior). + pub fn zpaq(name: impl Into, method: impl Into) -> Self { + let name = name.into(); + let method = crate::api::ZpaqMethodSpec::literal(method.into()); + Self::new_rate_backend(name, 0.0, move || { + RateBackendPredictor::from_backend( + RateBackend::Zpaq { + method: method.clone(), + }, + DEFAULT_MIN_PROB, + ) + }) + } + + /// Expert name. + pub fn name(&self) -> &str { + &self.name + } + + /// Log prior weight (unnormalized). + pub fn log_prior(&self) -> f64 { + self.log_prior + } + + /// Build a fresh predictor instance for evaluation or analysis. + pub fn build_predictor(&self) -> Box { + (self.builder)().into_box() + } + + fn build(&self) -> ExpertState { + ExpertState { + name: self.name.clone(), + log_weight: self.log_prior, + log_prior: self.log_prior, + predictor: (self.builder)(), + cum_log_loss: 0.0, + } + } +} + +#[cfg(feature = "backend-mixture")] +pub(crate) fn expert_configs_from_compiled_mixture( + backend: &CompiledRateBackend, +) -> Result, String> { + let crate::spec::core::RateBackendPlan::Mixture { experts, .. } = backend.plan() else { + return Err("compiled backend is not a mixture backend".to_string()); + }; + experts + .iter() + .map(|expert| { + let compiled = + crate::spec::core::compiled_rate_backend_from_plan(expert.backend.clone()) + .map_err(|err| err.to_string())?; + Ok(ExpertConfig::from_compiled_rate_backend( + expert.name.clone(), + expert.log_prior, + compiled, + )) + }) + .collect::, String>>() +} + +#[cfg(feature = "backend-mixture")] +pub(crate) fn expert_configs_from_compiled_mixture_with_builder( + backend: &CompiledRateBackend, + builder: fn(&CompiledRateBackend, f64) -> Result, + min_prob: f64, +) -> Result, String> { + let crate::spec::core::RateBackendPlan::Mixture { experts, .. } = backend.plan() else { + return Err("compiled backend is not a mixture backend".to_string()); + }; + experts + .iter() + .map(|expert| { + let compiled = + crate::spec::core::compiled_rate_backend_from_plan(expert.backend.clone()) + .map_err(|err| err.to_string())?; + // Validate once up front so mixture construction fails before we + // commit any `ExpertConfig` values. The stored builder still has to + // create a fresh predictor later because each runtime needs its own + // independent expert state. + builder(&compiled, min_prob).map(|_| ())?; + let name = expert + .name + .clone() + .unwrap_or_else(|| compiled.default_name()); + Ok(ExpertConfig::new_rate_backend( + name, + expert.log_prior, + move || { + builder(&compiled, min_prob) + .expect("compiled mixture expert builder should succeed") + }, + )) + }) + .collect::, String>>() +} + +enum ExpertLifecycleToken { + Full(ExpertPredictor), + Compact(Box), +} + +struct ExpertStateLifecycleCheckpoint { + log_weight: f64, + log_prior: f64, + cum_log_loss: f64, + predictor: ExpertLifecycleToken, +} + +#[derive(Clone)] +struct ExpertState { + name: String, + log_weight: f64, + log_prior: f64, + predictor: ExpertPredictor, + cum_log_loss: f64, +} + +impl ExpertState { + #[inline] + fn begin_stream(&mut self, total_symbols: Option) -> Result<(), String> { + self.predictor.begin_stream(total_symbols) + } + + #[inline] + fn finish_stream(&mut self) -> Result<(), String> { + self.predictor.finish_stream() + } + + #[inline] + fn log_prob(&mut self, symbol: u8) -> f64 { + self.predictor.log_prob(symbol) + } + + #[inline] + fn log_prob_update(&mut self, symbol: u8) -> f64 { + self.predictor.log_prob_update(symbol) + } + + #[inline] + fn update(&mut self, symbol: u8) { + self.predictor.update(symbol); + } + + #[inline] + fn reset_frozen(&mut self, total_symbols: Option) -> Result<(), String> { + self.predictor.reset_frozen(total_symbols) + } + + #[inline] + fn begin_fresh_stream(&mut self, total_symbols: Option) -> Result<(), String> { + self.predictor.begin_fresh_stream(total_symbols) + } + + #[inline] + fn update_frozen(&mut self, symbol: u8) { + self.predictor.update_frozen(symbol); + } + + fn snapshot_lifecycle(&mut self, op: ExpertLifecycleOp) -> ExpertLifecycleToken { + self.predictor.lifecycle_checkpoint(op.to_predictor_op()) + } + + fn restore_lifecycle(&mut self, op: ExpertLifecycleOp, token: ExpertLifecycleToken) { + self.predictor + .restore_lifecycle(op.to_predictor_op(), token); + } + + fn discard_lifecycle(&mut self, op: ExpertLifecycleOp, token: ExpertLifecycleToken) { + self.predictor + .discard_lifecycle(op.to_predictor_op(), token); + } + + fn snapshot_state_lifecycle( + &mut self, + op: ExpertLifecycleOp, + ) -> ExpertStateLifecycleCheckpoint { + ExpertStateLifecycleCheckpoint { + log_weight: self.log_weight, + log_prior: self.log_prior, + cum_log_loss: self.cum_log_loss, + predictor: self.snapshot_lifecycle(op), + } + } + + fn restore_state_lifecycle( + &mut self, + op: ExpertLifecycleOp, + checkpoint: ExpertStateLifecycleCheckpoint, + ) { + self.log_weight = checkpoint.log_weight; + self.log_prior = checkpoint.log_prior; + self.cum_log_loss = checkpoint.cum_log_loss; + self.restore_lifecycle(op, checkpoint.predictor); + } + + fn discard_state_lifecycle( + &mut self, + op: ExpertLifecycleOp, + checkpoint: ExpertStateLifecycleCheckpoint, + ) { + self.discard_lifecycle(op, checkpoint.predictor); + } +} + +fn reset_expert_losses(experts: &mut [ExpertState]) { + for expert in experts { + expert.cum_log_loss = 0.0; + } +} + +fn reset_experts_to_priors(experts: &mut [ExpertState]) -> Vec { + let prior = normalized_expert_prior_weights(experts); + set_log_weights_from_linear(experts, &prior); + reset_expert_losses(experts); + prior +} + +fn apply_bayes_update_from_logps( + experts: &mut [ExpertState], + expert_logps: &[f64], + scratch_mix: &mut Vec, +) -> f64 { + let n = experts.len(); + scratch_mix.resize(n, 0.0); + for idx in 0..n { + scratch_mix[idx] = experts[idx].log_weight + expert_logps[idx]; + } + let log_mix = logsumexp(&scratch_mix[..n]); + for idx in 0..n { + experts[idx].cum_log_loss -= expert_logps[idx]; + experts[idx].log_weight += expert_logps[idx] - log_mix; + } + log_mix +} + +fn apply_fading_update_from_logps( + experts: &mut [ExpertState], + expert_logps: &[f64], + scratch_mix: &mut Vec, + decay: f64, +) -> f64 { + let n = experts.len(); + scratch_mix.resize(n, 0.0); + for idx in 0..n { + scratch_mix[idx] = decay * experts[idx].log_weight; + } + let log_prior_norm = logsumexp(&scratch_mix[..n]); + for idx in 0..n { + scratch_mix[idx] += expert_logps[idx]; + } + let log_evidence = logsumexp(&scratch_mix[..n]); + for idx in 0..n { + experts[idx].cum_log_loss -= expert_logps[idx]; + experts[idx].log_weight = + decay * experts[idx].log_weight + expert_logps[idx] - log_evidence; + } + log_evidence - log_prior_norm +} + +// The switching update coordinates expert weights, scratch buffers, schedule +// parameters, and the mutation counter in one hot-path pass; a config wrapper +// would obscure which state is read-only versus updated in place. +#[allow(clippy::too_many_arguments)] +fn apply_switching_update_from_logps( + experts: &mut [ExpertState], + expert_logps: &[f64], + scratch_joint: &mut Vec, + scratch_weights: &mut Vec, + prior: &[f64], + schedule: MixtureScheduleMode, + alpha: f64, + update_count: &mut u64, +) -> f64 { + let n = experts.len(); + scratch_joint.resize(n, 0.0); + scratch_weights.resize(n, 0.0); + // Index form required for coordinated writes to two scratch vecs + experts. + #[allow(clippy::needless_range_loop)] + for idx in 0..n { + experts[idx].cum_log_loss -= expert_logps[idx]; + scratch_joint[idx] = experts[idx].log_weight + expert_logps[idx]; + } + let log_mix = logsumexp(&scratch_joint[..n]); + #[allow(clippy::needless_range_loop)] + for idx in 0..n { + scratch_weights[idx] = (scratch_joint[idx] - log_mix).exp(); + } + + let alpha = switching_alpha_for_update(schedule, alpha, *update_count); + *update_count = (*update_count).saturating_add(1); + if n == 1 || alpha <= 0.0 { + set_log_weights_from_linear(experts, scratch_weights); + return log_mix; + } + + let mut switch_out_sum = 0.0; + let mut num_switch_targets = 0usize; + for &prior_weight in prior { + if prior_weight < 1.0 { + num_switch_targets += 1; + } + } + if num_switch_targets <= 1 { + set_log_weights_from_linear(experts, scratch_weights); + return log_mix; + } + + for idx in 0..n { + let denom = 1.0 - prior[idx]; + if denom > 0.0 { + switch_out_sum += scratch_weights[idx] / denom; + } + } + for idx in 0..n { + let stay = (1.0 - alpha) * scratch_weights[idx]; + let switch_in = if prior[idx] > 0.0 { + let denom = 1.0 - prior[idx]; + let switchable_mass = if denom > 0.0 { + switch_out_sum - scratch_weights[idx] / denom + } else { + 0.0 + }; + alpha * prior[idx] * switchable_mass + } else { + 0.0 + }; + scratch_joint[idx] = stay + switch_in; + } + normalize_simplex_weights(scratch_joint); + set_log_weights_from_linear(experts, scratch_joint); + log_mix +} + +fn mix_log_prob_convex(lambda: &[f64], logps: &[f64]) -> f64 { + let mut mix = 0.0; + for (weight, &logp) in lambda.iter().zip(logps.iter()) { + if *weight > 0.0 { + mix += *weight * logp.exp(); + } + } + clamp_prob(mix, DEFAULT_MIN_PROB).ln() +} + +fn apply_convex_update_from_logps( + experts: &mut [ExpertState], + expert_logps: &[f64], + lambda: &mut [f64], + projection_scratch: &mut Vec, + schedule: MixtureScheduleMode, + alpha: f64, + update_count: &mut u64, +) -> f64 { + let log_mix = mix_log_prob_convex(lambda, expert_logps); + for idx in 0..experts.len() { + experts[idx].cum_log_loss -= expert_logps[idx]; + } + *update_count = (*update_count).saturating_add(1); + let step_size = convex_step_size_for_update(schedule, alpha, *update_count); + for (weight, &logp) in lambda.iter_mut().zip(expert_logps.iter()) { + let grad = -(logp - log_mix).exp(); + *weight -= step_size * grad; + } + project_simplex_with_scratch(lambda, projection_scratch); + log_mix +} + +fn apply_mdl_update_from_logps( + experts: &mut [ExpertState], + expert_logps: &[f64], + best_idx: usize, + last_best: &mut usize, +) -> f64 { + for idx in 0..experts.len() { + experts[idx].cum_log_loss -= expert_logps[idx]; + } + *last_best = best_idx; + expert_logps + .get(best_idx) + .copied() + .unwrap_or(f64::NEG_INFINITY) +} + +fn finish_neural_update_from_logps( + mixture: &mut NeuralMixture, + symbol: u8, + logp: f64, + update_weights: bool, +) { + if update_weights { + mixture + .neural + .update_weights_symbol(&mixture.scratch_expert_logps, mixture.min_prob); + } + mixture.total_log_loss -= logp; + mixture.analyzer.update(symbol); + mixture.neural.set_context_state(mixture.analyzer.state()); + mixture.invalidate_eval_cache(); +} + +/// Exponential-weights Bayes mixture (log-loss Hedge). +#[derive(Clone)] +pub struct BayesMixture { + experts: Vec, + scratch_logps: Vec, + scratch_mix: Vec, + bitwise: MixtureBitPrefixState, + cached_symbol: u8, + cached_log_mix: f64, + cache_valid: bool, + total_log_loss: f64, +} + +impl BayesMixture { + /// Construct a normalized Bayes mixture from expert configs. + pub fn new(configs: &[ExpertConfig]) -> Self { + let mut experts: Vec = configs.iter().map(|c| c.build()).collect(); + let log_priors: Vec = experts.iter().map(|e| e.log_prior).collect(); + let norm = logsumexp(&log_priors); + for e in &mut experts { + e.log_weight -= norm; + } + Self { + experts, + scratch_logps: vec![0.0; configs.len()], + scratch_mix: vec![0.0; configs.len()], + bitwise: MixtureBitPrefixState::default(), + cached_symbol: 0, + cached_log_mix: f64::NEG_INFINITY, + cache_valid: false, + total_log_loss: 0.0, + } + } + + /// Log-probability (natural log) of the mixture for `symbol`, then update. + pub fn step(&mut self, symbol: u8) -> f64 { + if self.experts.is_empty() { + return f64::NEG_INFINITY; + } + if self.cache_valid && self.cached_symbol == symbol { + for expert in &mut self.experts { + expert.update(symbol); + } + } else { + for (i, expert) in self.experts.iter_mut().enumerate() { + self.scratch_logps[i] = expert.log_prob_update(symbol); + } + } + let log_mix = apply_bayes_update_from_logps( + &mut self.experts, + &self.scratch_logps, + &mut self.scratch_mix, + ); + self.cache_valid = false; + self.total_log_loss -= log_mix; + log_mix + } + + fn predict_log_prob(&mut self, symbol: u8) -> f64 { + if self.experts.is_empty() { + return f64::NEG_INFINITY; + } + for (i, expert) in self.experts.iter_mut().enumerate() { + self.scratch_logps[i] = expert.log_prob(symbol); + self.scratch_mix[i] = expert.log_weight + self.scratch_logps[i]; + } + let log_mix = logsumexp(&self.scratch_mix); + self.cached_symbol = symbol; + self.cached_log_mix = log_mix; + self.cache_valid = true; + log_mix + } + + fn fill_log_probs(&mut self, out: &mut [f64; 256]) { + if self.experts.is_empty() { + out.fill(f64::NEG_INFINITY); + return; + } + out.fill(f64::NEG_INFINITY); + let norm = logsumexp_weights(&self.experts); + let mut row = [0.0f64; 256]; + for expert in &mut self.experts { + expert.predictor.fill_log_probs(&mut row); + let lw = expert.log_weight - norm; + for b in 0..256 { + out[b] = logsumexp2(out[b], lw + row[b]); + } + } + } + + /// Posterior weights (normalized) over experts. + pub fn posterior(&self) -> Vec { + let norm = logsumexp_weights(&self.experts); + self.experts + .iter() + .map(|e| (e.log_weight - norm).exp()) + .collect() + } + + /// Index and log-loss (nats) of the current best expert. + pub fn min_expert_log_loss(&self) -> (usize, f64) { + let mut best_idx = 0usize; + let mut best_loss = f64::INFINITY; + for (i, e) in self.experts.iter().enumerate() { + if e.cum_log_loss < best_loss { + best_loss = e.cum_log_loss; + best_idx = i; + } + } + (best_idx, best_loss) + } + + /// Index and posterior mass of the most likely expert. + pub fn max_posterior(&self) -> (usize, f64) { + let norm = logsumexp_weights(&self.experts); + let mut best_idx = 0usize; + let mut best_p = 0.0; + for (i, e) in self.experts.iter().enumerate() { + let p = (e.log_weight - norm).exp(); + if p > best_p { + best_p = p; + best_idx = i; + } + } + (best_idx, best_p) + } + + /// Total log-loss of the mixture so far (nats). + pub fn total_log_loss(&self) -> f64 { + self.total_log_loss + } + + /// Expert cumulative log-losses (nats) and names. + pub fn expert_log_losses(&self) -> Vec<(String, f64)> { + self.experts + .iter() + .map(|e| (e.name.clone(), e.cum_log_loss)) + .collect() + } + + /// Expert names in order. + pub fn expert_names(&self) -> Vec { + self.experts.iter().map(|e| e.name.clone()).collect() + } + + #[inline] + fn clear_stream_state(&mut self) { + self.cache_valid = false; + self.total_log_loss = 0.0; + } + + fn reset_frozen(&mut self, total_symbols: Option) -> Result<(), String> { + reset_expert_frozen_stream(&mut self.experts, total_symbols)?; + self.clear_stream_state(); + Ok(()) + } + + fn begin_fresh_stream(&mut self, total_symbols: Option) -> Result<(), String> { + begin_expert_fresh_stream(&mut self.experts, total_symbols)?; + reset_experts_to_priors(&mut self.experts); + self.clear_stream_state(); + Ok(()) + } + + fn update_frozen(&mut self, symbol: u8) { + for expert in &mut self.experts { + expert.update_frozen(symbol); + } + self.cache_valid = false; + } +} + +/// Exponential-weights Bayes mixture with exponential forgetting on weights. +/// +/// This is a non-stationary control: weights are discounted each step by `decay`. +#[derive(Clone)] +pub struct FadingBayesMixture { + experts: Vec, + decay: f64, + scratch_logps: Vec, + scratch_mix: Vec, + bitwise: MixtureBitPrefixState, + cached_symbol: u8, + cached_log_predictive: f64, + cached_log_evidence: f64, + cache_valid: bool, + total_log_loss: f64, +} + +impl FadingBayesMixture { + /// Construct a fading Bayes mixture with decay in `[0, 1]`. + pub fn new(configs: &[ExpertConfig], decay: f64) -> Self { + let mut experts: Vec = configs.iter().map(|c| c.build()).collect(); + let log_priors: Vec = experts.iter().map(|e| e.log_prior).collect(); + let norm = logsumexp(&log_priors); + for e in &mut experts { + e.log_weight -= norm; + } + let decay = decay.clamp(0.0, 1.0); + Self { + experts, + decay, + scratch_logps: vec![0.0; configs.len()], + scratch_mix: vec![0.0; configs.len()], + bitwise: MixtureBitPrefixState::default(), + cached_symbol: 0, + cached_log_predictive: f64::NEG_INFINITY, + cached_log_evidence: f64::NEG_INFINITY, + cache_valid: false, + total_log_loss: 0.0, + } + } + + /// Log-probability (natural log) of the fading mixture for `symbol`, then update. + pub fn step(&mut self, symbol: u8) -> f64 { + if self.experts.is_empty() { + return f64::NEG_INFINITY; + } + if self.cache_valid && self.cached_symbol == symbol { + for expert in &mut self.experts { + expert.update(symbol); + } + } else { + for (i, expert) in self.experts.iter_mut().enumerate() { + self.scratch_logps[i] = expert.log_prob_update(symbol); + } + } + let log_predictive = apply_fading_update_from_logps( + &mut self.experts, + &self.scratch_logps, + &mut self.scratch_mix, + self.decay, + ); + self.cache_valid = false; + self.total_log_loss -= log_predictive; + log_predictive + } + + fn predict_log_prob(&mut self, symbol: u8) -> f64 { + if self.experts.is_empty() { + return f64::NEG_INFINITY; + } + for (i, expert) in self.experts.iter_mut().enumerate() { + self.scratch_logps[i] = expert.log_prob(symbol); + self.scratch_mix[i] = self.decay * expert.log_weight; + } + let log_prior_norm = logsumexp(&self.scratch_mix); + for i in 0..self.experts.len() { + self.scratch_mix[i] += self.scratch_logps[i]; + } + let log_evidence = logsumexp(&self.scratch_mix); + let log_predictive = log_evidence - log_prior_norm; + self.cached_symbol = symbol; + self.cached_log_predictive = log_predictive; + self.cached_log_evidence = log_evidence; + self.cache_valid = true; + log_predictive + } + + fn fill_log_probs(&mut self, out: &mut [f64; 256]) { + if self.experts.is_empty() { + out.fill(f64::NEG_INFINITY); + return; + } + out.fill(f64::NEG_INFINITY); + let mut decayed = Vec::with_capacity(self.experts.len()); + for expert in &self.experts { + decayed.push(self.decay * expert.log_weight); + } + let norm = logsumexp(&decayed); + let mut row = [0.0f64; 256]; + for (i, expert) in self.experts.iter_mut().enumerate() { + expert.predictor.fill_log_probs(&mut row); + let lw = decayed[i] - norm; + for b in 0..256 { + out[b] = logsumexp2(out[b], lw + row[b]); + } + } + } + + /// Posterior weights (normalized) over experts. + pub fn posterior(&self) -> Vec { + let norm = logsumexp_weights(&self.experts); + self.experts + .iter() + .map(|e| (e.log_weight - norm).exp()) + .collect() + } + + /// Index and log-loss (nats) of the current best expert (non-discounted loss). + pub fn min_expert_log_loss(&self) -> (usize, f64) { + let mut best_idx = 0usize; + let mut best_loss = f64::INFINITY; + for (i, e) in self.experts.iter().enumerate() { + if e.cum_log_loss < best_loss { + best_loss = e.cum_log_loss; + best_idx = i; + } + } + (best_idx, best_loss) + } + + /// Total log-loss of the mixture so far (nats). + pub fn total_log_loss(&self) -> f64 { + self.total_log_loss + } + + /// Expert names in order. + pub fn expert_names(&self) -> Vec { + self.experts.iter().map(|e| e.name.clone()).collect() + } + + #[inline] + fn clear_stream_state(&mut self) { + self.cache_valid = false; + self.total_log_loss = 0.0; + } + + fn reset_frozen(&mut self, total_symbols: Option) -> Result<(), String> { + reset_expert_frozen_stream(&mut self.experts, total_symbols)?; + self.clear_stream_state(); + Ok(()) + } + + fn begin_fresh_stream(&mut self, total_symbols: Option) -> Result<(), String> { + begin_expert_fresh_stream(&mut self.experts, total_symbols)?; + reset_experts_to_priors(&mut self.experts); + self.clear_stream_state(); + Ok(()) + } + + fn update_frozen(&mut self, symbol: u8) { + for expert in &mut self.experts { + expert.update_frozen(symbol); + } + self.cache_valid = false; + } +} + +/// Switching mixture: allows occasional switches between experts. +#[derive(Clone)] +pub struct SwitchingMixture { + experts: Vec, + prior: Vec, + alpha: f64, + schedule: MixtureScheduleMode, + scratch_logps: Vec, + scratch_joint: Vec, + scratch_weights: Vec, + bitwise: MixtureBitPrefixState, + cached_symbol: u8, + cached_log_mix: f64, + cache_valid: bool, + total_log_loss: f64, + update_count: u64, +} + +impl SwitchingMixture { + /// Construct a switching mixture. + pub fn new(configs: &[ExpertConfig], alpha: f64, schedule: MixtureScheduleMode) -> Self { + let mut experts: Vec = configs.iter().map(|c| c.build()).collect(); + let prior = normalized_prior_weights(configs); + set_log_weights_from_linear(&mut experts, &prior); + Self { + experts, + prior, + alpha, + schedule, + scratch_logps: vec![0.0; configs.len()], + scratch_joint: vec![0.0; configs.len()], + scratch_weights: vec![0.0; configs.len()], + bitwise: MixtureBitPrefixState::default(), + cached_symbol: 0, + cached_log_mix: f64::NEG_INFINITY, + cache_valid: false, + total_log_loss: 0.0, + update_count: 0, + } + } + + /// Log-probability (natural log) of the switching mixture for `symbol`, then update. + pub fn step(&mut self, symbol: u8) -> f64 { + if self.experts.is_empty() { + return f64::NEG_INFINITY; + } + if self.cache_valid && self.cached_symbol == symbol { + for expert in &mut self.experts { + expert.update(symbol); + } + } else { + for (i, expert) in self.experts.iter_mut().enumerate() { + self.scratch_logps[i] = expert.log_prob_update(symbol); + } + } + let log_mix = apply_switching_update_from_logps( + &mut self.experts, + &self.scratch_logps, + &mut self.scratch_joint, + &mut self.scratch_weights, + &self.prior, + self.schedule, + self.alpha, + &mut self.update_count, + ); + self.cache_valid = false; + self.total_log_loss -= log_mix; + log_mix + } + + fn predict_log_prob(&mut self, symbol: u8) -> f64 { + if self.experts.is_empty() { + return f64::NEG_INFINITY; + } + for i in 0..self.experts.len() { + let lp = self.experts[i].log_prob(symbol); + self.scratch_logps[i] = lp; + self.scratch_joint[i] = self.experts[i].log_weight + lp; + } + let log_mix = logsumexp(&self.scratch_joint); + self.cached_symbol = symbol; + self.cached_log_mix = log_mix; + self.cache_valid = true; + log_mix + } + + fn fill_log_probs(&mut self, out: &mut [f64; 256]) { + if self.experts.is_empty() { + out.fill(f64::NEG_INFINITY); + return; + } + out.fill(f64::NEG_INFINITY); + let norm = logsumexp_weights(&self.experts); + let mut row = [0.0f64; 256]; + for expert in &mut self.experts { + expert.predictor.fill_log_probs(&mut row); + let lw = expert.log_weight - norm; + for b in 0..256 { + out[b] = logsumexp2(out[b], lw + row[b]); + } + } + } + + /// Posterior weights (normalized) over experts. + pub fn posterior(&self) -> Vec { + let norm = logsumexp_weights(&self.experts); + self.experts + .iter() + .map(|e| (e.log_weight - norm).exp()) + .collect() + } + + /// Index and log-loss (nats) of the current best expert. + pub fn min_expert_log_loss(&self) -> (usize, f64) { + let mut best_idx = 0usize; + let mut best_loss = f64::INFINITY; + for (i, e) in self.experts.iter().enumerate() { + if e.cum_log_loss < best_loss { + best_loss = e.cum_log_loss; + best_idx = i; + } + } + (best_idx, best_loss) + } + + /// Index and posterior mass of the most likely expert. + pub fn max_posterior(&self) -> (usize, f64) { + let norm = logsumexp_weights(&self.experts); + let mut best_idx = 0usize; + let mut best_p = 0.0; + for (i, e) in self.experts.iter().enumerate() { + let p = (e.log_weight - norm).exp(); + if p > best_p { + best_p = p; + best_idx = i; + } + } + (best_idx, best_p) + } + + /// Total log-loss of the mixture so far (nats). + pub fn total_log_loss(&self) -> f64 { + self.total_log_loss + } + + /// Expert cumulative log-losses (nats) and names. + pub fn expert_log_losses(&self) -> Vec<(String, f64)> { + self.experts + .iter() + .map(|e| (e.name.clone(), e.cum_log_loss)) + .collect() + } + + /// Expert names in order. + pub fn expert_names(&self) -> Vec { + self.experts.iter().map(|e| e.name.clone()).collect() + } + + #[inline] + fn clear_stream_state(&mut self) { + self.cache_valid = false; + self.total_log_loss = 0.0; + self.update_count = 0; + } + + fn reset_frozen(&mut self, total_symbols: Option) -> Result<(), String> { + reset_expert_frozen_stream(&mut self.experts, total_symbols)?; + self.clear_stream_state(); + Ok(()) + } + + fn begin_fresh_stream(&mut self, total_symbols: Option) -> Result<(), String> { + begin_expert_fresh_stream(&mut self.experts, total_symbols)?; + set_log_weights_from_linear(&mut self.experts, &self.prior); + reset_expert_losses(&mut self.experts); + self.clear_stream_state(); + Ok(()) + } + + fn update_frozen(&mut self, symbol: u8) { + for expert in &mut self.experts { + expert.update_frozen(symbol); + } + self.cache_valid = false; + } +} + +/// Convex mixture with projected-simplex online updates. +#[derive(Clone)] +pub struct ConvexMixture { + experts: Vec, + alpha: f64, + schedule: MixtureScheduleMode, + lambda: Vec, + scratch_logps: Vec, + projection_scratch: Vec, + bitwise: MixtureBitPrefixState, + cached_symbol: u8, + cached_log_mix: f64, + cache_valid: bool, + total_log_loss: f64, + update_count: u64, +} + +impl ConvexMixture { + /// Construct a convex mixture with prior-derived initial weights. + pub fn new(configs: &[ExpertConfig], alpha: f64, schedule: MixtureScheduleMode) -> Self { + Self { + experts: configs.iter().map(|c| c.build()).collect(), + alpha, + schedule, + lambda: normalized_prior_weights(configs), + scratch_logps: vec![0.0; configs.len()], + projection_scratch: Vec::with_capacity(configs.len()), + bitwise: MixtureBitPrefixState::default(), + cached_symbol: 0, + cached_log_mix: f64::NEG_INFINITY, + cache_valid: false, + total_log_loss: 0.0, + update_count: 0, + } + } + + fn mix_log_prob(&self, logps: &[f64]) -> f64 { + mix_log_prob_convex(&self.lambda, logps) + } + + /// Log-probability (natural log) of the convex mixture for `symbol`, then update. + pub fn step(&mut self, symbol: u8) -> f64 { + if self.experts.is_empty() { + return f64::NEG_INFINITY; + } + + if self.cache_valid && self.cached_symbol == symbol { + for expert in &mut self.experts { + expert.update(symbol); + } + } else { + for (i, expert) in self.experts.iter_mut().enumerate() { + self.scratch_logps[i] = expert.log_prob_update(symbol); + } + } + let log_mix = apply_convex_update_from_logps( + &mut self.experts, + &self.scratch_logps, + &mut self.lambda, + &mut self.projection_scratch, + self.schedule, + self.alpha, + &mut self.update_count, + ); + self.cache_valid = false; + self.total_log_loss -= log_mix; + log_mix + } + + fn predict_log_prob(&mut self, symbol: u8) -> f64 { + if self.experts.is_empty() { + return f64::NEG_INFINITY; + } + for (i, expert) in self.experts.iter_mut().enumerate() { + self.scratch_logps[i] = expert.log_prob(symbol); + } + let log_mix = self.mix_log_prob(&self.scratch_logps); + self.cached_symbol = symbol; + self.cached_log_mix = log_mix; + self.cache_valid = true; + log_mix + } + + fn fill_log_probs(&mut self, out: &mut [f64; 256]) { + if self.experts.is_empty() { + out.fill(f64::NEG_INFINITY); + return; + } + out.fill(f64::NEG_INFINITY); + let mut row = [0.0f64; 256]; + for (index, expert) in self.experts.iter_mut().enumerate() { + expert.predictor.fill_log_probs(&mut row); + let weight = self.lambda.get(index).copied().unwrap_or(0.0); + if weight <= 0.0 { + continue; + } + let log_weight = weight.ln(); + for byte in 0..256 { + out[byte] = logsumexp2(out[byte], log_weight + row[byte]); + } + } + } + + #[inline] + fn clear_stream_state(&mut self) { + self.cache_valid = false; + self.total_log_loss = 0.0; + self.update_count = 0; + } + + fn reset_frozen(&mut self, total_symbols: Option) -> Result<(), String> { + reset_expert_frozen_stream(&mut self.experts, total_symbols)?; + self.clear_stream_state(); + Ok(()) + } + + fn begin_fresh_stream(&mut self, total_symbols: Option) -> Result<(), String> { + begin_expert_fresh_stream(&mut self.experts, total_symbols)?; + self.lambda = reset_experts_to_priors(&mut self.experts); + self.clear_stream_state(); + Ok(()) + } + + fn update_frozen(&mut self, symbol: u8) { + for expert in &mut self.experts { + expert.update_frozen(symbol); + } + self.cache_valid = false; + } +} + +/// MDL-style selector: predicts with the current best expert (by cumulative loss). +#[derive(Clone)] +pub struct MdlSelector { + experts: Vec, + scratch_logps: Vec, + bitwise: MixtureBitPrefixState, + total_log_loss: f64, + last_best: usize, + cached_symbol: u8, + cached_best_idx: usize, + cached_best_logp: f64, + cache_valid: bool, +} + +/// Bytewise neural mixer (Loosely PAQ inspired) +/// +/// This model is a context-conditioned two-stage gating network trained online +/// from per-symbol expert likelihoods: +/// 1) context-local first-stage expert gates, +/// 2) context-local second-stage meta-gate over stage-1 outputs, +/// 3) per-symbol SGD updates with optional tiny-error skip. +#[derive(Clone)] +pub struct NeuralMixture { + experts: Vec, + neural: NeuralMixCore, + analyzer: TextContextAnalyzer, + min_prob: f64, + scratch_expert_logps: Vec, + scratch_mix_weights: Vec, + bitwise: MixtureBitPrefixState, + eval_cache_valid: bool, + eval_cache_full_valid: bool, + eval_cache_history: NeuralHistoryState, + eval_cache_symbol: u8, + eval_cache_logp: f64, + eval_cache_mix_logps: [f64; 256], + eval_cache_expert_logps: Vec<[f64; 256]>, + total_log_loss: f64, +} + +impl NeuralMixture { + /// Construct a neural mixture. `learning_rate` is taken from `MixtureSpec.alpha`. + pub fn new(configs: &[ExpertConfig], learning_rate: f64) -> Self { + let mut experts: Vec = configs.iter().map(|c| c.build()).collect(); + let n = experts.len(); + + let mut prior_weights = vec![0.0; n]; + if n > 0 { + let log_priors: Vec = experts.iter().map(|e| e.log_prior).collect(); + let norm = logsumexp(&log_priors); + for (i, e) in experts.iter_mut().enumerate() { + let p = (e.log_prior - norm).exp(); + prior_weights[i] = p; + } + } + + let base_lr = if learning_rate.is_finite() { + learning_rate.abs().clamp(1e-6, 1.0) + } else { + 0.03 + }; + let effective_lr = (base_lr * 25.0).clamp(1e-6, 1.0); + let analyzer = TextContextAnalyzer::new(); + let mut neural = + NeuralMixCore::new(n, &prior_weights, effective_lr * 0.5, effective_lr, 1e-5); + neural.set_context_state(analyzer.state()); + let eval_cache_history = neural.history_state(); + + Self { + experts, + neural, + analyzer, + min_prob: DEFAULT_MIN_PROB, + scratch_expert_logps: vec![0.0; n], + scratch_mix_weights: vec![0.0; n], + bitwise: MixtureBitPrefixState::default(), + eval_cache_valid: false, + eval_cache_full_valid: false, + eval_cache_history, + eval_cache_symbol: 0, + eval_cache_logp: f64::NEG_INFINITY, + eval_cache_mix_logps: [f64::NEG_INFINITY; 256], + eval_cache_expert_logps: vec![[f64::NEG_INFINITY; 256]; n], + total_log_loss: 0.0, + } + } + + #[inline] + fn invalidate_eval_cache(&mut self) { + self.eval_cache_valid = false; + self.eval_cache_full_valid = false; + } + + fn sync_history_state(&mut self) -> NeuralHistoryState { + let history = self.analyzer.state(); + if self.neural.history_state() != history { + self.neural.set_context_state(history); + } + if self.eval_cache_history != history { + self.invalidate_eval_cache(); + self.eval_cache_history = history; + } + history + } + + fn ensure_full_evaluation(&mut self) { + self.sync_history_state(); + if self.eval_cache_full_valid { + return; + } + + self.neural.evaluate_expert_weights(); + self.scratch_mix_weights + .copy_from_slice(self.neural.expert_weights()); + let mut mix_pdf = [0.0f64; 256]; + for i in 0..self.experts.len() { + let row = &mut self.eval_cache_expert_logps[i]; + self.experts[i].predictor.fill_log_probs(row); + let w = self.scratch_mix_weights[i]; + for (dst, &lp) in mix_pdf.iter_mut().zip(row.iter()) { + *dst += w * clamp_prob(lp.exp(), self.min_prob); + } + } + + let sum: f64 = mix_pdf.iter().sum(); + if !sum.is_finite() || sum <= 0.0 { + let uniform = (1.0f64 / 256.0).ln(); + self.eval_cache_mix_logps.fill(uniform); + } else { + let inv = 1.0 / sum; + for (dst, &p_raw) in self.eval_cache_mix_logps.iter_mut().zip(mix_pdf.iter()) { + let p = clamp_unit_prob(p_raw * inv, self.min_prob); + *dst = p.ln(); + } + } + + self.eval_cache_full_valid = true; + } + + fn evaluate_symbol(&mut self, symbol: u8) -> f64 { + let history = self.sync_history_state(); + if self.eval_cache_valid + && self.eval_cache_history == history + && self.eval_cache_symbol == symbol + { + return self.eval_cache_logp; + } + + if self.eval_cache_full_valid && self.eval_cache_history == history { + for (dst, row) in self + .scratch_expert_logps + .iter_mut() + .zip(self.eval_cache_expert_logps.iter()) + { + *dst = row[symbol as usize]; + } + let logp = self.eval_cache_mix_logps[symbol as usize]; + self.eval_cache_valid = true; + self.eval_cache_symbol = symbol; + self.eval_cache_logp = logp; + return logp; + } + + let expert_count = self.experts.len(); + for i in 0..expert_count { + self.scratch_expert_logps[i] = self.experts[i].log_prob(symbol); + } + let p = self + .neural + .evaluate_symbol(&self.scratch_expert_logps, self.min_prob); + let logp = clamp_unit_prob(p, self.min_prob).ln(); + self.eval_cache_valid = true; + self.eval_cache_history = history; + self.eval_cache_symbol = symbol; + self.eval_cache_logp = logp; + logp + } + + fn predict_log_prob(&mut self, symbol: u8) -> f64 { + if self.experts.is_empty() { + return f64::NEG_INFINITY; + } + if self.experts.len() == 1 { + return self.experts[0].log_prob(symbol); + } + self.evaluate_symbol(symbol) + } + + fn fill_log_probs(&mut self, out: &mut [f64; 256]) { + if self.experts.is_empty() { + out.fill(f64::NEG_INFINITY); + return; + } + if self.experts.len() == 1 { + self.experts[0].predictor.fill_log_probs(out); + return; + } + self.ensure_full_evaluation(); + out.copy_from_slice(&self.eval_cache_mix_logps); + } + + /// Log-probability (natural log) of the neural mixture for `symbol`, then update. + pub fn step(&mut self, symbol: u8) -> f64 { + if self.experts.is_empty() { + return f64::NEG_INFINITY; + } + + if self.experts.len() == 1 { + let expert = &mut self.experts[0]; + let logp = expert.log_prob_update(symbol); + expert.cum_log_loss -= logp; + self.total_log_loss -= logp; + self.analyzer.update(symbol); + self.neural.set_context_state(self.analyzer.state()); + self.invalidate_eval_cache(); + return logp; + } + + let history = self.sync_history_state(); + let logp = if self.eval_cache_valid + && self.eval_cache_history == history + && self.eval_cache_symbol == symbol + { + let logp = self.eval_cache_logp; + for i in 0..self.experts.len() { + let expert = &mut self.experts[i]; + expert.cum_log_loss -= self.scratch_expert_logps[i]; + expert.update(symbol); + } + logp + } else if self.eval_cache_full_valid && self.eval_cache_history == history { + for i in 0..self.experts.len() { + self.scratch_expert_logps[i] = self.eval_cache_expert_logps[i][symbol as usize]; + } + let logp = self.eval_cache_mix_logps[symbol as usize]; + for i in 0..self.experts.len() { + let expert = &mut self.experts[i]; + expert.cum_log_loss -= self.scratch_expert_logps[i]; + expert.update(symbol); + } + logp + } else { + for i in 0..self.experts.len() { + let expert = &mut self.experts[i]; + self.scratch_expert_logps[i] = expert.log_prob_update(symbol); + expert.cum_log_loss -= self.scratch_expert_logps[i]; + } + let p = self + .neural + .evaluate_symbol(&self.scratch_expert_logps, self.min_prob); + clamp_unit_prob(p, self.min_prob).ln() + }; + finish_neural_update_from_logps(self, symbol, logp, true); + logp + } + + /// Total log-loss of the mixture so far (nats). + pub fn total_log_loss(&self) -> f64 { + self.total_log_loss + } + + #[inline] + fn clear_stream_state(&mut self) { + self.analyzer = TextContextAnalyzer::new(); + self.neural.set_context_state(self.analyzer.state()); + self.invalidate_eval_cache(); + self.eval_cache_history = self.neural.history_state(); + self.total_log_loss = 0.0; + } + + fn reset_frozen(&mut self, total_symbols: Option) -> Result<(), String> { + reset_expert_frozen_stream(&mut self.experts, total_symbols)?; + self.clear_stream_state(); + Ok(()) + } + + fn begin_fresh_stream(&mut self, total_symbols: Option) -> Result<(), String> { + begin_expert_fresh_stream(&mut self.experts, total_symbols)?; + let prior = reset_experts_to_priors(&mut self.experts); + self.neural.reset_to_priors(&prior); + self.clear_stream_state(); + Ok(()) + } + + fn update_frozen(&mut self, symbol: u8) { + for expert in &mut self.experts { + expert.update_frozen(symbol); + } + self.analyzer.update(symbol); + self.neural.set_context_state(self.analyzer.state()); + self.invalidate_eval_cache(); + self.eval_cache_history = self.neural.history_state(); + } +} + +impl MdlSelector { + /// Construct an MDL-style expert selector. + pub fn new(configs: &[ExpertConfig]) -> Self { + let experts: Vec = configs.iter().map(|c| c.build()).collect(); + let last_best = 0usize; + Self { + experts, + scratch_logps: vec![0.0; configs.len()], + bitwise: MixtureBitPrefixState::default(), + total_log_loss: 0.0, + last_best, + cached_symbol: 0, + cached_best_idx: 0, + cached_best_logp: f64::NEG_INFINITY, + cache_valid: false, + } + } + + /// Log-probability (natural log) of the MDL selector for `symbol`, then update. + pub fn step(&mut self, symbol: u8) -> f64 { + if self.experts.is_empty() { + return f64::NEG_INFINITY; + } + let used_cache = self.cache_valid && self.cached_symbol == symbol; + let best_idx = if used_cache { + self.scratch_logps[self.cached_best_idx] = self.cached_best_logp; + for (i, expert) in self.experts.iter_mut().enumerate() { + if i == self.cached_best_idx { + continue; + } + self.scratch_logps[i] = expert.log_prob(symbol); + } + self.cached_best_idx + } else { + for (i, expert) in self.experts.iter_mut().enumerate() { + self.scratch_logps[i] = expert.log_prob_update(symbol); + } + let mut best_idx = 0usize; + let mut best_loss = f64::INFINITY; + for (i, expert) in self.experts.iter().enumerate() { + if expert.cum_log_loss < best_loss { + best_loss = expert.cum_log_loss; + best_idx = i; + } + } + best_idx + }; + self.cache_valid = false; + for expert in &mut self.experts { + if used_cache { + expert.update(symbol); + } + } + let logp = apply_mdl_update_from_logps( + &mut self.experts, + &self.scratch_logps, + best_idx, + &mut self.last_best, + ); + self.total_log_loss -= logp; + logp + } + + fn predict_log_prob(&mut self, symbol: u8) -> f64 { + if self.experts.is_empty() { + return f64::NEG_INFINITY; + } + let mut best_idx = 0usize; + let mut best_loss = f64::INFINITY; + for (i, expert) in self.experts.iter().enumerate() { + if expert.cum_log_loss < best_loss { + best_loss = expert.cum_log_loss; + best_idx = i; + } + } + let logp = self.experts[best_idx].log_prob(symbol); + self.cached_symbol = symbol; + self.cached_best_idx = best_idx; + self.cached_best_logp = logp; + self.cache_valid = true; + logp + } + + fn fill_log_probs(&mut self, out: &mut [f64; 256]) { + if self.experts.is_empty() { + out.fill(f64::NEG_INFINITY); + return; + } + let mut best_idx = 0usize; + let mut best_loss = f64::INFINITY; + for (i, expert) in self.experts.iter().enumerate() { + if expert.cum_log_loss < best_loss { + best_loss = expert.cum_log_loss; + best_idx = i; + } + } + self.experts[best_idx].predictor.fill_log_probs(out); + } + + /// Index of the current best expert. + pub fn best_index(&self) -> usize { + self.last_best + } + + /// Index and log-loss (nats) of the current best expert. + pub fn min_expert_log_loss(&self) -> (usize, f64) { + let mut best_idx = 0usize; + let mut best_loss = f64::INFINITY; + for (i, e) in self.experts.iter().enumerate() { + if e.cum_log_loss < best_loss { + best_loss = e.cum_log_loss; + best_idx = i; + } + } + (best_idx, best_loss) + } + + /// Total log-loss of the selector so far (nats). + pub fn total_log_loss(&self) -> f64 { + self.total_log_loss + } + + /// Expert cumulative log-losses (nats) and names. + pub fn expert_log_losses(&self) -> Vec<(String, f64)> { + self.experts + .iter() + .map(|e| (e.name.clone(), e.cum_log_loss)) + .collect() + } + + /// Expert names in order. + pub fn expert_names(&self) -> Vec { + self.experts.iter().map(|e| e.name.clone()).collect() + } + + #[inline] + fn clear_stream_state(&mut self) { + self.cache_valid = false; + self.total_log_loss = 0.0; + } + + fn reset_frozen(&mut self, total_symbols: Option) -> Result<(), String> { + reset_expert_frozen_stream(&mut self.experts, total_symbols)?; + self.clear_stream_state(); + Ok(()) + } + + fn begin_fresh_stream(&mut self, total_symbols: Option) -> Result<(), String> { + begin_expert_fresh_stream(&mut self.experts, total_symbols)?; + reset_experts_to_priors(&mut self.experts); + self.last_best = 0; + self.clear_stream_state(); + Ok(()) + } + + fn update_frozen(&mut self, symbol: u8) { + for expert in &mut self.experts { + expert.update_frozen(symbol); + } + self.cache_valid = false; + } +} + +#[derive(Clone)] +struct ExpertStateCheckpoint { + log_weight: f64, + log_prior: f64, + cum_log_loss: f64, + predictor: OnlineBytePredictorCheckpoint, +} + +fn checkpoint_experts(experts: &mut [ExpertState]) -> Option> { + experts + .iter_mut() + .map(|expert| { + expert + .predictor + .checkpoint_if_supported() + .map(|predictor| ExpertStateCheckpoint { + log_weight: expert.log_weight, + log_prior: expert.log_prior, + cum_log_loss: expert.cum_log_loss, + predictor, + }) + }) + .collect() +} + +fn restore_experts(experts: &mut [ExpertState], checkpoints: &[ExpertStateCheckpoint]) { + assert_eq!( + experts.len(), + checkpoints.len(), + "mixture checkpoint expert count mismatch" + ); + for (expert, checkpoint) in experts.iter_mut().zip(checkpoints.iter()) { + expert.log_weight = checkpoint.log_weight; + expert.log_prior = checkpoint.log_prior; + expert.cum_log_loss = checkpoint.cum_log_loss; + assert!( + expert + .predictor + .restore_checkpoint_if_supported(&checkpoint.predictor), + "mixture expert rejected its structural checkpoint" + ); + } +} + +fn clear_expert_checkpoints(experts: &mut [ExpertState]) { + for expert in experts { + expert.predictor.clear_checkpoints_if_supported(); + } +} + +fn discard_expert_checkpoints( + experts: &mut [ExpertState], + checkpoints: Vec, +) { + assert_eq!( + experts.len(), + checkpoints.len(), + "mixture checkpoint expert count mismatch" + ); + for (expert, checkpoint) in experts.iter_mut().zip(checkpoints.into_iter()) { + assert!( + expert + .predictor + .discard_checkpoint_if_supported(checkpoint.predictor), + "mixture expert rejected checkpoint discard" + ); + } +} + +fn lifecycle_checkpoint_experts( + experts: &mut [ExpertState], + op: ExpertLifecycleOp, +) -> Vec { + experts + .iter_mut() + .map(|expert| expert.snapshot_state_lifecycle(op)) + .collect() +} + +fn restore_lifecycle_experts( + experts: &mut [ExpertState], + checkpoints: Vec, + op: ExpertLifecycleOp, +) { + assert_eq!( + experts.len(), + checkpoints.len(), + "mixture lifecycle checkpoint expert count mismatch" + ); + for (expert, checkpoint) in experts.iter_mut().zip(checkpoints.into_iter()) { + expert.restore_state_lifecycle(op, checkpoint); + } +} + +fn discard_lifecycle_experts( + experts: &mut [ExpertState], + checkpoints: Vec, + op: ExpertLifecycleOp, +) { + assert_eq!( + experts.len(), + checkpoints.len(), + "mixture lifecycle checkpoint expert count mismatch" + ); + for (expert, checkpoint) in experts.iter_mut().zip(checkpoints.into_iter()) { + expert.discard_state_lifecycle(op, checkpoint); + } +} + +#[derive(Clone)] +#[doc(hidden)] +pub struct BayesMixtureCheckpoint { + experts: Vec, + scratch_logps: Vec, + scratch_mix: Vec, + bitwise: MixtureBitPrefixState, + cached_symbol: u8, + cached_log_mix: f64, + cache_valid: bool, + total_log_loss: f64, +} + +#[derive(Clone)] +#[doc(hidden)] +pub struct FadingBayesMixtureCheckpoint { + experts: Vec, + scratch_logps: Vec, + scratch_mix: Vec, + bitwise: MixtureBitPrefixState, + cached_symbol: u8, + cached_log_predictive: f64, + cached_log_evidence: f64, + cache_valid: bool, + total_log_loss: f64, +} + +#[derive(Clone)] +#[doc(hidden)] +pub struct SwitchingMixtureCheckpoint { + experts: Vec, + scratch_logps: Vec, + scratch_joint: Vec, + scratch_weights: Vec, + bitwise: MixtureBitPrefixState, + cached_symbol: u8, + cached_log_mix: f64, + cache_valid: bool, + total_log_loss: f64, + update_count: u64, +} + +#[derive(Clone)] +#[doc(hidden)] +pub struct ConvexMixtureCheckpoint { + experts: Vec, + lambda: Vec, + scratch_logps: Vec, + projection_scratch: Vec, + bitwise: MixtureBitPrefixState, + cached_symbol: u8, + cached_log_mix: f64, + cache_valid: bool, + total_log_loss: f64, + update_count: u64, +} + +#[derive(Clone)] +#[doc(hidden)] +pub struct MdlSelectorCheckpoint { + experts: Vec, + scratch_logps: Vec, + bitwise: MixtureBitPrefixState, + total_log_loss: f64, + last_best: usize, + cached_symbol: u8, + cached_best_idx: usize, + cached_best_logp: f64, + cache_valid: bool, +} + +#[derive(Clone)] +#[doc(hidden)] +pub struct NeuralMixtureCheckpoint { + experts: Vec, + neural: NeuralMixCore, + analyzer: TextContextAnalyzer, + scratch_expert_logps: Vec, + scratch_mix_weights: Vec, + bitwise: MixtureBitPrefixState, + eval_cache_valid: bool, + eval_cache_full_valid: bool, + eval_cache_history: NeuralHistoryState, + eval_cache_symbol: u8, + eval_cache_logp: f64, + eval_cache_mix_logps: [f64; 256], + eval_cache_expert_logps: Vec<[f64; 256]>, + total_log_loss: f64, +} + +#[derive(Clone)] +#[doc(hidden)] +// Mirrors `MixtureRuntime`: the neural checkpoint owns inline 256-way +// probability caches, and boxing it would add allocation to normal checkpoint +// capture/restore without reducing resident runtime state. +#[allow(clippy::large_enum_variant)] +pub enum MixtureRuntimeCheckpoint { + Bayes(BayesMixtureCheckpoint), + Fading(FadingBayesMixtureCheckpoint), + Switching(SwitchingMixtureCheckpoint), + Convex(ConvexMixtureCheckpoint), + Mdl(MdlSelectorCheckpoint), + Neural(NeuralMixtureCheckpoint), +} + +struct BayesMixtureLifecycleCheckpoint { + experts: Vec, + scratch_logps: Vec, + scratch_mix: Vec, + bitwise: MixtureBitPrefixState, + cached_symbol: u8, + cached_log_mix: f64, + cache_valid: bool, + total_log_loss: f64, +} + +struct FadingBayesMixtureLifecycleCheckpoint { + experts: Vec, + scratch_logps: Vec, + scratch_mix: Vec, + bitwise: MixtureBitPrefixState, + cached_symbol: u8, + cached_log_predictive: f64, + cached_log_evidence: f64, + cache_valid: bool, + total_log_loss: f64, +} + +struct SwitchingMixtureLifecycleCheckpoint { + experts: Vec, + scratch_logps: Vec, + scratch_joint: Vec, + scratch_weights: Vec, + bitwise: MixtureBitPrefixState, + cached_symbol: u8, + cached_log_mix: f64, + cache_valid: bool, + total_log_loss: f64, + update_count: u64, +} + +struct ConvexMixtureLifecycleCheckpoint { + experts: Vec, + lambda: Vec, + scratch_logps: Vec, + projection_scratch: Vec, + bitwise: MixtureBitPrefixState, + cached_symbol: u8, + cached_log_mix: f64, + cache_valid: bool, + total_log_loss: f64, + update_count: u64, +} + +struct MdlSelectorLifecycleCheckpoint { + experts: Vec, + scratch_logps: Vec, + bitwise: MixtureBitPrefixState, + total_log_loss: f64, + last_best: usize, + cached_symbol: u8, + cached_best_idx: usize, + cached_best_logp: f64, + cache_valid: bool, +} + +struct NeuralMixtureLifecycleCheckpoint { + experts: Vec, + neural: NeuralMixCore, + analyzer: TextContextAnalyzer, + scratch_expert_logps: Vec, + scratch_mix_weights: Vec, + bitwise: MixtureBitPrefixState, + eval_cache_valid: bool, + eval_cache_full_valid: bool, + eval_cache_history: NeuralHistoryState, + eval_cache_symbol: u8, + eval_cache_logp: f64, + eval_cache_mix_logps: [f64; 256], + eval_cache_expert_logps: Vec<[f64; 256]>, + total_log_loss: f64, +} + +#[allow(clippy::large_enum_variant)] +enum MixtureRuntimeLifecycleCheckpoint { + Bayes(BayesMixtureLifecycleCheckpoint), + Fading(FadingBayesMixtureLifecycleCheckpoint), + Switching(SwitchingMixtureLifecycleCheckpoint), + Convex(ConvexMixtureLifecycleCheckpoint), + Mdl(MdlSelectorLifecycleCheckpoint), + Neural(NeuralMixtureLifecycleCheckpoint), +} + +// ============================================================================= +// Mixture Runtime Helper (for RateBackend::Mixture) +// ============================================================================= + +/// Runtime wrapper over concrete mixture strategies. +#[allow(clippy::large_enum_variant)] +#[derive(Clone)] +pub enum MixtureRuntime { + /// Bayes mixture. + Bayes(BayesMixture), + /// Fading Bayes mixture. + Fading(FadingBayesMixture), + /// Switching mixture. + Switching(SwitchingMixture), + /// Convex mixture. + Convex(ConvexMixture), + /// MDL selector. + Mdl(MdlSelector), + /// Bytewise neural logistic mixer. + Neural(NeuralMixture), +} + +impl MixtureRuntime { + pub(crate) fn checkpoint(&mut self) -> Option { + match self { + MixtureRuntime::Bayes(m) => { + Some(MixtureRuntimeCheckpoint::Bayes(BayesMixtureCheckpoint { + experts: checkpoint_experts(&mut m.experts)?, + scratch_logps: m.scratch_logps.clone(), + scratch_mix: m.scratch_mix.clone(), + bitwise: m.bitwise.clone(), + cached_symbol: m.cached_symbol, + cached_log_mix: m.cached_log_mix, + cache_valid: m.cache_valid, + total_log_loss: m.total_log_loss, + })) + } + MixtureRuntime::Fading(m) => Some(MixtureRuntimeCheckpoint::Fading( + FadingBayesMixtureCheckpoint { + experts: checkpoint_experts(&mut m.experts)?, + scratch_logps: m.scratch_logps.clone(), + scratch_mix: m.scratch_mix.clone(), + bitwise: m.bitwise.clone(), + cached_symbol: m.cached_symbol, + cached_log_predictive: m.cached_log_predictive, + cached_log_evidence: m.cached_log_evidence, + cache_valid: m.cache_valid, + total_log_loss: m.total_log_loss, + }, + )), + MixtureRuntime::Switching(m) => Some(MixtureRuntimeCheckpoint::Switching( + SwitchingMixtureCheckpoint { + experts: checkpoint_experts(&mut m.experts)?, + scratch_logps: m.scratch_logps.clone(), + scratch_joint: m.scratch_joint.clone(), + scratch_weights: m.scratch_weights.clone(), + bitwise: m.bitwise.clone(), + cached_symbol: m.cached_symbol, + cached_log_mix: m.cached_log_mix, + cache_valid: m.cache_valid, + total_log_loss: m.total_log_loss, + update_count: m.update_count, + }, + )), + MixtureRuntime::Convex(m) => { + Some(MixtureRuntimeCheckpoint::Convex(ConvexMixtureCheckpoint { + experts: checkpoint_experts(&mut m.experts)?, + lambda: m.lambda.clone(), + scratch_logps: m.scratch_logps.clone(), + projection_scratch: m.projection_scratch.clone(), + bitwise: m.bitwise.clone(), + cached_symbol: m.cached_symbol, + cached_log_mix: m.cached_log_mix, + cache_valid: m.cache_valid, + total_log_loss: m.total_log_loss, + update_count: m.update_count, + })) + } + MixtureRuntime::Mdl(m) => Some(MixtureRuntimeCheckpoint::Mdl(MdlSelectorCheckpoint { + experts: checkpoint_experts(&mut m.experts)?, + scratch_logps: m.scratch_logps.clone(), + bitwise: m.bitwise.clone(), + total_log_loss: m.total_log_loss, + last_best: m.last_best, + cached_symbol: m.cached_symbol, + cached_best_idx: m.cached_best_idx, + cached_best_logp: m.cached_best_logp, + cache_valid: m.cache_valid, + })), + MixtureRuntime::Neural(m) => { + Some(MixtureRuntimeCheckpoint::Neural(NeuralMixtureCheckpoint { + experts: checkpoint_experts(&mut m.experts)?, + neural: m.neural.clone(), + analyzer: m.analyzer.clone(), + scratch_expert_logps: m.scratch_expert_logps.clone(), + scratch_mix_weights: m.scratch_mix_weights.clone(), + bitwise: m.bitwise.clone(), + eval_cache_valid: m.eval_cache_valid, + eval_cache_full_valid: m.eval_cache_full_valid, + eval_cache_history: m.eval_cache_history, + eval_cache_symbol: m.eval_cache_symbol, + eval_cache_logp: m.eval_cache_logp, + eval_cache_mix_logps: m.eval_cache_mix_logps, + eval_cache_expert_logps: m.eval_cache_expert_logps.clone(), + total_log_loss: m.total_log_loss, + })) + } + } + } + + pub(crate) fn restore_checkpoint(&mut self, checkpoint: &MixtureRuntimeCheckpoint) { + match (self, checkpoint) { + (MixtureRuntime::Bayes(m), MixtureRuntimeCheckpoint::Bayes(ck)) => { + restore_experts(&mut m.experts, &ck.experts); + m.scratch_logps = ck.scratch_logps.clone(); + m.scratch_mix = ck.scratch_mix.clone(); + m.bitwise = ck.bitwise.clone(); + m.cached_symbol = ck.cached_symbol; + m.cached_log_mix = ck.cached_log_mix; + m.cache_valid = ck.cache_valid; + m.total_log_loss = ck.total_log_loss; + } + (MixtureRuntime::Fading(m), MixtureRuntimeCheckpoint::Fading(ck)) => { + restore_experts(&mut m.experts, &ck.experts); + m.scratch_logps = ck.scratch_logps.clone(); + m.scratch_mix = ck.scratch_mix.clone(); + m.bitwise = ck.bitwise.clone(); + m.cached_symbol = ck.cached_symbol; + m.cached_log_predictive = ck.cached_log_predictive; + m.cached_log_evidence = ck.cached_log_evidence; + m.cache_valid = ck.cache_valid; + m.total_log_loss = ck.total_log_loss; + } + (MixtureRuntime::Switching(m), MixtureRuntimeCheckpoint::Switching(ck)) => { + restore_experts(&mut m.experts, &ck.experts); + m.scratch_logps = ck.scratch_logps.clone(); + m.scratch_joint = ck.scratch_joint.clone(); + m.scratch_weights = ck.scratch_weights.clone(); + m.bitwise = ck.bitwise.clone(); + m.cached_symbol = ck.cached_symbol; + m.cached_log_mix = ck.cached_log_mix; + m.cache_valid = ck.cache_valid; + m.total_log_loss = ck.total_log_loss; + m.update_count = ck.update_count; + } + (MixtureRuntime::Convex(m), MixtureRuntimeCheckpoint::Convex(ck)) => { + restore_experts(&mut m.experts, &ck.experts); + m.lambda = ck.lambda.clone(); + m.scratch_logps = ck.scratch_logps.clone(); + m.projection_scratch = ck.projection_scratch.clone(); + m.bitwise = ck.bitwise.clone(); + m.cached_symbol = ck.cached_symbol; + m.cached_log_mix = ck.cached_log_mix; + m.cache_valid = ck.cache_valid; + m.total_log_loss = ck.total_log_loss; + m.update_count = ck.update_count; + } + (MixtureRuntime::Mdl(m), MixtureRuntimeCheckpoint::Mdl(ck)) => { + restore_experts(&mut m.experts, &ck.experts); + m.scratch_logps = ck.scratch_logps.clone(); + m.bitwise = ck.bitwise.clone(); + m.total_log_loss = ck.total_log_loss; + m.last_best = ck.last_best; + m.cached_symbol = ck.cached_symbol; + m.cached_best_idx = ck.cached_best_idx; + m.cached_best_logp = ck.cached_best_logp; + m.cache_valid = ck.cache_valid; + } + (MixtureRuntime::Neural(m), MixtureRuntimeCheckpoint::Neural(ck)) => { + restore_experts(&mut m.experts, &ck.experts); + m.neural = ck.neural.clone(); + m.analyzer = ck.analyzer.clone(); + m.scratch_expert_logps = ck.scratch_expert_logps.clone(); + m.scratch_mix_weights = ck.scratch_mix_weights.clone(); + m.bitwise = ck.bitwise.clone(); + m.eval_cache_valid = ck.eval_cache_valid; + m.eval_cache_full_valid = ck.eval_cache_full_valid; + m.eval_cache_history = ck.eval_cache_history; + m.eval_cache_symbol = ck.eval_cache_symbol; + m.eval_cache_logp = ck.eval_cache_logp; + m.eval_cache_mix_logps = ck.eval_cache_mix_logps; + m.eval_cache_expert_logps = ck.eval_cache_expert_logps.clone(); + m.total_log_loss = ck.total_log_loss; + } + _ => panic!("mismatched MixtureRuntime checkpoint variant"), + } + } + + pub(crate) fn discard_checkpoint(&mut self, checkpoint: MixtureRuntimeCheckpoint) { + match (self, checkpoint) { + (MixtureRuntime::Bayes(m), MixtureRuntimeCheckpoint::Bayes(ck)) => { + discard_expert_checkpoints(&mut m.experts, ck.experts); + } + (MixtureRuntime::Fading(m), MixtureRuntimeCheckpoint::Fading(ck)) => { + discard_expert_checkpoints(&mut m.experts, ck.experts); + } + (MixtureRuntime::Switching(m), MixtureRuntimeCheckpoint::Switching(ck)) => { + discard_expert_checkpoints(&mut m.experts, ck.experts); + } + (MixtureRuntime::Convex(m), MixtureRuntimeCheckpoint::Convex(ck)) => { + discard_expert_checkpoints(&mut m.experts, ck.experts); + } + (MixtureRuntime::Mdl(m), MixtureRuntimeCheckpoint::Mdl(ck)) => { + discard_expert_checkpoints(&mut m.experts, ck.experts); + } + (MixtureRuntime::Neural(m), MixtureRuntimeCheckpoint::Neural(ck)) => { + discard_expert_checkpoints(&mut m.experts, ck.experts); + } + _ => panic!("mismatched MixtureRuntime checkpoint variant"), + } + } + + fn lifecycle_checkpoint( + &mut self, + op: OnlineBytePredictorLifecycleOp, + ) -> MixtureRuntimeLifecycleCheckpoint { + let expert_op = ExpertLifecycleOp::from_predictor_op(op); + match self { + MixtureRuntime::Bayes(m) => { + MixtureRuntimeLifecycleCheckpoint::Bayes(BayesMixtureLifecycleCheckpoint { + experts: lifecycle_checkpoint_experts(&mut m.experts, expert_op), + scratch_logps: m.scratch_logps.clone(), + scratch_mix: m.scratch_mix.clone(), + bitwise: m.bitwise.clone(), + cached_symbol: m.cached_symbol, + cached_log_mix: m.cached_log_mix, + cache_valid: m.cache_valid, + total_log_loss: m.total_log_loss, + }) + } + MixtureRuntime::Fading(m) => { + MixtureRuntimeLifecycleCheckpoint::Fading(FadingBayesMixtureLifecycleCheckpoint { + experts: lifecycle_checkpoint_experts(&mut m.experts, expert_op), + scratch_logps: m.scratch_logps.clone(), + scratch_mix: m.scratch_mix.clone(), + bitwise: m.bitwise.clone(), + cached_symbol: m.cached_symbol, + cached_log_predictive: m.cached_log_predictive, + cached_log_evidence: m.cached_log_evidence, + cache_valid: m.cache_valid, + total_log_loss: m.total_log_loss, + }) + } + MixtureRuntime::Switching(m) => { + MixtureRuntimeLifecycleCheckpoint::Switching(SwitchingMixtureLifecycleCheckpoint { + experts: lifecycle_checkpoint_experts(&mut m.experts, expert_op), + scratch_logps: m.scratch_logps.clone(), + scratch_joint: m.scratch_joint.clone(), + scratch_weights: m.scratch_weights.clone(), + bitwise: m.bitwise.clone(), + cached_symbol: m.cached_symbol, + cached_log_mix: m.cached_log_mix, + cache_valid: m.cache_valid, + total_log_loss: m.total_log_loss, + update_count: m.update_count, + }) + } + MixtureRuntime::Convex(m) => { + MixtureRuntimeLifecycleCheckpoint::Convex(ConvexMixtureLifecycleCheckpoint { + experts: lifecycle_checkpoint_experts(&mut m.experts, expert_op), + lambda: m.lambda.clone(), + scratch_logps: m.scratch_logps.clone(), + projection_scratch: m.projection_scratch.clone(), + bitwise: m.bitwise.clone(), + cached_symbol: m.cached_symbol, + cached_log_mix: m.cached_log_mix, + cache_valid: m.cache_valid, + total_log_loss: m.total_log_loss, + update_count: m.update_count, + }) + } + MixtureRuntime::Mdl(m) => { + MixtureRuntimeLifecycleCheckpoint::Mdl(MdlSelectorLifecycleCheckpoint { + experts: lifecycle_checkpoint_experts(&mut m.experts, expert_op), + scratch_logps: m.scratch_logps.clone(), + bitwise: m.bitwise.clone(), + total_log_loss: m.total_log_loss, + last_best: m.last_best, + cached_symbol: m.cached_symbol, + cached_best_idx: m.cached_best_idx, + cached_best_logp: m.cached_best_logp, + cache_valid: m.cache_valid, + }) + } + MixtureRuntime::Neural(m) => { + MixtureRuntimeLifecycleCheckpoint::Neural(NeuralMixtureLifecycleCheckpoint { + experts: lifecycle_checkpoint_experts(&mut m.experts, expert_op), + neural: m.neural.clone(), + analyzer: m.analyzer.clone(), + scratch_expert_logps: m.scratch_expert_logps.clone(), + scratch_mix_weights: m.scratch_mix_weights.clone(), + bitwise: m.bitwise.clone(), + eval_cache_valid: m.eval_cache_valid, + eval_cache_full_valid: m.eval_cache_full_valid, + eval_cache_history: m.eval_cache_history, + eval_cache_symbol: m.eval_cache_symbol, + eval_cache_logp: m.eval_cache_logp, + eval_cache_mix_logps: m.eval_cache_mix_logps, + eval_cache_expert_logps: m.eval_cache_expert_logps.clone(), + total_log_loss: m.total_log_loss, + }) + } + } + } + + fn restore_lifecycle_checkpoint( + &mut self, + op: OnlineBytePredictorLifecycleOp, + checkpoint: MixtureRuntimeLifecycleCheckpoint, + ) { + let expert_op = ExpertLifecycleOp::from_predictor_op(op); + match (self, checkpoint) { + (MixtureRuntime::Bayes(m), MixtureRuntimeLifecycleCheckpoint::Bayes(ck)) => { + restore_lifecycle_experts(&mut m.experts, ck.experts, expert_op); + m.scratch_logps = ck.scratch_logps; + m.scratch_mix = ck.scratch_mix; + m.bitwise = ck.bitwise; + m.cached_symbol = ck.cached_symbol; + m.cached_log_mix = ck.cached_log_mix; + m.cache_valid = ck.cache_valid; + m.total_log_loss = ck.total_log_loss; + } + (MixtureRuntime::Fading(m), MixtureRuntimeLifecycleCheckpoint::Fading(ck)) => { + restore_lifecycle_experts(&mut m.experts, ck.experts, expert_op); + m.scratch_logps = ck.scratch_logps; + m.scratch_mix = ck.scratch_mix; + m.bitwise = ck.bitwise; + m.cached_symbol = ck.cached_symbol; + m.cached_log_predictive = ck.cached_log_predictive; + m.cached_log_evidence = ck.cached_log_evidence; + m.cache_valid = ck.cache_valid; + m.total_log_loss = ck.total_log_loss; + } + (MixtureRuntime::Switching(m), MixtureRuntimeLifecycleCheckpoint::Switching(ck)) => { + restore_lifecycle_experts(&mut m.experts, ck.experts, expert_op); + m.scratch_logps = ck.scratch_logps; + m.scratch_joint = ck.scratch_joint; + m.scratch_weights = ck.scratch_weights; + m.bitwise = ck.bitwise; + m.cached_symbol = ck.cached_symbol; + m.cached_log_mix = ck.cached_log_mix; + m.cache_valid = ck.cache_valid; + m.total_log_loss = ck.total_log_loss; + m.update_count = ck.update_count; + } + (MixtureRuntime::Convex(m), MixtureRuntimeLifecycleCheckpoint::Convex(ck)) => { + restore_lifecycle_experts(&mut m.experts, ck.experts, expert_op); + m.lambda = ck.lambda; + m.scratch_logps = ck.scratch_logps; + m.projection_scratch = ck.projection_scratch; + m.bitwise = ck.bitwise; + m.cached_symbol = ck.cached_symbol; + m.cached_log_mix = ck.cached_log_mix; + m.cache_valid = ck.cache_valid; + m.total_log_loss = ck.total_log_loss; + m.update_count = ck.update_count; + } + (MixtureRuntime::Mdl(m), MixtureRuntimeLifecycleCheckpoint::Mdl(ck)) => { + restore_lifecycle_experts(&mut m.experts, ck.experts, expert_op); + m.scratch_logps = ck.scratch_logps; + m.bitwise = ck.bitwise; + m.total_log_loss = ck.total_log_loss; + m.last_best = ck.last_best; + m.cached_symbol = ck.cached_symbol; + m.cached_best_idx = ck.cached_best_idx; + m.cached_best_logp = ck.cached_best_logp; + m.cache_valid = ck.cache_valid; + } + (MixtureRuntime::Neural(m), MixtureRuntimeLifecycleCheckpoint::Neural(ck)) => { + restore_lifecycle_experts(&mut m.experts, ck.experts, expert_op); + m.neural = ck.neural; + m.analyzer = ck.analyzer; + m.scratch_expert_logps = ck.scratch_expert_logps; + m.scratch_mix_weights = ck.scratch_mix_weights; + m.bitwise = ck.bitwise; + m.eval_cache_valid = ck.eval_cache_valid; + m.eval_cache_full_valid = ck.eval_cache_full_valid; + m.eval_cache_history = ck.eval_cache_history; + m.eval_cache_symbol = ck.eval_cache_symbol; + m.eval_cache_logp = ck.eval_cache_logp; + m.eval_cache_mix_logps = ck.eval_cache_mix_logps; + m.eval_cache_expert_logps = ck.eval_cache_expert_logps; + m.total_log_loss = ck.total_log_loss; + } + _ => panic!("mismatched MixtureRuntime lifecycle checkpoint variant"), + } + } + + fn discard_lifecycle_checkpoint( + &mut self, + op: OnlineBytePredictorLifecycleOp, + checkpoint: MixtureRuntimeLifecycleCheckpoint, + ) { + let expert_op = ExpertLifecycleOp::from_predictor_op(op); + match (self, checkpoint) { + (MixtureRuntime::Bayes(m), MixtureRuntimeLifecycleCheckpoint::Bayes(ck)) => { + discard_lifecycle_experts(&mut m.experts, ck.experts, expert_op); + } + (MixtureRuntime::Fading(m), MixtureRuntimeLifecycleCheckpoint::Fading(ck)) => { + discard_lifecycle_experts(&mut m.experts, ck.experts, expert_op); + } + (MixtureRuntime::Switching(m), MixtureRuntimeLifecycleCheckpoint::Switching(ck)) => { + discard_lifecycle_experts(&mut m.experts, ck.experts, expert_op); + } + (MixtureRuntime::Convex(m), MixtureRuntimeLifecycleCheckpoint::Convex(ck)) => { + discard_lifecycle_experts(&mut m.experts, ck.experts, expert_op); + } + (MixtureRuntime::Mdl(m), MixtureRuntimeLifecycleCheckpoint::Mdl(ck)) => { + discard_lifecycle_experts(&mut m.experts, ck.experts, expert_op); + } + (MixtureRuntime::Neural(m), MixtureRuntimeLifecycleCheckpoint::Neural(ck)) => { + discard_lifecycle_experts(&mut m.experts, ck.experts, expert_op); + } + _ => panic!("mismatched MixtureRuntime lifecycle checkpoint variant"), + } + } + + pub(crate) fn clear_checkpoints_if_supported(&mut self) { + match self { + MixtureRuntime::Bayes(m) => clear_expert_checkpoints(&mut m.experts), + MixtureRuntime::Fading(m) => clear_expert_checkpoints(&mut m.experts), + MixtureRuntime::Switching(m) => clear_expert_checkpoints(&mut m.experts), + MixtureRuntime::Convex(m) => clear_expert_checkpoints(&mut m.experts), + MixtureRuntime::Mdl(m) => clear_expert_checkpoints(&mut m.experts), + MixtureRuntime::Neural(m) => clear_expert_checkpoints(&mut m.experts), + } + } + + pub(crate) fn supports_frozen_reset(&self) -> bool { + match self { + MixtureRuntime::Bayes(m) => experts_support_frozen_reset(&m.experts), + MixtureRuntime::Fading(m) => experts_support_frozen_reset(&m.experts), + MixtureRuntime::Switching(m) => experts_support_frozen_reset(&m.experts), + MixtureRuntime::Convex(m) => experts_support_frozen_reset(&m.experts), + MixtureRuntime::Mdl(m) => experts_support_frozen_reset(&m.experts), + MixtureRuntime::Neural(m) => experts_support_frozen_reset(&m.experts), + } + } + + pub(crate) fn begin_stream(&mut self, total_symbols: Option) -> Result<(), String> { + match self { + MixtureRuntime::Bayes(m) => begin_expert_stream(&mut m.experts, total_symbols), + MixtureRuntime::Fading(m) => begin_expert_stream(&mut m.experts, total_symbols), + MixtureRuntime::Switching(m) => begin_expert_stream(&mut m.experts, total_symbols), + MixtureRuntime::Convex(m) => begin_expert_stream(&mut m.experts, total_symbols), + MixtureRuntime::Mdl(m) => begin_expert_stream(&mut m.experts, total_symbols), + MixtureRuntime::Neural(m) => begin_expert_stream(&mut m.experts, total_symbols), + } + } + + pub(crate) fn begin_fresh_stream(&mut self, total_symbols: Option) -> Result<(), String> { + match self { + MixtureRuntime::Bayes(m) => m.begin_fresh_stream(total_symbols), + MixtureRuntime::Fading(m) => m.begin_fresh_stream(total_symbols), + MixtureRuntime::Switching(m) => m.begin_fresh_stream(total_symbols), + MixtureRuntime::Convex(m) => m.begin_fresh_stream(total_symbols), + MixtureRuntime::Mdl(m) => m.begin_fresh_stream(total_symbols), + MixtureRuntime::Neural(m) => m.begin_fresh_stream(total_symbols), + } + } + + pub(crate) fn finish_stream(&mut self) -> Result<(), String> { + match self { + MixtureRuntime::Bayes(m) => finish_expert_stream(&mut m.experts), + MixtureRuntime::Fading(m) => finish_expert_stream(&mut m.experts), + MixtureRuntime::Switching(m) => finish_expert_stream(&mut m.experts), + MixtureRuntime::Convex(m) => finish_expert_stream(&mut m.experts), + MixtureRuntime::Mdl(m) => finish_expert_stream(&mut m.experts), + MixtureRuntime::Neural(m) => finish_expert_stream(&mut m.experts), + } + } + + pub(crate) fn reset_frozen(&mut self, total_symbols: Option) -> Result<(), String> { + match self { + MixtureRuntime::Bayes(m) => m.reset_frozen(total_symbols), + MixtureRuntime::Fading(m) => m.reset_frozen(total_symbols), + MixtureRuntime::Switching(m) => m.reset_frozen(total_symbols), + MixtureRuntime::Convex(m) => m.reset_frozen(total_symbols), + MixtureRuntime::Mdl(m) => m.reset_frozen(total_symbols), + MixtureRuntime::Neural(m) => m.reset_frozen(total_symbols), + } + } + + /// Non-mutating log-probability (nats) for `symbol` at current state. + pub(crate) fn peek_log_prob(&mut self, symbol: u8) -> f64 { + match self { + MixtureRuntime::Bayes(m) => m.predict_log_prob(symbol), + MixtureRuntime::Fading(m) => m.predict_log_prob(symbol), + MixtureRuntime::Switching(m) => m.predict_log_prob(symbol), + MixtureRuntime::Convex(m) => m.predict_log_prob(symbol), + MixtureRuntime::Mdl(m) => m.predict_log_prob(symbol), + MixtureRuntime::Neural(m) => m.predict_log_prob(symbol), + } + } + + /// Step the mixture and return log-probability (nats). + pub(crate) fn step(&mut self, symbol: u8) -> f64 { + match self { + MixtureRuntime::Bayes(m) => m.step(symbol), + MixtureRuntime::Fading(m) => m.step(symbol), + MixtureRuntime::Switching(m) => m.step(symbol), + MixtureRuntime::Convex(m) => m.step(symbol), + MixtureRuntime::Mdl(m) => m.step(symbol), + MixtureRuntime::Neural(m) => m.step(symbol), + } + } + + pub(crate) fn update_frozen(&mut self, symbol: u8) { + match self { + MixtureRuntime::Bayes(m) => m.update_frozen(symbol), + MixtureRuntime::Fading(m) => m.update_frozen(symbol), + MixtureRuntime::Switching(m) => m.update_frozen(symbol), + MixtureRuntime::Convex(m) => m.update_frozen(symbol), + MixtureRuntime::Mdl(m) => m.update_frozen(symbol), + MixtureRuntime::Neural(m) => m.update_frozen(symbol), + } + } + + pub(crate) fn fill_log_probs(&mut self, out: &mut [f64; 256]) { + match self { + MixtureRuntime::Bayes(m) => m.fill_log_probs(out), + MixtureRuntime::Fading(m) => m.fill_log_probs(out), + MixtureRuntime::Switching(m) => m.fill_log_probs(out), + MixtureRuntime::Convex(m) => m.fill_log_probs(out), + MixtureRuntime::Mdl(m) => m.fill_log_probs(out), + MixtureRuntime::Neural(m) => m.fill_log_probs(out), + } + } + + pub(crate) fn has_native_msb_byte_prefix(&self) -> bool { + match self { + MixtureRuntime::Bayes(m) => experts_have_native_msb_byte_prefix(&m.experts), + MixtureRuntime::Fading(m) => experts_have_native_msb_byte_prefix(&m.experts), + MixtureRuntime::Switching(m) => experts_have_native_msb_byte_prefix(&m.experts), + MixtureRuntime::Convex(m) => experts_have_native_msb_byte_prefix(&m.experts), + MixtureRuntime::Mdl(m) => experts_have_native_msb_byte_prefix(&m.experts), + MixtureRuntime::Neural(m) => experts_have_native_msb_byte_prefix(&m.experts), + } + } + + pub(crate) fn begin_native_msb_byte_prefix(&mut self) -> Result { + match self { + MixtureRuntime::Bayes(m) => { + let weights: Vec = normalized_expert_log_weights(&m.experts); + m.bitwise.begin(&mut m.experts, &weights) + } + MixtureRuntime::Fading(m) => { + let weights: Vec = normalized_scaled_expert_log_weights(&m.experts, m.decay); + m.bitwise.begin(&mut m.experts, &weights) + } + MixtureRuntime::Switching(m) => { + let weights: Vec = normalized_expert_log_weights(&m.experts); + m.bitwise.begin(&mut m.experts, &weights) + } + MixtureRuntime::Convex(m) => m.bitwise.begin(&mut m.experts, &m.lambda), + MixtureRuntime::Mdl(m) => { + let best_idx: usize = best_expert_index(&m.experts); + let mut weights: Vec = vec![0.0; m.experts.len()]; + if let Some(slot) = weights.get_mut(best_idx) { + *slot = 1.0; + } + m.bitwise.begin(&mut m.experts, &weights) + } + MixtureRuntime::Neural(m) => { + if m.experts.len() == 1 { + m.scratch_mix_weights.resize(1, 1.0); + m.scratch_mix_weights[0] = 1.0; + } else { + m.sync_history_state(); + m.neural.evaluate_expert_weights(); + m.scratch_mix_weights + .copy_from_slice(m.neural.expert_weights()); + } + m.bitwise.begin(&mut m.experts, &m.scratch_mix_weights) + } + } + } + + pub(crate) fn abort_empty_native_msb_byte_prefix(&mut self) -> Result { + match self { + MixtureRuntime::Bayes(m) => m.bitwise.abort_empty(&mut m.experts), + MixtureRuntime::Fading(m) => m.bitwise.abort_empty(&mut m.experts), + MixtureRuntime::Switching(m) => m.bitwise.abort_empty(&mut m.experts), + MixtureRuntime::Convex(m) => m.bitwise.abort_empty(&mut m.experts), + MixtureRuntime::Mdl(m) => m.bitwise.abort_empty(&mut m.experts), + MixtureRuntime::Neural(m) => m.bitwise.abort_empty(&mut m.experts), + } + } + + pub(crate) fn native_msb_prefix_prob_one(&mut self, bit_idx: usize) -> Result { + match self { + MixtureRuntime::Bayes(m) => m.bitwise.prob_one(&mut m.experts, bit_idx), + MixtureRuntime::Fading(m) => m.bitwise.prob_one(&mut m.experts, bit_idx), + MixtureRuntime::Switching(m) => m.bitwise.prob_one(&mut m.experts, bit_idx), + MixtureRuntime::Convex(m) => m.bitwise.prob_one(&mut m.experts, bit_idx), + MixtureRuntime::Mdl(m) => m.bitwise.prob_one(&mut m.experts, bit_idx), + MixtureRuntime::Neural(m) => m.bitwise.prob_one(&mut m.experts, bit_idx), + } + } + + pub(crate) fn observe_native_msb_prefix_bit( + &mut self, + bit_idx: usize, + bit: bool, + ) -> Result<(), String> { + match self { + MixtureRuntime::Bayes(m) => m.bitwise.observe(&mut m.experts, bit_idx, bit), + MixtureRuntime::Fading(m) => m.bitwise.observe(&mut m.experts, bit_idx, bit), + MixtureRuntime::Switching(m) => m.bitwise.observe(&mut m.experts, bit_idx, bit), + MixtureRuntime::Convex(m) => m.bitwise.observe(&mut m.experts, bit_idx, bit), + MixtureRuntime::Mdl(m) => m.bitwise.observe(&mut m.experts, bit_idx, bit), + MixtureRuntime::Neural(m) => m.bitwise.observe(&mut m.experts, bit_idx, bit), + } + } + + pub(crate) fn finish_native_msb_byte_prefix(&mut self, symbol: u8) -> Result<(), String> { + match self { + MixtureRuntime::Bayes(m) => finish_bayes_native_prefix(m, symbol), + MixtureRuntime::Fading(m) => finish_fading_native_prefix(m, symbol), + MixtureRuntime::Switching(m) => finish_switching_native_prefix(m, symbol), + MixtureRuntime::Convex(m) => finish_convex_native_prefix(m, symbol), + MixtureRuntime::Mdl(m) => finish_mdl_native_prefix(m, symbol), + MixtureRuntime::Neural(m) => finish_neural_native_prefix(m, symbol), + } + } +} + +fn experts_have_native_msb_byte_prefix(experts: &[ExpertState]) -> bool { + experts + .iter() + .any(|expert| expert.predictor.has_native_msb_byte_prefix()) +} + +fn normalized_expert_log_weights(experts: &[ExpertState]) -> Vec { + let norm: f64 = logsumexp_weights(experts); + experts + .iter() + .map(|expert| (expert.log_weight - norm).exp()) + .collect() +} + +fn normalized_scaled_expert_log_weights(experts: &[ExpertState], scale: f64) -> Vec { + let mut log_weights: Vec = experts + .iter() + .map(|expert| scale * expert.log_weight) + .collect(); + let norm: f64 = logsumexp(&log_weights); + for weight in &mut log_weights { + *weight = (*weight - norm).exp(); + } + log_weights +} + +fn best_expert_index(experts: &[ExpertState]) -> usize { + let mut best_idx: usize = 0; + let mut best_loss: f64 = f64::INFINITY; + for (idx, expert) in experts.iter().enumerate() { + if expert.cum_log_loss < best_loss { + best_loss = expert.cum_log_loss; + best_idx = idx; + } + } + best_idx +} + +fn finish_bayes_native_prefix(m: &mut BayesMixture, symbol: u8) -> Result<(), String> { + m.bitwise.finish_adaptive(&mut m.experts, symbol)?; + let log_mix = + apply_bayes_update_from_logps(&mut m.experts, &m.bitwise.logps, &mut m.scratch_mix); + m.cache_valid = false; + m.total_log_loss -= log_mix; + Ok(()) +} + +fn finish_fading_native_prefix(m: &mut FadingBayesMixture, symbol: u8) -> Result<(), String> { + m.bitwise.finish_adaptive(&mut m.experts, symbol)?; + let log_predictive = apply_fading_update_from_logps( + &mut m.experts, + &m.bitwise.logps, + &mut m.scratch_mix, + m.decay, + ); + m.cache_valid = false; + m.total_log_loss -= log_predictive; + Ok(()) +} + +fn finish_switching_native_prefix(m: &mut SwitchingMixture, symbol: u8) -> Result<(), String> { + m.bitwise.finish_adaptive(&mut m.experts, symbol)?; + let log_mix = apply_switching_update_from_logps( + &mut m.experts, + &m.bitwise.logps, + &mut m.scratch_joint, + &mut m.scratch_weights, + &m.prior, + m.schedule, + m.alpha, + &mut m.update_count, + ); + m.cache_valid = false; + m.total_log_loss -= log_mix; + Ok(()) +} + +fn finish_convex_native_prefix(m: &mut ConvexMixture, symbol: u8) -> Result<(), String> { + m.bitwise.finish_adaptive(&mut m.experts, symbol)?; + let log_mix = apply_convex_update_from_logps( + &mut m.experts, + &m.bitwise.logps, + &mut m.lambda, + &mut m.projection_scratch, + m.schedule, + m.alpha, + &mut m.update_count, + ); + m.cache_valid = false; + m.total_log_loss -= log_mix; + Ok(()) +} + +fn finish_mdl_native_prefix(m: &mut MdlSelector, symbol: u8) -> Result<(), String> { + let best_idx: usize = best_expert_index(&m.experts); + m.bitwise.finish_adaptive(&mut m.experts, symbol)?; + let logp = + apply_mdl_update_from_logps(&mut m.experts, &m.bitwise.logps, best_idx, &mut m.last_best); + m.cache_valid = false; + m.total_log_loss -= logp; + Ok(()) +} + +fn finish_neural_native_prefix(m: &mut NeuralMixture, symbol: u8) -> Result<(), String> { + m.bitwise.finish_adaptive(&mut m.experts, symbol)?; + m.scratch_expert_logps + .copy_from_slice(&m.bitwise.logps[..m.experts.len()]); + for idx in 0..m.experts.len() { + m.experts[idx].cum_log_loss -= m.scratch_expert_logps[idx]; + } + let logp: f64 = m + .bitwise + .weights + .iter() + .zip(m.bitwise.logps.iter()) + .map(|(&w, &lp)| w * lp.exp()) + .sum::() + .max(m.min_prob) + .ln(); + finish_neural_update_from_logps(m, symbol, logp, m.experts.len() > 1); + m.eval_cache_history = m.neural.history_state(); + Ok(()) +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum ExpertLifecycleOp { + BeginStream, + BeginFreshStream, + ResetFrozen, + FinishStream, +} + +impl ExpertLifecycleOp { + fn as_str(self) -> &'static str { + match self { + Self::BeginStream => "begin_stream", + Self::BeginFreshStream => "begin_fresh_stream", + Self::ResetFrozen => "reset_frozen", + Self::FinishStream => "finish_stream", + } + } + + fn to_predictor_op(self) -> OnlineBytePredictorLifecycleOp { + match self { + Self::BeginStream => OnlineBytePredictorLifecycleOp::BeginStream, + Self::BeginFreshStream => OnlineBytePredictorLifecycleOp::BeginFreshStream, + Self::ResetFrozen => OnlineBytePredictorLifecycleOp::ResetFrozen, + Self::FinishStream => OnlineBytePredictorLifecycleOp::FinishStream, + } + } + + fn from_predictor_op(op: OnlineBytePredictorLifecycleOp) -> Self { + match op { + OnlineBytePredictorLifecycleOp::BeginStream => Self::BeginStream, + OnlineBytePredictorLifecycleOp::BeginFreshStream => Self::BeginFreshStream, + OnlineBytePredictorLifecycleOp::ResetFrozen => Self::ResetFrozen, + OnlineBytePredictorLifecycleOp::FinishStream => Self::FinishStream, + } + } +} + +/// Apply a fallible lifecycle operation to every expert all-or-nothing. +/// +/// Lifecycle hooks may mutate non-journaled state such as CTW history, neural +/// online-policy buffers, or wrapper caches before reporting an error. The +/// update checkpoints used for speculative byte/bit prediction are therefore not +/// sufficient here: rollback must restore the whole expert state that existed +/// before the lifecycle operation began. A full expert snapshot is faithful by +/// construction, and neural model weights are already `Arc`-shared by their +/// backends, so this does not deep-copy those parameters. +fn transact_expert_lifecycle( + experts: &mut [ExpertState], + op: ExpertLifecycleOp, + mut apply: impl FnMut(&mut ExpertState) -> Result<(), String>, +) -> Result<(), String> { + if experts.is_empty() { + return Ok(()); + } + let mut backups: Vec<(usize, ExpertLifecycleToken)> = Vec::with_capacity(experts.len()); + for idx in 0..experts.len() { + let name = experts[idx].name.clone(); + let token = experts[idx].snapshot_lifecycle(op); + backups.push((idx, token)); + if let Err(err) = apply(&mut experts[idx]) { + for (restore_idx, token) in backups.into_iter().rev() { + experts[restore_idx].restore_lifecycle(op, token); + } + return Err(format!( + "mixture expert lifecycle {} failed for expert #{idx} '{name}': {err}", + op.as_str() + )); + } + } + for (idx, token) in backups { + experts[idx].discard_lifecycle(op, token); + } + Ok(()) +} + +fn begin_expert_stream( + experts: &mut [ExpertState], + total_symbols: Option, +) -> Result<(), String> { + transact_expert_lifecycle(experts, ExpertLifecycleOp::BeginStream, |expert| { + expert.begin_stream(total_symbols) + }) +} + +fn begin_expert_fresh_stream( + experts: &mut [ExpertState], + total_symbols: Option, +) -> Result<(), String> { + transact_expert_lifecycle(experts, ExpertLifecycleOp::BeginFreshStream, |expert| { + expert.begin_fresh_stream(total_symbols) + }) +} + +fn reset_expert_frozen_stream( + experts: &mut [ExpertState], + total_symbols: Option, +) -> Result<(), String> { + transact_expert_lifecycle(experts, ExpertLifecycleOp::ResetFrozen, |expert| { + expert.reset_frozen(total_symbols) + }) +} + +fn experts_support_frozen_reset(experts: &[ExpertState]) -> bool { + experts + .iter() + .all(|expert| expert.predictor.supports_frozen_reset()) +} + +fn finish_expert_stream(experts: &mut [ExpertState]) -> Result<(), String> { + transact_expert_lifecycle(experts, ExpertLifecycleOp::FinishStream, |expert| { + expert.finish_stream() + }) +} + +#[cfg(test)] +pub(crate) fn build_mixture_runtime( + spec: &MixtureSpec, + experts: &[ExpertConfig], +) -> Result { + spec.validate().map_err(|err| err.to_string())?; + build_mixture_runtime_from_fields(spec.kind, spec.schedule, spec.alpha, spec.decay, experts) +} + +#[cfg(feature = "backend-mixture")] +pub(crate) fn build_mixture_runtime_from_compiled( + backend: &CompiledRateBackend, + experts: &[ExpertConfig], +) -> Result { + let crate::spec::core::RateBackendPlan::Mixture { + kind, + schedule, + alpha, + decay, + .. + } = backend.plan() + else { + return Err("compiled backend is not a mixture backend".to_string()); + }; + build_mixture_runtime_from_fields(*kind, *schedule, *alpha, *decay, experts) +} + +fn build_mixture_runtime_from_fields( + kind: MixtureKind, + schedule: MixtureScheduleMode, + alpha: f64, + decay: Option, + experts: &[ExpertConfig], +) -> Result { + match kind { + MixtureKind::Bayes => Ok(MixtureRuntime::Bayes(BayesMixture::new(experts))), + MixtureKind::FadingBayes => { + let decay = decay.ok_or_else(|| "fading Bayes mixture requires decay".to_string())?; + Ok(MixtureRuntime::Fading(FadingBayesMixture::new( + experts, decay, + ))) + } + MixtureKind::Switching => Ok(MixtureRuntime::Switching(SwitchingMixture::new( + experts, alpha, schedule, + ))), + MixtureKind::Convex => Ok(MixtureRuntime::Convex(ConvexMixture::new( + experts, alpha, schedule, + ))), + MixtureKind::Mdl => Ok(MixtureRuntime::Mdl(MdlSelector::new(experts))), + MixtureKind::Neural => Ok(MixtureRuntime::Neural(NeuralMixture::new(experts, alpha))), + } +} + +#[cfg(test)] +mod lifecycle_tests { + use super::*; + use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }; + + #[derive(Clone)] + struct LifecycleMockPredict { + state: usize, + fail: Option, + } + + impl LifecycleMockPredict { + fn apply(&mut self, op: ExpertLifecycleOp, next_state: usize) -> Result<(), String> { + self.state = next_state; + if self.fail == Some(op) { + Err(format!("{} failed after mutation", op.as_str())) + } else { + Ok(()) + } + } + } + + impl OnlineBytePredictor for LifecycleMockPredict { + fn begin_stream(&mut self, _total_symbols: Option) -> Result<(), String> { + self.apply(ExpertLifecycleOp::BeginStream, 11) + } + + fn begin_fresh_stream(&mut self, _total_symbols: Option) -> Result<(), String> { + self.apply(ExpertLifecycleOp::BeginFreshStream, 22) + } + + fn reset_frozen(&mut self, _total_symbols: Option) -> Result<(), String> { + self.apply(ExpertLifecycleOp::ResetFrozen, 33) + } + + fn finish_stream(&mut self) -> Result<(), String> { + self.apply(ExpertLifecycleOp::FinishStream, 44) + } + + fn log_prob(&mut self, _symbol: u8) -> f64 { + self.state as f64 + } + + fn update(&mut self, _symbol: u8) {} + } + + fn lifecycle_expert(name: &str, state: usize, fail: Option) -> ExpertState { + ExpertState { + name: name.to_string(), + log_weight: 0.0, + log_prior: 0.0, + predictor: ExpertPredictor::generic(Box::new(LifecycleMockPredict { state, fail })), + cum_log_loss: 0.0, + } + } + + fn lifecycle_state(expert: &mut ExpertState) -> usize { + expert.log_prob(0) as usize + } + + fn assert_lifecycle_failure_rolls_back(op: ExpertLifecycleOp) { + let mut experts: Vec = vec![ + lifecycle_expert("mutated", 1, None), + lifecycle_expert("failing", 2, Some(op)), + ]; + + let err = match op { + ExpertLifecycleOp::BeginStream => begin_expert_stream(&mut experts, Some(8)), + ExpertLifecycleOp::BeginFreshStream => begin_expert_fresh_stream(&mut experts, Some(8)), + ExpertLifecycleOp::ResetFrozen => reset_expert_frozen_stream(&mut experts, Some(8)), + ExpertLifecycleOp::FinishStream => finish_expert_stream(&mut experts), + } + .expect_err("second expert should fail after mutating"); + + assert!( + err.contains(op.as_str()), + "error should name lifecycle operation: {err}" + ); + assert!( + err.contains("expert #1 'failing'"), + "error should identify failing expert: {err}" + ); + assert!( + err.contains("failed after mutation"), + "error should preserve source context: {err}" + ); + assert_eq!( + lifecycle_state(&mut experts[0]), + 1, + "earlier successful expert mutation must be rolled back" + ); + assert_eq!( + lifecycle_state(&mut experts[1]), + 2, + "failing expert mutation must be rolled back" + ); + } + + #[test] + fn lifecycle_begin_stream_failure_rolls_back_all_experts() { + assert_lifecycle_failure_rolls_back(ExpertLifecycleOp::BeginStream); + } + + #[test] + fn lifecycle_begin_fresh_stream_failure_rolls_back_all_experts() { + assert_lifecycle_failure_rolls_back(ExpertLifecycleOp::BeginFreshStream); + } + + #[test] + fn lifecycle_reset_frozen_failure_rolls_back_all_experts() { + assert_lifecycle_failure_rolls_back(ExpertLifecycleOp::ResetFrozen); + } + + #[test] + fn lifecycle_finish_stream_failure_rolls_back_all_experts() { + assert_lifecycle_failure_rolls_back(ExpertLifecycleOp::FinishStream); + } + + struct CloneCountingLifecyclePredict { + state: usize, + fail: Option, + clones: Arc, + } + + impl Clone for CloneCountingLifecyclePredict { + fn clone(&self) -> Self { + self.clones.fetch_add(1, Ordering::Relaxed); + Self { + state: self.state, + fail: self.fail, + clones: self.clones.clone(), + } + } + } + + impl OnlineBytePredictor for CloneCountingLifecyclePredict { + fn begin_stream(&mut self, _total_symbols: Option) -> Result<(), String> { + self.state = 99; + if self.fail == Some(ExpertLifecycleOp::BeginStream) { + Err("failed after mutation".to_string()) + } else { + Ok(()) + } + } + + fn log_prob(&mut self, _symbol: u8) -> f64 { + self.state as f64 + } + + fn update(&mut self, _symbol: u8) {} + } + + #[test] + fn lifecycle_failure_snapshots_only_attempted_experts() { + let clones = Arc::new(AtomicUsize::new(0)); + let mut experts = vec![ + ExpertState { + name: "failing".to_string(), + log_weight: 0.0, + log_prior: 0.0, + predictor: ExpertPredictor::generic(Box::new(CloneCountingLifecyclePredict { + state: 1, + fail: Some(ExpertLifecycleOp::BeginStream), + clones: clones.clone(), + })), + cum_log_loss: 0.0, + }, + ExpertState { + name: "unreached".to_string(), + log_weight: 0.0, + log_prior: 0.0, + predictor: ExpertPredictor::generic(Box::new(CloneCountingLifecyclePredict { + state: 2, + fail: None, + clones: clones.clone(), + })), + cum_log_loss: 0.0, + }, + ]; + + begin_expert_stream(&mut experts, Some(1)).expect_err("first expert should fail"); + + assert_eq!( + clones.load(Ordering::Relaxed), + 1, + "transaction must not snapshot experts after the first failure" + ); + assert_eq!(lifecycle_state(&mut experts[0]), 1); + assert_eq!(lifecycle_state(&mut experts[1]), 2); + } +} + +#[cfg(all(test, feature = "all-backends"))] +mod tests { + use super::*; + use crate::api::CalibratedSpec; + use std::sync::{ + Arc, + atomic::{AtomicU64, AtomicUsize, Ordering}, + }; + + #[derive(Clone)] + struct AlwaysPredict { + byte: u8, + } + + impl OnlineBytePredictor for AlwaysPredict { + fn log_prob(&mut self, symbol: u8) -> f64 { + if symbol == self.byte { + 0.0 + } else { + f64::NEG_INFINITY + } + } + + fn update(&mut self, _symbol: u8) {} + } + + #[derive(Clone)] + struct FixedProbPredict { + prob_zero: f64, + } + + impl OnlineBytePredictor for FixedProbPredict { + fn log_prob(&mut self, symbol: u8) -> f64 { + let p = if symbol == 0 { + self.prob_zero + } else { + (1.0 - self.prob_zero) / 255.0 + }; + p.ln() + } + + fn update(&mut self, _symbol: u8) {} + } + + fn weighted_cfg(name: &'static str, weight: f64, prob_zero: f64) -> ExpertConfig { + ExpertConfig::new(name, weight.ln(), move || { + Box::new(FixedProbPredict { prob_zero }) + }) + } + + fn assert_weights_close(actual: &[f64], expected: &[f64], label: &str) { + assert_eq!(actual.len(), expected.len(), "{label} length mismatch"); + for (index, (&a, &e)) in actual.iter().zip(expected.iter()).enumerate() { + assert!( + (a - e).abs() < 1e-12, + "{label}[{index}]: expected {e}, got {a}" + ); + } + } + + fn assert_expert_losses_reset(experts: &[ExpertState], label: &str) { + for expert in experts { + assert!( + expert.cum_log_loss.abs() < 1e-12, + "{label} expert '{}' loss should reset, got {}", + expert.name, + expert.cum_log_loss + ); + } + } + + #[derive(Clone)] + struct NativeBitProbPredict { + prob_one: f64, + } + + impl OnlineBytePredictor for NativeBitProbPredict { + fn log_prob(&mut self, symbol: u8) -> f64 { + let p = if symbol == 0 { + 1.0 - self.prob_one + } else { + self.prob_one / 255.0 + }; + p.max(DEFAULT_MIN_PROB).ln() + } + + fn update(&mut self, _symbol: u8) {} + + fn has_native_msb_byte_prefix(&self) -> bool { + true + } + + fn begin_native_msb_byte_prefix(&mut self) -> Result { + Ok(true) + } + + fn native_msb_prefix_prob_one(&mut self, _bit_idx: usize) -> Result { + Ok(self.prob_one) + } + + fn observe_native_msb_prefix_bit( + &mut self, + _bit_idx: usize, + _bit: bool, + ) -> Result<(), String> { + Ok(()) + } + + fn finish_native_msb_byte_prefix(&mut self, _symbol: u8) -> Result<(), String> { + Ok(()) + } + + // Dummy checkpoint methods so that MixtureBitPrefixState::begin succeeds + // for this mock (which advertises native MSB support). These are safe + // no-ops: the test mock performs no real mutation, and begin discards + // the captured checkpoints on the happy path after all experts prepare. + fn checkpoint_if_supported(&mut self) -> Option { + Some(OnlineBytePredictorCheckpoint::rate_backend( + RateBackendPredictorCheckpoint::Full(Box::new(RateBackendPredictor::Disabled { + reason: "dummy_test_checkpoint".to_string(), + })), + )) + } + + fn restore_checkpoint_if_supported( + &mut self, + _checkpoint: &OnlineBytePredictorCheckpoint, + ) -> bool { + true + } + + fn discard_checkpoint_if_supported( + &mut self, + _checkpoint: OnlineBytePredictorCheckpoint, + ) -> bool { + true + } + } + + #[derive(Clone)] + struct NativeWithoutCheckpointPredict { + begin_calls: Arc, + } + + impl OnlineBytePredictor for NativeWithoutCheckpointPredict { + fn log_prob(&mut self, symbol: u8) -> f64 { + if symbol == 0 { 0.0 } else { f64::NEG_INFINITY } + } + + fn update(&mut self, _symbol: u8) {} + + fn has_native_msb_byte_prefix(&self) -> bool { + true + } + + fn begin_native_msb_byte_prefix(&mut self) -> Result { + self.begin_calls.fetch_add(1, Ordering::Relaxed); + Ok(true) + } + } + + #[test] + fn mixture_bit_prefix_observe_without_prob_one_primes_current_bit() { + let configs = [ + ExpertConfig::uniform("high", || Box::new(NativeBitProbPredict { prob_one: 0.9 })), + ExpertConfig::uniform("low", || Box::new(NativeBitProbPredict { prob_one: 0.1 })), + ]; + let mut experts: Vec = configs.iter().map(ExpertConfig::build).collect(); + let mut bitwise = MixtureBitPrefixState::default(); + assert!( + bitwise.begin(&mut experts, &[0.5, 0.5]).expect("begin"), + "native prefix should activate when experts support native bit stepping" + ); + + // Drive full 8-bit sequence (per MixtureBitPrefixState expected_bit_idx contract + // and finish_adaptive validation at 733) to exercise the "observe without prior + // prob_one" priming path on every step, then finish. This satisfies the state + // machine while preserving the original test intent (priming on observe-only + // updates + final log-likelihood distinction). Single-observe + immediate finish + // violated the sequential contract (now enforced post-mock fix). + let symbol: u8 = 0xFF; // all-1s so high-p1 (0.9) expert has higher final likelihood than low-p1 (0.1) after 8 steps (preserves original distinction intent) + for bit_idx in 0..8 { + let bit = (symbol & (1u8 << (7 - bit_idx))) != 0; + bitwise + .observe(&mut experts, bit_idx, bit) + .expect("observe without prior prob_one"); + if bit_idx == 0 { + assert!( + (bitwise.likelihoods[0] - 0.9).abs() < 1e-12, + "high-prob expert likelihood should use freshly primed p1" + ); + assert!( + (bitwise.likelihoods[1] - 0.1).abs() < 1e-12, + "low-prob expert likelihood should use freshly primed p1" + ); + assert!( + bitwise.primed_bit_idx.is_none(), + "observe should clear priming for the next bit" + ); + } + } + + bitwise + .finish_adaptive(&mut experts, symbol) + .expect("finish adaptive"); + assert!( + bitwise.logps[0] > bitwise.logps[1], + "log-likelihoods should distinguish disagreeing experts after observe-only updates" + ); + } + + #[test] + fn mixture_bit_prefix_requires_monotone_bit_indices() { + let configs = [ExpertConfig::ctw("left", 4), ExpertConfig::ctw("right", 5)]; + let mut experts: Vec = configs.iter().map(ExpertConfig::build).collect(); + let mut bitwise = MixtureBitPrefixState::default(); + assert!(bitwise.begin(&mut experts, &[0.5, 0.5]).expect("begin")); + + let err = bitwise + .prob_one(&mut experts, 1) + .expect_err("bit 1 cannot be queried before bit 0 is observed"); + assert!(err.contains("expected 0")); + + bitwise + .observe(&mut experts, 0, true) + .expect("observe bit 0"); + let err = bitwise + .observe(&mut experts, 0, false) + .expect_err("duplicate observe must be rejected"); + assert!(err.contains("expected 1")); + + let err = bitwise + .prob_one(&mut experts, 8) + .expect_err("bit index 8 must be rejected"); + assert!(err.contains("out of range")); + } + + #[test] + fn mixture_bit_prefix_skips_native_mode_without_checkpoint_support() { + let begin_calls = Arc::new(AtomicUsize::new(0)); + let shared = Arc::clone(&begin_calls); + let configs = [ExpertConfig::uniform("native", move || { + Box::new(NativeWithoutCheckpointPredict { + begin_calls: Arc::clone(&shared), + }) + })]; + let mut experts: Vec = configs.iter().map(ExpertConfig::build).collect(); + let mut bitwise = MixtureBitPrefixState::default(); + + assert!( + !bitwise.begin(&mut experts, &[1.0]).expect("begin"), + "native prefix should be disabled when rollback checkpoints are unavailable" + ); + assert_eq!( + begin_calls.load(Ordering::Relaxed), + 0, + "unsupported native experts must not be entered speculatively" + ); + } + + #[cfg(feature = "backend-ctw")] + #[test] + fn ctw_native_prefix_rejects_invalid_bit_indices() { + let mut predictor = + RateBackendPredictor::from_backend(RateBackend::Ctw { depth: 4 }, DEFAULT_MIN_PROB); + assert!( + predictor + .begin_native_msb_byte_prefix() + .expect("begin native prefix"), + "ctw byte predictor should support native prefix stepping" + ); + + let err = predictor + .native_msb_prefix_prob_one(8) + .expect_err("bit index 8 must be rejected"); + assert!(err.contains("out of range")); + + let err = predictor + .observe_native_msb_prefix_bit(1, true) + .expect_err("bit 1 cannot be observed before bit 0"); + assert!(err.contains("expected 0")); + + predictor + .native_msb_prefix_prob_one(0) + .expect("query bit 0"); + predictor + .observe_native_msb_prefix_bit(0, true) + .expect("observe bit 0"); + let err = predictor + .finish_native_msb_byte_prefix(0b1000_0000) + .expect_err("finish must require a complete byte"); + assert!(err.contains("requires 8 observed bits")); + } + + #[test] + fn bayes_mixture_prefers_correct_expert() { + let configs = vec![ + ExpertConfig::uniform("zero", || Box::new(AlwaysPredict { byte: 0 })), + ExpertConfig::uniform("one", || Box::new(AlwaysPredict { byte: 1 })), + ]; + let mut mix = BayesMixture::new(&configs); + for _ in 0..10 { + mix.step(0); + } + let post = mix.posterior(); + assert!(post[0] > 0.999); + assert!(post[1] < 1e-6); + } + + fn counting_cfg(name: &'static str, calls: Arc) -> ExpertConfig { + ExpertConfig::uniform(name, move || { + Box::new(CountingPredict { + calls: calls.clone(), + }) + }) + } + + #[test] + fn bayes_predict_then_step_reuses_cached_log_probs() { + let c0 = Arc::new(AtomicUsize::new(0)); + let c1 = Arc::new(AtomicUsize::new(0)); + let mut mix = BayesMixture::new(&[ + counting_cfg("c0", c0.clone()), + counting_cfg("c1", c1.clone()), + ]); + let _ = mix.predict_log_prob(0); + let after_predict = c0.load(Ordering::Relaxed) + c1.load(Ordering::Relaxed); + assert_eq!(after_predict, 2); + let _ = mix.step(0); + let after_step = c0.load(Ordering::Relaxed) + c1.load(Ordering::Relaxed); + assert_eq!(after_step, after_predict); + } + + #[test] + fn fading_predict_then_step_reuses_cached_log_probs() { + let c0 = Arc::new(AtomicUsize::new(0)); + let c1 = Arc::new(AtomicUsize::new(0)); + let mut mix = FadingBayesMixture::new( + &[ + counting_cfg("c0", c0.clone()), + counting_cfg("c1", c1.clone()), + ], + 0.95, + ); + let _ = mix.predict_log_prob(0); + let after_predict = c0.load(Ordering::Relaxed) + c1.load(Ordering::Relaxed); + assert_eq!(after_predict, 2); + let _ = mix.step(0); + let after_step = c0.load(Ordering::Relaxed) + c1.load(Ordering::Relaxed); + assert_eq!(after_step, after_predict); + } + + #[test] + fn switching_predict_then_step_reuses_cached_log_probs() { + let c0 = Arc::new(AtomicUsize::new(0)); + let c1 = Arc::new(AtomicUsize::new(0)); + let mut mix = SwitchingMixture::new( + &[ + counting_cfg("c0", c0.clone()), + counting_cfg("c1", c1.clone()), + ], + 0.05, + MixtureScheduleMode::Default, + ); + let _ = mix.predict_log_prob(0); + let after_predict = c0.load(Ordering::Relaxed) + c1.load(Ordering::Relaxed); + assert_eq!(after_predict, 2); + let _ = mix.step(0); + let after_step = c0.load(Ordering::Relaxed) + c1.load(Ordering::Relaxed); + assert_eq!(after_step, after_predict); + } + + #[test] + fn switching_mixture_matches_fixed_share_update_for_uniform_prior() { + let configs = vec![weighted_cfg("a", 0.5, 0.8), weighted_cfg("b", 0.5, 0.3)]; + let alpha = 0.2; + let mut mix = SwitchingMixture::new(&configs, alpha, MixtureScheduleMode::Default); + + let predicted = mix.predict_log_prob(0).exp(); + assert!((predicted - 0.55).abs() < 1e-12, "predicted={predicted}"); + + let observed = mix.step(0).exp(); + assert!((observed - 0.55).abs() < 1e-12, "observed={observed}"); + + let post = mix.posterior(); + let posterior_a = 0.5 * 0.8 / 0.55; + let posterior_b = 0.5 * 0.3 / 0.55; + let expected_a = (1.0 - alpha) * posterior_a + alpha * posterior_b; + let expected_b = (1.0 - alpha) * posterior_b + alpha * posterior_a; + assert!( + (post[0] - expected_a).abs() < 1e-12 && (post[1] - expected_b).abs() < 1e-12, + "expected [{expected_a}, {expected_b}], got {:?}", + post + ); + } + + #[test] + fn switching_mixture_switches_according_to_prior_over_other_experts() { + let configs = vec![ + weighted_cfg("a", 0.5, 0.75), + weighted_cfg("b", 0.3, 0.25), + weighted_cfg("c", 0.2, 0.60), + ]; + let alpha = 0.15; + let mut mix = SwitchingMixture::new(&configs, alpha, MixtureScheduleMode::Default); + + let _ = mix.step(0); + let post = mix.posterior(); + + let current = [0.5_f64, 0.3, 0.2]; + let likelihood = [0.75_f64, 0.25, 0.60]; + let mix_prob = current + .iter() + .zip(likelihood.iter()) + .map(|(w, p)| w * p) + .sum::(); + let posterior = [ + current[0] * likelihood[0] / mix_prob, + current[1] * likelihood[1] / mix_prob, + current[2] * likelihood[2] / mix_prob, + ]; + let prior = [0.5_f64, 0.3, 0.2]; + let mut expected = [0.0_f64; 3]; + for j in 0..3 { + let stay = (1.0 - alpha) * posterior[j]; + let switch_in = alpha + * prior[j] + * (0..3) + .filter(|&k| k != j) + .map(|k| posterior[k] / (1.0 - prior[k])) + .sum::(); + expected[j] = stay + switch_in; + } + + for i in 0..3 { + assert!( + (post[i] - expected[i]).abs() < 1e-12, + "expert {i}: expected {} got {}", + expected[i], + post[i] + ); + } + } + + #[test] + fn switching_theorem_schedule_uses_one_over_t() { + assert!( + (switching_alpha_for_update(MixtureScheduleMode::Theorem, 0.99, 0) - 0.5).abs() < 1e-12 + ); + assert!( + (switching_alpha_for_update(MixtureScheduleMode::Theorem, 0.99, 1) - (1.0 / 3.0)).abs() + < 1e-12 + ); + + let configs = vec![weighted_cfg("a", 0.5, 0.8), weighted_cfg("b", 0.5, 0.3)]; + let mut mix = SwitchingMixture::new(&configs, 0.99, MixtureScheduleMode::Theorem); + let _ = mix.step(0); + let post = mix.posterior(); + let posterior_a = 0.5 * 0.8 / 0.55; + let posterior_b = 0.5 * 0.3 / 0.55; + let expected_a = 0.5 * posterior_a + 0.5 * posterior_b; + let expected_b = expected_a; + assert!((post[0] - expected_a).abs() < 1e-12); + assert!((post[1] - expected_b).abs() < 1e-12); + } + + #[test] + fn convex_theorem_schedule_uses_paper_step_size() { + let eta = convex_step_size_for_update(MixtureScheduleMode::Theorem, 9.0, 1); + assert!((eta - DEFAULT_MIN_PROB).abs() < 1e-18); + + let configs = vec![weighted_cfg("a", 0.5, 0.8), weighted_cfg("b", 0.5, 0.3)]; + let mut mix = ConvexMixture::new(&configs, 9.0, MixtureScheduleMode::Theorem); + let observed = mix.step(0).exp(); + assert!((observed - 0.55).abs() < 1e-12, "observed={observed}"); + + let expected = [ + 0.5 + eta * ((0.8 / 0.55) - 1.0), + 0.5 + eta * ((0.3 / 0.55) - 1.0), + ]; + assert!((mix.lambda[0] - expected[0]).abs() < 1e-12); + assert!((mix.lambda[1] - expected[1]).abs() < 1e-12); + } + + #[test] + fn mixture_begin_fresh_stream_resets_wrapper_state_to_priors() { + let configs = vec![weighted_cfg("a", 0.7, 0.9), weighted_cfg("b", 0.3, 0.2)]; + let prior = [0.7_f64, 0.3_f64]; + + let mut bayes = BayesMixture::new(&configs); + let _ = bayes.step(0); + assert!(bayes.posterior()[0] > prior[0]); + bayes.begin_fresh_stream(Some(1)).expect("bayes fresh"); + assert_weights_close(&bayes.posterior(), &prior, "bayes posterior"); + assert_expert_losses_reset(&bayes.experts, "bayes"); + assert_eq!(bayes.total_log_loss(), 0.0); + + let mut fading = FadingBayesMixture::new(&configs, 0.8); + let _ = fading.step(0); + assert!(fading.posterior()[0] > prior[0]); + fading.begin_fresh_stream(Some(1)).expect("fading fresh"); + assert_weights_close(&fading.posterior(), &prior, "fading posterior"); + assert_expert_losses_reset(&fading.experts, "fading"); + assert_eq!(fading.total_log_loss(), 0.0); + + let mut switching = SwitchingMixture::new(&configs, 0.15, MixtureScheduleMode::Default); + let _ = switching.step(0); + assert!(switching.posterior()[0] > prior[0]); + switching + .begin_fresh_stream(Some(1)) + .expect("switching fresh"); + assert_weights_close(&switching.posterior(), &prior, "switching posterior"); + assert_expert_losses_reset(&switching.experts, "switching"); + assert_eq!(switching.update_count, 0); + assert_eq!(switching.total_log_loss(), 0.0); + + let mut convex = ConvexMixture::new(&configs, 0.2, MixtureScheduleMode::Default); + let _ = convex.step(0); + assert!(convex.lambda[0] > prior[0]); + convex.begin_fresh_stream(Some(1)).expect("convex fresh"); + assert_weights_close(&convex.lambda, &prior, "convex lambda"); + assert_expert_losses_reset(&convex.experts, "convex"); + assert_eq!(convex.update_count, 0); + assert_eq!(convex.total_log_loss, 0.0); + + let mut mdl = MdlSelector::new(&configs); + let _ = mdl.step(0); + assert!(mdl.experts.iter().any(|expert| expert.cum_log_loss > 0.0)); + mdl.begin_fresh_stream(Some(1)).expect("mdl fresh"); + assert_expert_losses_reset(&mdl.experts, "mdl"); + assert_eq!(mdl.best_index(), 0); + assert_eq!(mdl.total_log_loss(), 0.0); + + let mut neural = NeuralMixture::new(&configs, 0.05); + let mut fresh_neural = NeuralMixture::new(&configs, 0.05); + let _ = neural.step(0); + neural.begin_fresh_stream(Some(1)).expect("neural fresh"); + let reset_logp = neural.predict_log_prob(0); + let fresh_logp = fresh_neural.predict_log_prob(0); + assert!( + (reset_logp - fresh_logp).abs() < 1e-12, + "neural wrapper state should match a fresh wrapper after restart" + ); + assert_expert_losses_reset(&neural.experts, "neural"); + assert_eq!(neural.total_log_loss(), 0.0); + } + + #[test] + fn mdl_predict_then_step_reuses_best_expert_log_prob() { + let c0 = Arc::new(AtomicUsize::new(0)); + let c1 = Arc::new(AtomicUsize::new(0)); + let mut mdl = MdlSelector::new(&[ + counting_cfg("c0", c0.clone()), + counting_cfg("c1", c1.clone()), + ]); + let _ = mdl.predict_log_prob(0); + let after_predict = c0.load(Ordering::Relaxed) + c1.load(Ordering::Relaxed); + assert_eq!(after_predict, 1); + let _ = mdl.step(0); + let after_step = c0.load(Ordering::Relaxed) + c1.load(Ordering::Relaxed); + assert_eq!(after_step, 2); + } + + #[test] + fn neural_mixture_adapts_to_correct_symbol() { + let configs = vec![ + ExpertConfig::uniform("zero", || Box::new(AlwaysPredict { byte: 0 })), + ExpertConfig::uniform("one", || Box::new(AlwaysPredict { byte: 1 })), + ]; + let mut mix = NeuralMixture::new(&configs, 0.05); + + let mut early = 0.0; + let mut late = 0.0; + for t in 0..200 { + let lp = mix.step(0); + if t < 20 { + early -= lp; + } + if t >= 180 { + late -= lp; + } + } + + let early_avg = early / 20.0; + let late_avg = late / 20.0; + assert!( + late_avg < early_avg, + "late_avg={late_avg} early_avg={early_avg}" + ); + assert!(late_avg < 0.35, "late_avg={late_avg}"); + } + + #[derive(Clone)] + struct CountingPredict { + calls: Arc, + } + + impl OnlineBytePredictor for CountingPredict { + fn log_prob(&mut self, symbol: u8) -> f64 { + self.calls.fetch_add(1, Ordering::Relaxed); + if symbol == 0 { 0.0 } else { -20.0 } + } + + fn update(&mut self, _symbol: u8) {} + } + + #[derive(Clone)] + struct CountingFillPredict { + log_calls: Arc, + fill_calls: Arc, + } + + impl OnlineBytePredictor for CountingFillPredict { + fn log_prob(&mut self, symbol: u8) -> f64 { + self.log_calls.fetch_add(1, Ordering::Relaxed); + if symbol == 0 { 0.0 } else { -20.0 } + } + + fn fill_log_probs(&mut self, out: &mut [f64; 256]) { + self.fill_calls.fetch_add(1, Ordering::Relaxed); + out.fill(-20.0); + out[0] = 0.0; + } + + fn update(&mut self, _symbol: u8) {} + } + + #[derive(Clone)] + struct BeginAwarePredict { + seen_total: Arc, + began: bool, + } + + impl OnlineBytePredictor for BeginAwarePredict { + fn begin_stream(&mut self, total_symbols: Option) -> Result<(), String> { + let total = total_symbols.ok_or_else(|| "missing total symbols".to_string())?; + self.seen_total.store(total, Ordering::Relaxed); + self.began = true; + Ok(()) + } + + fn log_prob(&mut self, _symbol: u8) -> f64 { + if self.began { 0.0 } else { f64::NEG_INFINITY } + } + + fn update(&mut self, _symbol: u8) {} + } + + #[derive(Clone)] + struct StatePreservingFreshPredict { + learned: usize, + began: bool, + } + + impl OnlineBytePredictor for StatePreservingFreshPredict { + fn begin_stream(&mut self, _total_symbols: Option) -> Result<(), String> { + self.began = true; + Ok(()) + } + + fn reset_frozen(&mut self, _total_symbols: Option) -> Result<(), String> { + self.began = true; + Ok(()) + } + + fn log_prob(&mut self, symbol: u8) -> f64 { + if self.began && self.learned > 0 && symbol == b'K' { + 0.0 + } else { + -12.0 + } + } + + fn update(&mut self, symbol: u8) { + if symbol == b'K' { + self.learned += 1; + } + } + } + + #[derive(Clone)] + struct FailingNonResettableFreshPredict { + learned: usize, + begin_calls: Arc, + } + + impl OnlineBytePredictor for FailingNonResettableFreshPredict { + fn supports_frozen_reset(&self) -> bool { + false + } + + fn begin_stream(&mut self, total_symbols: Option) -> Result<(), String> { + self.begin_calls.fetch_add(1, Ordering::Relaxed); + total_symbols + .map(|_| ()) + .ok_or_else(|| "missing total symbols".to_string()) + } + + fn log_prob(&mut self, symbol: u8) -> f64 { + if self.learned > 0 && symbol == b'Q' { + 0.0 + } else { + -15.0 + } + } + + fn update(&mut self, symbol: u8) { + if symbol == b'Q' { + self.learned += 1; + } + } + } + + #[derive(Clone)] + struct ResettingNonResettableFreshPredict { + learned: usize, + } + + impl OnlineBytePredictor for ResettingNonResettableFreshPredict { + fn supports_frozen_reset(&self) -> bool { + false + } + + fn begin_stream(&mut self, _total_symbols: Option) -> Result<(), String> { + self.learned = 0; + Ok(()) + } + + fn log_prob(&mut self, symbol: u8) -> f64 { + if self.learned > 0 && symbol == b'R' { + 0.0 + } else { + -15.0 + } + } + + fn update(&mut self, symbol: u8) { + if symbol == b'R' { + self.learned += 1; + } + } + } + + fn assert_log_prob_update_matches_separate(label: &str, backend: RateBackend) { + let mut separate = RateBackendPredictor::from_backend(backend.clone(), DEFAULT_MIN_PROB); + let mut combined = RateBackendPredictor::from_backend(backend, DEFAULT_MIN_PROB); + let data = b"combined step check data"; + + for &b in data { + let logp_separate = separate.log_prob(b); + separate.update(b); + let logp_combined = combined.log_prob_update(b); + let diff = (logp_separate - logp_combined).abs(); + assert!( + diff <= 1e-12, + "[{label}] symbol={b} separate={logp_separate} combined={logp_combined} diff={diff}" + ); + + let mut sep_row = [0.0; 256]; + let mut combo_row = [0.0; 256]; + separate.fill_log_probs(&mut sep_row); + combined.fill_log_probs(&mut combo_row); + for i in 0..256 { + let diff = (sep_row[i] - combo_row[i]).abs(); + assert!( + diff <= 1e-12, + "row mismatch at {i}: {} vs {}", + sep_row[i], + combo_row[i] + ); + } + } + } + + fn assert_fill_matches_symbol_queries(label: &str, backend: RateBackend) { + let mut bulk = RateBackendPredictor::from_backend(backend.clone(), DEFAULT_MIN_PROB); + let mut queried = RateBackendPredictor::from_backend(backend, DEFAULT_MIN_PROB); + let data = b"continuation consistency prompt"; + + bulk.begin_stream(Some(data.len() as u64)) + .expect("bulk begin"); + queried + .begin_stream(Some(data.len() as u64)) + .expect("query begin"); + for &b in data { + bulk.update(b); + queried.update(b); + } + + let mut bulk_row = [0.0; 256]; + bulk.fill_log_probs(&mut bulk_row); + for (sym, &bulk_logp) in bulk_row.iter().enumerate() { + let queried_logp = queried.log_prob(sym as u8); + let diff = (bulk_logp - queried_logp).abs(); + assert!( + diff <= 1e-12, + "[{label}] sym={sym} bulk={bulk_logp} queried={queried_logp} diff={diff}" + ); + } + } + + fn assert_fill_matches_symbol_queries_after_frozen_conditioning( + label: &str, + backend: RateBackend, + ) { + let fit = b"If a frog is green, dogs are red.\nIf a toad is green, cats are red.\n"; + let condition = b"If a cat is red, toads are \n"; + let total = (fit.len() + condition.len()) as u64; + + let mut bulk = RateBackendPredictor::from_backend(backend.clone(), DEFAULT_MIN_PROB); + let mut queried = RateBackendPredictor::from_backend(backend, DEFAULT_MIN_PROB); + + bulk.begin_stream(Some(total)).expect("bulk begin"); + queried.begin_stream(Some(total)).expect("query begin"); + for &b in fit { + bulk.update(b); + queried.update(b); + } + bulk.reset_frozen(Some(condition.len() as u64)) + .expect("bulk reset frozen"); + queried + .reset_frozen(Some(condition.len() as u64)) + .expect("query reset frozen"); + for &b in condition { + bulk.update_frozen(b); + queried.update_frozen(b); + } + + let mut bulk_row = [0.0; 256]; + bulk.fill_log_probs(&mut bulk_row); + for (sym, &bulk_logp) in bulk_row.iter().enumerate() { + let queried_logp = queried.log_prob(sym as u8); + let diff = (bulk_logp - queried_logp).abs(); + assert!( + diff <= 1e-12, + "[{label}] frozen sym={sym} bulk={bulk_logp} queried={queried_logp} diff={diff}" + ); + } + } + + #[test] + fn predictor_log_prob_update_matches_separate_update_for_rosa_backend() { + assert_log_prob_update_matches_separate("rosa", RateBackend::RosaPlus { max_order: -1 }); + } + + #[test] + fn predictor_log_prob_update_matches_separate_update_for_ctw_backend() { + assert_log_prob_update_matches_separate("ctw", RateBackend::Ctw { depth: 6 }); + } + + #[test] + fn predictor_log_prob_update_matches_separate_update_for_fac_ctw_backend() { + assert_log_prob_update_matches_separate( + "fac-ctw", + RateBackend::FacCtw { + base_depth: 6, + num_percept_bits: 8, + encoding_bits: 8, + msb_first: None, + }, + ); + } + + #[test] + fn predictor_fill_matches_symbol_queries_for_rosa_backend() { + assert_fill_matches_symbol_queries("rosa", RateBackend::RosaPlus { max_order: -1 }); + } + + #[test] + fn predictor_fill_matches_symbol_queries_for_ctw_backend() { + assert_fill_matches_symbol_queries("ctw", RateBackend::Ctw { depth: 6 }); + } + + #[test] + fn predictor_fill_matches_symbol_queries_for_match_backend() { + assert_fill_matches_symbol_queries( + "match", + RateBackend::Match { + hash_bits: 18, + min_len: 4, + max_len: 64, + base_mix: 0.02, + confidence_scale: 1.0, + }, + ); + } + + #[test] + fn predictor_fill_matches_symbol_queries_for_ppmd_backend() { + assert_fill_matches_symbol_queries( + "ppmd", + RateBackend::Ppmd { + order: 8, + memory_mb: 8, + }, + ); + } + + #[cfg(feature = "backend-rwkv")] + #[test] + fn predictor_fill_matches_symbol_queries_for_rwkv_backend() { + assert_fill_matches_symbol_queries( + "rwkv7", + RateBackend::Rwkv7Method { + method: crate::rwkvzip::parse_method_spec("cfg:hidden=64,layers=1,intermediate=64,decay_rank=8,a_rank=8,v_rank=8,g_rank=8,seed=31,train=none,lr=0.0,stride=1;policy:schedule=0..100:infer").expect("rwkv method spec"), + }, + ); + } + + #[test] + fn predictor_fill_matches_symbol_queries_for_rosa_backend_after_frozen_conditioning() { + assert_fill_matches_symbol_queries_after_frozen_conditioning( + "rosa", + RateBackend::RosaPlus { max_order: -1 }, + ); + } + + #[test] + fn predictor_frozen_conditioning_reuses_match_fit_corpus() { + let mut predictor = RateBackendPredictor::from_backend( + RateBackend::Match { + hash_bits: 20, + min_len: 3, + max_len: 32, + base_mix: 0.02, + confidence_scale: 1.0, + }, + DEFAULT_MIN_PROB, + ); + + for &b in b"abcabcX" { + predictor.update(b); + } + predictor + .reset_frozen(Some(6)) + .expect("reset frozen for match backend"); + for &b in b"abcabc" { + predictor.update_frozen(b); + } + let p_x = predictor.log_prob(b'X').exp(); + assert!( + p_x > 0.01, + "frozen conditioning should preserve fit corpus for match backend; p_x={p_x}" + ); + } + + #[test] + fn predictor_frozen_conditioning_reuses_sparse_match_fit_corpus() { + let mut predictor = RateBackendPredictor::from_backend( + RateBackend::SparseMatch { + hash_bits: 20, + min_len: 3, + max_len: 32, + gap_min: 0, + gap_max: 2, + base_mix: 0.02, + confidence_scale: 1.0, + }, + DEFAULT_MIN_PROB, + ); + + for &b in b"abcabcX" { + predictor.update(b); + } + predictor + .reset_frozen(Some(6)) + .expect("reset frozen for sparse-match backend"); + for &b in b"abcabc" { + predictor.update_frozen(b); + } + let p_x = predictor.log_prob(b'X').exp(); + assert!( + p_x > 0.01, + "frozen conditioning should preserve fit corpus for sparse-match backend; p_x={p_x}" + ); + } + + #[test] + fn neural_predict_then_step_reuses_evaluation_cache() { + let c0 = Arc::new(AtomicUsize::new(0)); + let c1 = Arc::new(AtomicUsize::new(0)); + let cfg0 = { + let c = c0.clone(); + ExpertConfig::uniform("c0", move || Box::new(CountingPredict { calls: c.clone() })) + }; + let cfg1 = { + let c = c1.clone(); + ExpertConfig::uniform("c1", move || Box::new(CountingPredict { calls: c.clone() })) + }; + let mut mix = NeuralMixture::new(&[cfg0, cfg1], 0.03); + + let _ = mix.predict_log_prob(0); + let after_predict = c0.load(Ordering::Relaxed) + c1.load(Ordering::Relaxed); + assert_eq!(after_predict, 2); + + let _ = mix.step(0); + let after_step = c0.load(Ordering::Relaxed) + c1.load(Ordering::Relaxed); + assert_eq!(after_step, after_predict); + } + + #[test] + fn neural_predict_multiple_symbols_reuses_single_evaluation() { + let c0 = Arc::new(AtomicUsize::new(0)); + let c1 = Arc::new(AtomicUsize::new(0)); + let cfg0 = { + let c = c0.clone(); + ExpertConfig::uniform("c0", move || Box::new(CountingPredict { calls: c.clone() })) + }; + let cfg1 = { + let c = c1.clone(); + ExpertConfig::uniform("c1", move || Box::new(CountingPredict { calls: c.clone() })) + }; + let mut mix = NeuralMixture::new(&[cfg0, cfg1], 0.03); + + let _ = mix.predict_log_prob(0); + let after_first = c0.load(Ordering::Relaxed) + c1.load(Ordering::Relaxed); + assert_eq!(after_first, 2); + + let _ = mix.predict_log_prob(1); + let after_second = c0.load(Ordering::Relaxed) + c1.load(Ordering::Relaxed); + assert_eq!(after_second, after_first + 2); + } + + #[test] + fn neural_fill_then_step_reuses_cached_full_rows() { + let log0 = Arc::new(AtomicUsize::new(0)); + let log1 = Arc::new(AtomicUsize::new(0)); + let fill0 = Arc::new(AtomicUsize::new(0)); + let fill1 = Arc::new(AtomicUsize::new(0)); + let cfg0 = { + let log_calls = log0.clone(); + let fill_calls = fill0.clone(); + ExpertConfig::uniform("c0", move || { + Box::new(CountingFillPredict { + log_calls: log_calls.clone(), + fill_calls: fill_calls.clone(), + }) + }) + }; + let cfg1 = { + let log_calls = log1.clone(); + let fill_calls = fill1.clone(); + ExpertConfig::uniform("c1", move || { + Box::new(CountingFillPredict { + log_calls: log_calls.clone(), + fill_calls: fill_calls.clone(), + }) + }) + }; + let mut mix = NeuralMixture::new(&[cfg0, cfg1], 0.03); + + let mut row = [0.0; 256]; + mix.fill_log_probs(&mut row); + assert_eq!(fill0.load(Ordering::Relaxed), 1); + assert_eq!(fill1.load(Ordering::Relaxed), 1); + assert_eq!(log0.load(Ordering::Relaxed), 0); + assert_eq!(log1.load(Ordering::Relaxed), 0); + + let _ = mix.step(0); + assert_eq!(fill0.load(Ordering::Relaxed), 1); + assert_eq!(fill1.load(Ordering::Relaxed), 1); + assert_eq!(log0.load(Ordering::Relaxed), 0); + assert_eq!(log1.load(Ordering::Relaxed), 0); + } + + #[test] + fn runtime_begin_stream_propagates_to_experts() { + let seen_total = Arc::new(AtomicU64::new(0)); + let cfg = { + let seen_total = seen_total.clone(); + ExpertConfig::uniform("begin-aware", move || { + Box::new(BeginAwarePredict { + seen_total: seen_total.clone(), + began: false, + }) + }) + }; + + let spec = MixtureSpec::new( + MixtureKind::Bayes, + vec![crate::MixtureExpertSpec { + name: Some("begin-aware".to_string()), + log_prior: 0.0, + backend: RateBackend::Ctw { depth: 1 }, + }], + ); + let mut runtime = build_mixture_runtime(&spec, &[cfg]).expect("runtime"); + runtime.begin_stream(Some(123)).expect("begin stream"); + let _ = runtime.step(0); + assert_eq!(seen_total.load(Ordering::Relaxed), 123); + } + + #[test] + fn runtime_begin_fresh_stream_preserves_resettable_expert_state() { + let build_calls = Arc::new(AtomicUsize::new(0)); + let cfg = { + let build_calls = build_calls.clone(); + ExpertConfig::uniform("state-preserving", move || { + build_calls.fetch_add(1, Ordering::Relaxed); + Box::new(StatePreservingFreshPredict { + learned: 0, + began: false, + }) + }) + }; + + let spec = MixtureSpec::new( + MixtureKind::Bayes, + vec![crate::MixtureExpertSpec { + name: Some("state-preserving".to_string()), + log_prior: 0.0, + backend: RateBackend::Ctw { depth: 1 }, + }], + ); + let mut runtime = build_mixture_runtime(&spec, &[cfg]).expect("runtime"); + runtime.begin_stream(Some(1)).expect("begin stream"); + let _ = runtime.step(b'K'); + + runtime + .begin_fresh_stream(Some(1)) + .expect("fresh stream restart"); + + assert_eq!( + build_calls.load(Ordering::Relaxed), + 1, + "resettable experts must not be rebuilt for fresh stream restarts" + ); + let logp = runtime.peek_log_prob(b'K'); + assert!( + logp > -1.0, + "fresh stream restart should preserve fitted expert state; logp={logp}" + ); + } + + #[test] + fn runtime_begin_fresh_stream_failure_preserves_existing_expert() { + let build_calls = Arc::new(AtomicUsize::new(0)); + let begin_calls = Arc::new(AtomicUsize::new(0)); + let cfg = { + let build_calls = build_calls.clone(); + let begin_calls = begin_calls.clone(); + ExpertConfig::uniform("non-resettable", move || { + build_calls.fetch_add(1, Ordering::Relaxed); + Box::new(FailingNonResettableFreshPredict { + learned: 0, + begin_calls: begin_calls.clone(), + }) + }) + }; + + let spec = MixtureSpec::new( + MixtureKind::Bayes, + vec![crate::MixtureExpertSpec { + name: Some("non-resettable".to_string()), + log_prior: 0.0, + backend: RateBackend::Ctw { depth: 1 }, + }], + ); + let mut runtime = build_mixture_runtime(&spec, &[cfg]).expect("runtime"); + runtime.begin_stream(Some(1)).expect("begin stream"); + let _ = runtime.step(b'Q'); + + let err = runtime + .begin_fresh_stream(None) + .expect_err("fresh stream restart should report the expert begin_stream failure"); + + assert!(err.contains("missing total symbols")); + assert_eq!( + build_calls.load(Ordering::Relaxed), + 1, + "failed fresh stream restarts must not rebuild or discard the old expert" + ); + assert_eq!(begin_calls.load(Ordering::Relaxed), 2); + let logp = runtime.peek_log_prob(b'Q'); + assert!( + logp > -1.0, + "old expert should remain usable after failed fresh restart; logp={logp}" + ); + } + + #[test] + fn runtime_begin_fresh_stream_failure_is_transactional_across_experts() { + let failing_begin_calls = Arc::new(AtomicUsize::new(0)); + let resettable_cfg = ExpertConfig::uniform("resetting", || { + Box::new(ResettingNonResettableFreshPredict { learned: 0 }) + }); + let failing_cfg = { + let failing_begin_calls = failing_begin_calls.clone(); + ExpertConfig::uniform("failing", move || { + Box::new(FailingNonResettableFreshPredict { + learned: 0, + begin_calls: failing_begin_calls.clone(), + }) + }) + }; + + let spec = MixtureSpec::new( + MixtureKind::Bayes, + vec![ + crate::MixtureExpertSpec { + name: Some("resetting".to_string()), + log_prior: 0.0, + backend: RateBackend::Ctw { depth: 1 }, + }, + crate::MixtureExpertSpec { + name: Some("failing".to_string()), + log_prior: 0.0, + backend: RateBackend::Ctw { depth: 1 }, + }, + ], + ); + let mut runtime = + build_mixture_runtime(&spec, &[resettable_cfg, failing_cfg]).expect("runtime"); + runtime.begin_stream(Some(1)).expect("begin stream"); + let _ = runtime.step(b'R'); + let logp_before = runtime.peek_log_prob(b'R'); + assert!( + logp_before > -1.0, + "resetting expert should have learned prior to restart; logp={logp_before}" + ); + + let err = runtime + .begin_fresh_stream(None) + .expect_err("second expert should fail begin_fresh_stream"); + assert!(err.contains("missing total symbols")); + assert_eq!( + failing_begin_calls.load(Ordering::Relaxed), + 2, + "failing expert begin should run once at initial begin and once at failed restart" + ); + + let logp_after = runtime.peek_log_prob(b'R'); + assert!( + logp_after > -1.0, + "failed fresh restart must restore earlier experts instead of leaving partial mutation; logp={logp_after}" + ); + } + + fn ctw_checkpoint_depth(predictor: &RateBackendPredictor) -> usize { + match predictor { + #[cfg(feature = "backend-ctw")] + RateBackendPredictor::Ctw { + checkpoint_depth, .. + } => *checkpoint_depth, + _ => panic!("expected ctw predictor"), + } + } + + fn predictor_log_probs(predictor: &mut RateBackendPredictor) -> [f64; 256] { + let mut row = [0.0f64; 256]; + predictor.fill_log_probs(&mut row); + row + } + + fn assert_log_prob_rows_close(actual: &[f64; 256], expected: &[f64; 256], label: &str) { + for (symbol, (&actual, &expected)) in actual.iter().zip(expected.iter()).enumerate() { + assert!( + (actual - expected).abs() < 1e-12, + "{label}[{symbol}]: expected {expected}, got {actual}" + ); + } + } + + #[test] + fn lifecycle_reset_rejects_active_ctw_prediction_checkpoint() { + let mut predictor = + RateBackendPredictor::from_backend(RateBackend::Ctw { depth: 4 }, DEFAULT_MIN_PROB); + for &symbol in b"abracadabra ctw checkpoint guard" { + predictor.update(symbol); + } + let mut baseline = predictor.clone(); + let checkpoint = predictor.checkpoint(); + + let err = predictor + .begin_fresh_stream(Some(0)) + .expect_err("ctw lifecycle reset must reject active compact prediction checkpoints"); + + assert!(err.contains("prediction checkpoints")); + assert_eq!(ctw_checkpoint_depth(&predictor), 1); + let expected = predictor_log_probs(&mut baseline); + let actual = predictor_log_probs(&mut predictor); + assert_log_prob_rows_close(&actual, &expected, "ctw active-checkpoint lifecycle reject"); + predictor.discard_checkpoint(checkpoint); + } + + #[test] + fn ctw_lifecycle_with_active_prefix_uses_full_clone() { + let mut predictor = + RateBackendPredictor::from_backend(RateBackend::Ctw { depth: 4 }, DEFAULT_MIN_PROB); + assert!( + predictor + .begin_native_msb_byte_prefix() + .expect("begin native prefix") + ); + predictor + .observe_native_msb_prefix_bit(0, true) + .expect("observe one prefix bit"); + + let checkpoint = + predictor.lifecycle_checkpoint(OnlineBytePredictorLifecycleOp::FinishStream); + + assert!( + matches!(checkpoint, RateBackendPredictorLifecycleCheckpoint::Full(_)), + "active native prefix with observed bits must not use compact lifecycle rollback" + ); + } + + #[test] + fn nested_mixture_lifecycle_failure_restores_wrapper_state() { + let inner_cfgs = vec![ + weighted_cfg("zero-heavy", 1.0, 0.90), + weighted_cfg("zero-light", 1.0, 0.10), + ]; + let mut inner = MixtureRuntime::Bayes(BayesMixture::new(&inner_cfgs)); + inner.begin_stream(Some(1)).expect("inner begin"); + let _ = inner.step(0); + let before = inner.peek_log_prob(0); + + let failing_begin_calls = Arc::new(AtomicUsize::new(0)); + let mut experts = vec![ + ExpertState { + name: "nested".to_string(), + log_weight: 0.0, + log_prior: 0.0, + predictor: ExpertPredictor::rate_backend(RateBackendPredictor::Mixture { + runtime: inner, + }), + cum_log_loss: 0.0, + }, + ExpertState { + name: "failing".to_string(), + log_weight: 0.0, + log_prior: 0.0, + predictor: ExpertPredictor::generic(Box::new(FailingNonResettableFreshPredict { + learned: 0, + begin_calls: failing_begin_calls.clone(), + })), + cum_log_loss: 0.0, + }, + ]; + + let err = begin_expert_fresh_stream(&mut experts, None) + .expect_err("outer lifecycle should fail on second expert"); + + assert!(err.contains("missing total symbols")); + let after = experts[0].log_prob(0); + assert!( + (after - before).abs() < 1e-12, + "nested mixture wrapper state should be restored: before={before}, after={after}" + ); + } + + #[test] + fn zpaq_fill_log_probs_does_not_drift_history() { + let backend = RateBackend::Zpaq { + method: crate::api::ZpaqMethodSpec::literal("1"), + }; + let mut baseline = RateBackendPredictor::from_backend(backend.clone(), DEFAULT_MIN_PROB); + let mut probe = RateBackendPredictor::from_backend(backend, DEFAULT_MIN_PROB); + + let history = b"history for zpaq predictor"; + for &b in history { + baseline.update(b); + probe.update(b); + } + + let mut row = [0.0f64; 256]; + probe.fill_log_probs(&mut row); + + let sym = b'k'; + let lp_base = baseline.log_prob(sym); + let lp_probe = probe.log_prob(sym); + assert!((lp_base - lp_probe).abs() < 1e-9); + assert!((row[sym as usize] - lp_base).abs() < 1e-9); + + baseline.update(sym); + probe.update(sym); + let next = b'q'; + let next_base = baseline.log_prob(next); + let next_probe = probe.log_prob(next); + assert!((next_base - next_probe).abs() < 1e-9); + } + + fn assert_checkpoint_roundtrip_restores_predictor(backend: RateBackend, history: &[u8]) { + let mut predictor = RateBackendPredictor::from_backend(backend.clone(), DEFAULT_MIN_PROB); + predictor.begin_stream(None).expect("begin stream"); + for &byte in history { + predictor.update(byte); + } + + let checkpoint = predictor.checkpoint(); + let mut baseline = predictor.clone(); + + for &byte in b"speculative branch" { + predictor.update(byte); + } + for &byte in b"frozen branch" { + predictor.update_frozen(byte); + } + for &byte in b"tail" { + predictor.update(byte); + } + + predictor.restore_checkpoint(&checkpoint); + predictor.clear_checkpoints_if_supported(); + + let mut baseline_row = [0.0; 256]; + let mut restored_row = [0.0; 256]; + baseline.fill_log_probs(&mut baseline_row); + predictor.fill_log_probs(&mut restored_row); + for (expected, actual) in baseline_row.iter().zip(restored_row.iter()) { + assert!( + (expected - actual).abs() < 1e-12, + "checkpoint restore drifted predictor state: expected={expected}, actual={actual}" + ); + } + } + + #[test] + fn rosa_checkpoint_restores_exact_predictor_state() { + assert_checkpoint_roundtrip_restores_predictor( + RateBackend::RosaPlus { max_order: -1 }, + b"rosa checkpoint base history", + ); + } + + #[test] + fn rosa_checkpoint_uses_compact_journal_marker() { + let mut predictor = RateBackendPredictor::from_backend( + RateBackend::RosaPlus { max_order: -1 }, + DEFAULT_MIN_PROB, + ); + match predictor.checkpoint() { + RateBackendPredictorCheckpoint::Rosa { journal_len } => { + assert_eq!(journal_len, 0); + } + _ => panic!("expected compact rosa checkpoint"), + } + predictor.clear_checkpoints_if_supported(); + } + + #[test] + fn ctw_checkpoint_restores_mixed_learned_and_frozen_updates() { + assert_checkpoint_roundtrip_restores_predictor( + RateBackend::Ctw { depth: 8 }, + b"ctw checkpoint base history", + ); + } + + #[test] + fn fac_ctw_checkpoint_restores_mixed_learned_and_frozen_updates() { + assert_checkpoint_roundtrip_restores_predictor( + RateBackend::FacCtw { + base_depth: 7, + num_percept_bits: 6, + encoding_bits: 6, + msb_first: None, + }, + b"fac-ctw checkpoint base history", + ); + } + + #[test] + fn ppmd_checkpoint_restores_mixed_learned_and_frozen_updates() { + assert_checkpoint_roundtrip_restores_predictor( + RateBackend::Ppmd { + order: 8, + memory_mb: 8, + }, + b"ppmd checkpoint base history", + ); + } + + #[test] + fn calibrated_checkpoint_restores_wrapped_predictor_and_calibration_state() { + assert_checkpoint_roundtrip_restores_predictor( + RateBackend::Calibrated { + spec: Arc::new(CalibratedSpec { + base: RateBackend::Ctw { depth: 6 }, + context: crate::CalibrationContextKind::Text, + bins: 33, + learning_rate: 0.02, + bias_clip: 4.0, + }), + }, + b"calibrated checkpoint base history", + ); + } + + fn assert_predictor_log_probs_normalize_to_one(backend: RateBackend) { + let mut predictor = RateBackendPredictor::from_backend(backend, DEFAULT_MIN_PROB); + for &b in b"normalization corpus for ctw/fac predictor checks" { + predictor.update(b); + } + let mut sum = 0.0f64; + for sym in 0u8..=255u8 { + sum += predictor.log_prob(sym).exp(); + } + assert!( + (sum - 1.0).abs() <= 1e-10, + "probability mass drift: sum={sum}" + ); + } + + #[test] + fn ctw_predictor_symbol_probs_normalize() { + assert_predictor_log_probs_normalize_to_one(RateBackend::Ctw { depth: 7 }); + } + + #[test] + fn fac_ctw_predictor_symbol_probs_normalize() { + assert_predictor_log_probs_normalize_to_one(RateBackend::FacCtw { + base_depth: 7, + num_percept_bits: 8, + encoding_bits: 8, + msb_first: None, + }); + } + + #[test] + fn expert_config_helpers_expose_names_priors_and_predictor_builders() { + let cfg = ExpertConfig::from_rate_backend( + Some("ctw-four".to_string()), + -0.75, + RateBackend::Ctw { depth: 4 }, + ); + assert_eq!(cfg.name(), "ctw-four"); + assert!((cfg.log_prior() + 0.75).abs() < 1e-12); + let mut predictor = cfg.build_predictor(); + let logp = predictor.log_prob(b'a'); + assert!(logp.is_finite()); + + let uniform = ExpertConfig::uniform("always-zero", || Box::new(AlwaysPredict { byte: 0 })); + assert_eq!(uniform.name(), "always-zero"); + assert_eq!(uniform.log_prior(), 0.0); + + let zpaq = ExpertConfig::zpaq("zpaq-one", "1"); + assert_eq!(zpaq.name(), "zpaq-one"); + assert_eq!(zpaq.log_prior(), 0.0); + let mut zpaq_predictor = zpaq.build_predictor(); + assert!(zpaq_predictor.log_prob(b'b').is_finite()); + } + + fn assert_runtime_variant_contracts(spec: MixtureSpec, expected_names: &[&str], symbol: u8) { + let configs = vec![ + ExpertConfig::new(expected_names[0].to_string(), 0.75f64.ln(), move || { + Box::new(FixedProbPredict { prob_zero: 0.8 }) + }), + ExpertConfig::new(expected_names[1].to_string(), 0.25f64.ln(), move || { + Box::new(FixedProbPredict { prob_zero: 0.35 }) + }), + ]; + let mut runtime = build_mixture_runtime(&spec, &configs).expect("runtime should build"); + + runtime.begin_stream(Some(8)).expect("begin stream"); + let peek = runtime.peek_log_prob(symbol); + assert!(peek.is_finite()); + + let mut row = [f64::NEG_INFINITY; 256]; + runtime.fill_log_probs(&mut row); + let mass: f64 = row.iter().map(|lp| lp.exp()).sum(); + assert!( + (mass - 1.0).abs() < 1e-8, + "mixture runtime PDF must normalize; mass={mass}" + ); + + let stepped = runtime.step(symbol); + assert!(stepped.is_finite()); + runtime.update_frozen(symbol.wrapping_add(1)); + runtime.finish_stream().expect("finish stream"); + runtime.reset_frozen(Some(3)).expect("reset frozen"); + runtime + .begin_stream(Some(3)) + .expect("begin stream after reset"); + + match &mut runtime { + MixtureRuntime::Bayes(m) => { + assert_eq!(m.expert_names(), expected_names); + assert_eq!(m.expert_log_losses().len(), 2); + assert_eq!(m.total_log_loss(), 0.0); + let (_, posterior) = m.max_posterior(); + assert!((0.0..=1.0).contains(&posterior)); + } + MixtureRuntime::Fading(m) => { + assert_eq!(m.expert_names(), expected_names); + assert_eq!(m.total_log_loss(), 0.0); + let posterior_mass: f64 = m.posterior().into_iter().sum(); + assert!((posterior_mass - 1.0).abs() < 1e-10); + } + MixtureRuntime::Switching(m) => { + assert_eq!(m.expert_names(), expected_names); + assert_eq!(m.expert_log_losses().len(), 2); + assert_eq!(m.total_log_loss(), 0.0); + let (_, posterior) = m.max_posterior(); + assert!((0.0..=1.0).contains(&posterior)); + } + MixtureRuntime::Convex(m) => { + let weight_sum: f64 = m.lambda.iter().sum(); + assert!((weight_sum - 1.0).abs() < 1e-10); + assert_eq!(m.total_log_loss, 0.0); + } + MixtureRuntime::Mdl(m) => { + assert_eq!(m.expert_names(), expected_names); + assert_eq!(m.expert_log_losses().len(), 2); + assert_eq!(m.total_log_loss(), 0.0); + assert!(m.best_index() < 2); + } + MixtureRuntime::Neural(m) => { + assert_eq!(m.total_log_loss(), 0.0); + } + } + } + + #[test] + fn runtime_variants_support_stream_fill_and_reset_contracts() { + assert_runtime_variant_contracts( + MixtureSpec::new( + MixtureKind::Bayes, + vec![ + crate::MixtureExpertSpec::new(RateBackend::Ctw { depth: 4 }).with_name("left"), + crate::MixtureExpertSpec::new(RateBackend::Ctw { depth: 5 }).with_name("right"), + ], + ), + &["left", "right"], + 0, + ); + + assert_runtime_variant_contracts( + MixtureSpec::new( + MixtureKind::FadingBayes, + vec![ + crate::MixtureExpertSpec::new(RateBackend::Ctw { depth: 4 }) + .with_name("fade-a"), + crate::MixtureExpertSpec::new(RateBackend::Ctw { depth: 5 }) + .with_name("fade-b"), + ], + ) + .with_decay(0.93), + &["fade-a", "fade-b"], + 0, + ); + + assert_runtime_variant_contracts( + MixtureSpec::new( + MixtureKind::Switching, + vec![ + crate::MixtureExpertSpec::new(RateBackend::Ctw { depth: 4 }) + .with_name("switch-a"), + crate::MixtureExpertSpec::new(RateBackend::Ctw { depth: 5 }) + .with_name("switch-b"), + ], + ) + .with_schedule(MixtureScheduleMode::Theorem), + &["switch-a", "switch-b"], + 1, + ); + + assert_runtime_variant_contracts( + MixtureSpec::new( + MixtureKind::Convex, + vec![ + crate::MixtureExpertSpec::new(RateBackend::Ctw { depth: 4 }) + .with_name("convex-a"), + crate::MixtureExpertSpec::new(RateBackend::Ctw { depth: 5 }) + .with_name("convex-b"), + ], + ) + .with_schedule(MixtureScheduleMode::Theorem) + .with_alpha(1.25), + &["convex-a", "convex-b"], + 0, + ); + + assert_runtime_variant_contracts( + MixtureSpec::new( + MixtureKind::Mdl, + vec![ + crate::MixtureExpertSpec::new(RateBackend::Ctw { depth: 4 }).with_name("mdl-a"), + crate::MixtureExpertSpec::new(RateBackend::Ctw { depth: 5 }).with_name("mdl-b"), + ], + ), + &["mdl-a", "mdl-b"], + 0, + ); + + assert_runtime_variant_contracts( + MixtureSpec::new( + MixtureKind::Neural, + vec![ + crate::MixtureExpertSpec::new(RateBackend::Ctw { depth: 4 }) + .with_name("neural-a"), + crate::MixtureExpertSpec::new(RateBackend::Ctw { depth: 5 }) + .with_name("neural-b"), + ], + ) + .with_alpha(0.04), + &["neural-a", "neural-b"], + 0, + ); + } + + #[cfg(feature = "backend-mixture")] + #[test] + fn compiled_mixture_helpers_roundtrip_expert_configs_and_runtime() { + let spec = MixtureSpec::new( + MixtureKind::Bayes, + vec![ + crate::MixtureExpertSpec::new(RateBackend::Ctw { depth: 3 }) + .with_name("compiled-ctw") + .with_log_prior(-0.5), + crate::MixtureExpertSpec::new(RateBackend::RosaPlus { max_order: 7 }) + .with_name("compiled-rosa") + .with_log_prior(-1.25), + ], + ); + let compiled = RateBackend::Mixture { + spec: Arc::new(spec.clone()), + } + .compile() + .expect("mixture backend should compile"); + + let configs = + expert_configs_from_compiled_mixture(&compiled).expect("compiled mixture configs"); + assert_eq!(configs.len(), 2); + assert_eq!(configs[0].name(), "compiled-ctw"); + assert_eq!(configs[1].name(), "compiled-rosa"); + assert!((configs[0].log_prior() + 0.5).abs() < 1e-12); + assert!((configs[1].log_prior() + 1.25).abs() < 1e-12); + assert!( + configs + .iter() + .all(|cfg| cfg.build_predictor().log_prob(b'x').is_finite()) + ); + + let mut runtime = build_mixture_runtime_from_compiled(&compiled, &configs) + .expect("compiled mixture runtime should build"); + runtime.begin_stream(Some(4)).expect("begin stream"); + assert!(runtime.peek_log_prob(0).is_finite()); + let mut row = [f64::NEG_INFINITY; 256]; + runtime.fill_log_probs(&mut row); + let mass: f64 = row.iter().map(|lp| lp.exp()).sum(); + assert!((mass - 1.0).abs() < 1e-8); + } + + #[test] + fn binary_token_mixture_builder_preserves_ctw_family_identity() { + let ctw_backend = RateBackend::Ctw { depth: 4 } + .compile() + .expect("compiled ctw backend"); + let mixture_backend = RateBackend::Mixture { + spec: Arc::new(MixtureSpec::new( + MixtureKind::Bayes, + vec![crate::MixtureExpertSpec::new(RateBackend::Ctw { depth: 4 })], + )), + } + .compile() + .expect("compiled mixture backend"); + + let mut direct = crate::runtime::build_rate_backend_binary_token_predictor( + &ctw_backend, + DEFAULT_MIN_PROB, + ) + .expect("direct ctw bit predictor"); + let mut mixture = crate::runtime::build_rate_backend_binary_token_predictor( + &mixture_backend, + DEFAULT_MIN_PROB, + ) + .expect("mixture bit predictor"); + + direct.begin_stream(Some(9)).expect("begin direct stream"); + mixture.begin_stream(Some(9)).expect("begin mixture stream"); + + for bit in [true, false, true, true, false, false, true, false, true] { + let direct_p0 = direct.log_prob(0); + let direct_p1 = direct.log_prob(1); + let mixture_p0 = mixture.log_prob(0); + let mixture_p1 = mixture.log_prob(1); + assert!((direct_p0 - mixture_p0).abs() < 1e-12); + assert!((direct_p1 - mixture_p1).abs() < 1e-12); + + direct.update(u8::from(bit)); + mixture.update(u8::from(bit)); + } + } + + #[test] + fn rate_backend_predictor_checkpoint_enum_stays_compact() { + let checkpoint_size: usize = std::mem::size_of::(); + assert!( + checkpoint_size < 128, + "RateBackendPredictorCheckpoint must stay pointer-sized after boxing Full; got {checkpoint_size}B" + ); + } +} diff --git a/src/neural_mix.rs b/crates/infotheory/src/neural_mix.rs similarity index 90% rename from src/neural_mix.rs rename to crates/infotheory/src/neural_mix.rs index 78df697b..e671941f 100644 --- a/src/neural_mix.rs +++ b/crates/infotheory/src/neural_mix.rs @@ -67,6 +67,36 @@ impl NeuralMixCore { } } + pub(crate) fn reset_to_priors(&mut self, prior_weights: &[f64]) { + debug_assert_eq!(prior_weights.len(), self.expert_count); + for table in &mut self.stage1_tables { + table.fill(0.0); + } + if self.expert_count > 0 + && let Some(global_table) = self.stage1_tables.first_mut() + { + for (dst, &p) in global_table[..self.expert_count] + .iter_mut() + .zip(prior_weights.iter()) + { + let p = if p.is_finite() { p.max(1e-12) } else { 1e-12 }; + *dst = p.ln(); + } + } + for row in &mut self.stage2_table { + row.fill(0.0); + } + self.context = NeuralContextState::default(); + self.expert_probs.fill(0.0); + self.stage1_mix.fill(0.0); + self.stage1_probs.fill(0.0); + self.stage2_mix.fill(0.0); + self.expert_weights.fill(0.0); + self.mix_prob = 1.0 / 256.0; + self.context_mixtures_valid = false; + self.evaluated = false; + } + #[inline] pub(crate) fn history_state(&self) -> NeuralHistoryState { self.context diff --git a/crates/infotheory/src/prediction.rs b/crates/infotheory/src/prediction.rs new file mode 100644 index 00000000..3757a9a4 --- /dev/null +++ b/crates/infotheory/src/prediction.rs @@ -0,0 +1,788 @@ +//! Shared online prediction abstractions for byte and bit consumers. +//! +//! The crate keeps byte prediction first-class while also exposing a bit-native +//! layer for consumers whose natural symbol is a bit. A byte model can be +//! queried as a bit model through a live prefix-mass view: the model remains a +//! byte model, but each bit query renormalizes over the surviving byte prefix. +//! +//! # Predictor Contract +//! +//! All `RateBackendPredictor` (and wrapper) implementations **must emit only +//! finite, non-negative** probabilities and log-probabilities. Callers (including +//! `BinaryPrediction` constructors, `BytePrefixMass`, mixture bit paths, and +//! entropy coders) may rely on this. Non-finite or negative outputs from a +//! predictor indicate an internal bug and are treated as contract violations +//! (surfaced via `panic!` with rich context in both debug and release builds for +//! infallible `BinaryPrediction` constructors and the `binary_prediction_from_*` +//! helpers). +//! Legitimate 0.5 / uniform policies remain only in the documented mathematical +//! cases below (measure-zero conditioning limits, not arithmetic corruption). +//! +//! Legitimate 0.5 / uniform policies (distinct from error masking): +//! - `BytePrefixMass::prediction` returns exact 0.5 when the current subtree +//! mass is zero (conditioning on a measure-zero event under the byte model; +//! the joint sequence prob is already zero; prevents NaN in coders). +//! - `from_raw_weights` (and thus public `from_pdf`/`from_log_probs`/`from_cdf`) +//! falls back to uniform 1/256 when the *input row* has zero or non-finite +//! total mass after sanitizing (construction-time robustness for invalid +//! caller-provided PDFs; `from_log_probs` documents the all-invalid case). + +/// Online byte-level predictor trait re-exported for prediction-oriented APIs. +pub use crate::mixture::OnlineBytePredictor; + +/// Bit ordering used when factorizing byte symbols. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +#[non_exhaustive] +pub enum BitOrder { + /// Most-significant bit first, matching Infotheory's AC bitwise fast path. + #[default] + MsbFirst, + /// Least-significant bit first. + LsbFirst, +} + +/// Semantic interpretation of a bit stream. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub enum BitStreamSemantics { + /// Bits are the fixed-width representation of byte symbols. + /// + /// This is a byte-native view: streams must begin and end on whole-byte + /// boundaries, and any provided total bit count must therefore be a + /// multiple of `8`. + BytePacked { + /// Bit ordering used when factorizing each byte symbol. + order: BitOrder, + }, + /// Bits are the actual modeled symbols, not a byte factorization. + /// + /// Native bit backends consume these symbols directly. Byte-native + /// backends instead adapt this view to the literal byte symbols `0` and + /// `1`, with probabilities renormalized over just those two outcomes. + /// + /// This therefore supports arbitrary non-multiple-of-8 lengths while + /// preserving truthful capability metadata: native bit support remains + /// distinguishable from byte-symbol adaptation. + BinaryTokens, +} + +impl Default for BitStreamSemantics { + fn default() -> Self { + // Generic bit sessions default to a byte-native view. Planner configs + // use their own binary-token default because AIXI/AIQI interfaces are + // commonly not byte-aligned. + Self::BytePacked { + order: BitOrder::MsbFirst, + } + } +} + +/// Probability pair for the next binary symbol. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct BinaryPrediction { + /// Probability of observing `0`. + pub p0: f64, + /// Probability of observing `1`. + pub p1: f64, +} + +impl BinaryPrediction { + /// Construct a normalized binary prediction from `P(1)` with a numerical floor. + /// + /// Finite values outside `[0, 1]` are clamped after applying the numerical + /// floor. Non-finite `floor` values are treated as the default floor. + /// + /// # Panics + /// + /// Panics when `p1` is not finite. This constructor is intended for + /// predictor outputs that have already satisfied the predictor contract; use + /// [`Self::checked_from_prob_one`] for caller-controlled input. + pub fn from_prob_one(p1: f64, floor: f64) -> Self { + Self::checked_from_prob_one(p1, floor).unwrap_or_else(|| { + panic!( + "RateBackendPredictor emitted non-finite p1 to BinaryPrediction::from_prob_one; \ + this is now a hard contract violation (predictors must emit only finite \ + non-negative values). See prediction.rs module docs and BinaryPrediction ctors." + ) + }) + } + + /// Checked variant of [`Self::from_prob_one`]. + /// + /// Returns `None` when `p1` is not finite. Finite values outside `[0, 1]` + /// are clamped using the same floor semantics as [`Self::from_prob_one`]. + pub fn checked_from_prob_one(p1: f64, floor: f64) -> Option { + if !p1.is_finite() { + return None; + } + let floor = binary_floor(floor); + let p1 = p1.clamp(floor, 1.0 - floor); + Some(Self { p0: 1.0 - p1, p1 }) + } + + /// Construct an exact normalized binary prediction from `P(1)`. + /// + /// This preserves hard support semantics: exact `0` and `1` probabilities + /// remain exact. Entropy coders should apply their own finite-count floor at + /// the coding boundary rather than here. + /// + /// # Panics + /// + /// Panics when `p1` is not finite. This constructor is intended for + /// predictor outputs that have already satisfied the predictor contract; use + /// [`Self::checked_from_prob_one_exact`] for caller-controlled input. + pub fn from_prob_one_exact(p1: f64) -> Self { + Self::checked_from_prob_one_exact(p1).unwrap_or_else(|| { + panic!( + "RateBackendPredictor emitted non-finite p1 to BinaryPrediction::from_prob_one_exact; \ + this is now a hard contract violation (predictors must emit only finite \ + non-negative values). See prediction.rs module docs and BinaryPrediction ctors." + ) + }) + } + + /// Checked variant of [`Self::from_prob_one_exact`]. + /// + /// Returns `None` when `p1` is not finite. Finite values outside `[0, 1]` + /// are clamped exactly as in [`Self::from_prob_one_exact`]. + pub fn checked_from_prob_one_exact(p1: f64) -> Option { + if !p1.is_finite() { + return None; + } + let p1 = p1.clamp(0.0, 1.0); + Some(Self { p0: 1.0 - p1, p1 }) + } + + /// Probability of `bit`. + #[inline] + pub fn prob(self, bit: bool) -> f64 { + if bit { self.p1 } else { self.p0 } + } +} + +/// Minimal bit predictor interface for true binary consumers. +pub trait OnlineBitPredictor { + /// Optional stream-start hook. + fn begin_bit_stream( + &mut self, + _total_bits: Option, + _semantics: BitStreamSemantics, + ) -> Result<(), String> { + Ok(()) + } + + /// Optional stream-finalization hook. + fn finish_bit_stream(&mut self) -> Result<(), String> { + Ok(()) + } + + /// Predict the next bit without updating state. + fn bit_prediction(&mut self) -> BinaryPrediction; + + /// Observe a bit while fitting/adapting. + fn update_bit(&mut self, bit: bool); + + /// Observe a bit as conditioning only. + fn update_bit_frozen(&mut self, bit: bool) { + self.update_bit(bit); + } +} + +/// Live byte-prefix state used to query a 256-way byte PDF as conditional bits. +#[derive(Clone, Debug)] +pub struct BytePrefixMass { + tree: [f64; 512], + node: usize, + order: BitOrder, + bits_seen: u8, + symbol: u8, +} + +const BYTE_PREFIX_TREE_ROOT: usize = 1; +const BYTE_PREFIX_TREE_LEAF_BASE: usize = 256; + +impl BytePrefixMass { + /// Build a prefix-mass state from a byte PDF. + /// + /// Slices shorter than 256 are treated as zero-padded on the right + /// (i.e. missing entries contribute 0 mass). This is an explicit + /// construction-time contract for the public API (graceful handling of + /// partial rows); see also the private `from_raw_weights` and module-level + /// docs on legitimate uniform fallbacks. + pub fn from_pdf(pdf: &[f64], order: BitOrder) -> Self { + let mut weights = [0.0f64; 256]; + for (idx, weight) in weights.iter_mut().enumerate() { + *weight = pdf.get(idx).copied().unwrap_or(0.0); + } + Self::from_raw_weights(weights, order) + } + + /// Build a prefix-mass state from byte log-probabilities or log-weights. + /// + /// Finite entries are exponentiated after subtracting the maximum finite + /// entry for numerical stability. Non-finite entries are treated as zero + /// mass, and an all-invalid row therefore falls back to the same uniform + /// distribution as [`Self::from_pdf`]. + /// + /// The input slice is truncated to at most 256 entries (excess ignored); + /// shorter slices are zero-padded (explicit contract, see [`Self::from_pdf`]). + pub fn from_log_probs(log_probs: &[f64], order: BitOrder) -> Self { + let log_probs = &log_probs[..log_probs.len().min(256)]; + let max_log = log_probs + .iter() + .copied() + .filter(|lp| lp.is_finite()) + .fold(f64::NEG_INFINITY, f64::max); + let mut weights = [0.0f64; 256]; + if max_log.is_finite() { + for (weight, &lp) in weights.iter_mut().zip(log_probs.iter()) { + *weight = if lp.is_finite() { + (lp - max_log).exp() + } else { + 0.0 + }; + } + } + Self::from_raw_weights(weights, order) + } + + /// Build a prefix-mass state from a normalized byte CDF row. + pub fn from_cdf(cdf: [f64; 257], order: BitOrder) -> Self { + let mut weights = [0.0f64; 256]; + for (idx, weight) in weights.iter_mut().enumerate() { + *weight = cdf[idx + 1] - cdf[idx]; + } + Self::from_raw_weights(weights, order) + } + + fn from_raw_weights(mut weights: [f64; 256], order: BitOrder) -> Self { + let mut total = 0.0f64; + for weight in &mut weights { + *weight = if weight.is_finite() && *weight > 0.0 { + *weight + } else { + 0.0 + }; + total += *weight; + } + if !total.is_finite() || total <= 0.0 { + weights.fill(1.0 / 256.0); + } else { + let inv = 1.0 / total; + for weight in &mut weights { + *weight *= inv; + } + } + Self::from_normalized_pdf(weights, order) + } + + fn from_normalized_pdf(pdf: [f64; 256], order: BitOrder) -> Self { + let mut tree = [0.0f64; 512]; + for (symbol, &mass) in pdf.iter().enumerate() { + let leaf = BYTE_PREFIX_TREE_LEAF_BASE + byte_prefix_leaf_offset(order, symbol as u8); + tree[leaf] = mass; + } + for node in (1..BYTE_PREFIX_TREE_LEAF_BASE).rev() { + tree[node] = tree[node * 2] + tree[node * 2 + 1]; + } + Self { + tree, + node: BYTE_PREFIX_TREE_ROOT, + order, + bits_seen: 0, + symbol: 0, + } + } + + /// Query the current conditional probability of the next bit. + pub fn prediction(&self) -> BinaryPrediction { + if self.is_complete() { + return BinaryPrediction::from_prob_one_exact(0.5); + } + let total = self.tree[self.node]; + if !total.is_finite() || total <= 0.0 { + // Legitimate policy: zero mass under the byte model means we are + // conditioning on a measure-zero event for the prefix. The joint + // probability of the observed sequence is already 0; returning the + // max-entropy distribution (0.5) prevents NaN propagation into + // arithmetic coders. + return BinaryPrediction::from_prob_one_exact(0.5); + } + let one = self.tree[self.node * 2 + 1]; + let p1 = one / total; + BinaryPrediction::from_prob_one_exact(p1) + } + + /// Observe a bit, discarding the impossible sibling branch. + pub fn observe(&mut self, bit: bool) { + debug_assert!( + !self.is_complete(), + "BytePrefixMass::observe called after a full byte was already observed" + ); + if self.is_complete() { + return; + } + self.node = self.node * 2 + usize::from(bit); + match self.order { + BitOrder::MsbFirst => { + self.symbol |= u8::from(bit) << (7 - self.bits_seen); + } + BitOrder::LsbFirst => { + self.symbol |= u8::from(bit) << self.bits_seen; + } + } + self.bits_seen = self.bits_seen.saturating_add(1); + } + + /// Whether a full byte has been observed. + #[inline] + pub fn is_complete(&self) -> bool { + self.bits_seen >= 8 + } + + /// Whether the current byte prefix has consumed at least one bit but has + /// not completed a full byte yet. + #[inline] + pub fn has_partial_bits(&self) -> bool { + self.bits_seen > 0 && !self.is_complete() + } + + /// Current completed symbol. Meaningful once [`Self::is_complete`] is true. + #[inline] + pub fn symbol(&self) -> u8 { + self.symbol + } +} + +#[inline] +fn byte_prefix_leaf_offset(order: BitOrder, symbol: u8) -> usize { + match order { + BitOrder::MsbFirst => usize::from(symbol), + BitOrder::LsbFirst => usize::from(symbol.reverse_bits()), + } +} + +#[inline] +fn binary_floor(floor: f64) -> f64 { + if floor.is_finite() { + floor.clamp(1e-12, 0.499_999_999_999) + } else { + 1e-12 + } +} + +/// Convert a pair of raw probabilities into a normalized [`BinaryPrediction`]. +/// +/// Inputs must be finite and non-negative (predictor contract). When both masses +/// are exactly zero the conditioning event has measure zero under the model; the +/// maximum-entropy extension `P(1)=0.5` is returned via [`BinaryPrediction::from_prob_one_exact`]. +#[inline] +pub(crate) fn binary_prediction_from_probs(p0: f64, p1: f64, floor: f64) -> BinaryPrediction { + assert!( + p0.is_finite() && p0 >= 0.0 && p1.is_finite() && p1 >= 0.0, + "RateBackendPredictor emitted invalid probability to binary_prediction_from_probs: p0={p0}, p1={p1}; \ + Predictor contract violation (must emit only finite non-negative values)" + ); + let sum: f64 = p0 + p1; + if sum > 0.0 { + BinaryPrediction::from_prob_one(p1 / sum, floor) + } else { + // Measure-zero conditioning limit: both symbol masses are exactly zero. + BinaryPrediction::from_prob_one_exact(0.5) + } +} + +/// Convert a pair of natural-log probabilities into a normalized [`BinaryPrediction`]. +/// +/// Uses a log-max shift before exponentiating to avoid catastrophic underflow +/// when both `logp0` and `logp1` are very negative (e.g. deep inside a long +/// conditioning context). Numerically, shifting by `max(logp0, logp1)` before +/// calling `exp` keeps the dominant term at `1.0` and the ratio exact. +/// +/// `floor` is forwarded to [`BinaryPrediction::from_prob_one`] and clamped to +/// `[1e-12, 0.5)` so that exact zero/one log-probs are softened at the coding +/// boundary rather than silently propagating infinities. +#[inline] +pub(crate) fn binary_prediction_from_log_probs( + logp0: f64, + logp1: f64, + floor: f64, +) -> BinaryPrediction { + if logp0.is_nan() || logp1.is_nan() { + panic!( + "RateBackendPredictor emitted NaN log probability to binary_prediction_from_log_probs: \ + logp0={logp0}, logp1={logp1}; contract violation" + ); + } + let max_log: f64 = logp0.max(logp1); + if max_log.is_infinite() { + if max_log == f64::NEG_INFINITY { + // Both log-probs are exactly -inf: measure-zero conditioning limit. + return BinaryPrediction::from_prob_one_exact(0.5); + } + panic!( + "RateBackendPredictor emitted +Inf log probability to binary_prediction_from_log_probs: \ + logp0={logp0}, logp1={logp1}; contract violation" + ); + } + // After the guards above, each logp is finite or exactly -inf; max_log is finite. + // IEEE 754: (-inf) - finite = -inf, and exp(-inf) = 0.0 — no explicit branch needed. + let p0: f64 = (logp0 - max_log).exp(); + let p1: f64 = (logp1 - max_log).exp(); + binary_prediction_from_probs(p0, p1, floor) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn normalize_pdf_for_test(mut pdf: [f64; 256]) -> [f64; 256] { + let sum: f64 = pdf.iter().sum(); + for p in &mut pdf { + *p /= sum; + } + pdf + } + + fn assert_binary_prediction_close(actual: BinaryPrediction, expected: BinaryPrediction) { + assert!((actual.p0 - expected.p0).abs() < 1e-12); + assert!((actual.p1 - expected.p1).abs() < 1e-12); + } + + #[test] + fn checked_binary_prediction_constructors_reject_non_finite_prob_one() { + for p1 in [f64::NAN, f64::INFINITY, f64::NEG_INFINITY] { + assert!(BinaryPrediction::checked_from_prob_one(p1, 0.0).is_none()); + assert!(BinaryPrediction::checked_from_prob_one_exact(p1).is_none()); + } + + assert_binary_prediction_close( + BinaryPrediction::checked_from_prob_one(1.2, 0.01).expect("finite p1 should construct"), + BinaryPrediction { + p0: 1.0 - 0.99, + p1: 0.99, + }, + ); + assert_eq!( + BinaryPrediction::checked_from_prob_one_exact(-0.5), + Some(BinaryPrediction { p0: 1.0, p1: 0.0 }) + ); + } + + fn bit_at(symbol: u8, order: BitOrder, bit_idx: u8) -> bool { + match order { + BitOrder::MsbFirst => ((symbol >> (7 - bit_idx)) & 1) == 1, + BitOrder::LsbFirst => ((symbol >> bit_idx) & 1) == 1, + } + } + + fn extend_observed_prefix(observed: &mut u8, order: BitOrder, bit_idx: u8, bit: bool) { + match order { + BitOrder::MsbFirst => *observed |= u8::from(bit) << (7 - bit_idx), + BitOrder::LsbFirst => *observed |= u8::from(bit) << bit_idx, + } + } + + fn prefix_matches(value: u8, observed: u8, order: BitOrder, bits_seen: u8) -> bool { + for bit_idx in 0..bits_seen { + if bit_at(value, order, bit_idx) != bit_at(observed, order, bit_idx) { + return false; + } + } + true + } + + fn direct_prediction( + pdf: &[f64; 256], + order: BitOrder, + observed: u8, + bits_seen: u8, + ) -> BinaryPrediction { + if bits_seen >= 8 { + return BinaryPrediction::from_prob_one_exact(0.5); + } + let mut p0 = 0.0f64; + let mut p1 = 0.0f64; + for (value, &mass) in pdf.iter().enumerate() { + let value = value as u8; + if !prefix_matches(value, observed, order, bits_seen) { + continue; + } + if bit_at(value, order, bits_seen) { + p1 += mass; + } else { + p0 += mass; + } + } + let total = p0 + p1; + let p1 = if total.is_finite() && total > 0.0 { + p1 / total + } else { + 0.5 + }; + BinaryPrediction::from_prob_one_exact(p1) + } + + fn cdf_from_pdf(pdf: &[f64; 256]) -> [f64; 257] { + let mut cdf = [0.0f64; 257]; + let mut acc = 0.0f64; + for idx in 0..256usize { + acc += pdf[idx]; + cdf[idx + 1] = acc; + } + cdf + } + + #[test] + fn byte_prefix_product_matches_symbol_probability_msb() { + let mut pdf = [0.0f64; 256]; + for (idx, slot) in pdf.iter_mut().enumerate() { + *slot = (idx + 1) as f64; + } + let pdf = normalize_pdf_for_test(pdf); + + let symbol = 0b1010_0110u8; + let mut prefix = BytePrefixMass::from_pdf(&pdf, BitOrder::MsbFirst); + let mut product = 1.0f64; + for bit_idx in 0..8u8 { + let bit = ((symbol >> (7 - bit_idx)) & 1) == 1; + let pred = prefix.prediction(); + product *= pred.prob(bit); + prefix.observe(bit); + } + assert!(prefix.is_complete()); + assert_eq!(prefix.symbol(), symbol); + assert!((product - pdf[symbol as usize]).abs() < 1e-12); + } + + #[test] + fn byte_prefix_product_matches_symbol_probability_lsb() { + let mut pdf = [0.0f64; 256]; + for (idx, slot) in pdf.iter_mut().enumerate() { + *slot = (idx + 3) as f64; + } + let pdf = normalize_pdf_for_test(pdf); + + let symbol = 0b1010_0110u8; + let mut prefix = BytePrefixMass::from_pdf(&pdf, BitOrder::LsbFirst); + let mut product = 1.0f64; + for bit_idx in 0..8u8 { + let bit = ((symbol >> bit_idx) & 1) == 1; + let pred = prefix.prediction(); + product *= pred.prob(bit); + prefix.observe(bit); + } + assert!(prefix.is_complete()); + assert_eq!(prefix.symbol(), symbol); + assert!((product - pdf[symbol as usize]).abs() < 1e-12); + } + + #[test] + fn byte_prefix_preserves_zero_mass_until_coder_boundary() { + let mut pdf = [0.0f64; 256]; + pdf[0b1010_0000] = 0.25; + pdf[0b1010_0001] = 0.75; + + let mut prefix = BytePrefixMass::from_pdf(&pdf, BitOrder::MsbFirst); + for bit_idx in 0..4u8 { + let bit = ((0b1010_0000u8 >> (7 - bit_idx)) & 1) == 1; + let prediction = prefix.prediction(); + assert_eq!(prediction.prob(bit), 1.0); + assert_eq!(prediction.prob(!bit), 0.0); + prefix.observe(bit); + } + + let impossible = prefix.prediction(); + assert_eq!(impossible.p1, 0.0); + assert_eq!(impossible.p0, 1.0); + } + + #[test] + fn byte_prefix_msb_prediction_preserves_tiny_positive_tail_mass() { + let eps = 5e-17; + let mut pdf = [eps; 256]; + pdf[0] = 1.0 - (255.0 * eps); + let pdf = normalize_pdf_for_test(pdf); + + let prediction = BytePrefixMass::from_pdf(&pdf, BitOrder::MsbFirst).prediction(); + let expected = pdf[128..256].iter().copied().sum::(); + assert!(expected > 0.0); + assert!(prediction.p1 > 0.0); + assert!((prediction.p1 - expected).abs() < 1e-18); + } + + #[test] + fn byte_prefix_lsb_prediction_preserves_tiny_positive_tail_mass() { + let eps = 5e-17; + let mut pdf = [eps; 256]; + pdf[0] = 1.0 - (255.0 * eps); + let pdf = normalize_pdf_for_test(pdf); + + let prediction = BytePrefixMass::from_pdf(&pdf, BitOrder::LsbFirst).prediction(); + let expected = pdf + .iter() + .enumerate() + .filter(|(idx, _)| (idx & 1) == 1) + .map(|(_, &p)| p) + .sum::(); + assert!(expected > 0.0); + assert!(prediction.p1 > 0.0); + assert!((prediction.p1 - expected).abs() < 1e-18); + } + + #[test] + fn byte_prefix_lsb_preserves_zero_mass_until_coder_boundary() { + let mut pdf = [0.0f64; 256]; + pdf[0b0000_1010] = 0.25; + pdf[0b1000_1010] = 0.75; + + let symbol = 0b0000_1010u8; + let mut prefix = BytePrefixMass::from_pdf(&pdf, BitOrder::LsbFirst); + for bit_idx in 0..7u8 { + let bit = ((symbol >> bit_idx) & 1) == 1; + let prediction = prefix.prediction(); + assert_eq!(prediction.prob(bit), 1.0); + assert_eq!(prediction.prob(!bit), 0.0); + prefix.observe(bit); + } + } + + #[test] + fn byte_prefix_prediction_matches_direct_reference_for_both_orders() { + let mut pdf = [0.0f64; 256]; + for (idx, slot) in pdf.iter_mut().enumerate() { + *slot = ((idx * 37 + 11) % 257 + 1) as f64; + } + let pdf = normalize_pdf_for_test(pdf); + + for order in [BitOrder::MsbFirst, BitOrder::LsbFirst] { + for symbol in 0u8..=255u8 { + let mut prefix = BytePrefixMass::from_pdf(&pdf, order); + let mut observed = 0u8; + for bit_idx in 0..8u8 { + let expected = direct_prediction(&pdf, order, observed, bit_idx); + let actual = prefix.prediction(); + assert_binary_prediction_close(actual, expected); + + let bit = bit_at(symbol, order, bit_idx); + prefix.observe(bit); + extend_observed_prefix(&mut observed, order, bit_idx, bit); + } + assert!(prefix.is_complete()); + assert_eq!(prefix.symbol(), symbol); + } + } + } + + #[test] + fn byte_prefix_from_cdf_matches_from_pdf_for_both_orders() { + let mut pdf = [0.0f64; 256]; + for (idx, slot) in pdf.iter_mut().enumerate() { + *slot = ((idx * 19 + 7) % 193 + 1) as f64; + } + let pdf = normalize_pdf_for_test(pdf); + let cdf = cdf_from_pdf(&pdf); + + for order in [BitOrder::MsbFirst, BitOrder::LsbFirst] { + let symbol = 0b1010_0110u8; + let mut from_pdf = BytePrefixMass::from_pdf(&pdf, order); + let mut from_cdf = BytePrefixMass::from_cdf(cdf, order); + for bit_idx in 0..8u8 { + assert_binary_prediction_close(from_pdf.prediction(), from_cdf.prediction()); + let bit = bit_at(symbol, order, bit_idx); + from_pdf.observe(bit); + from_cdf.observe(bit); + } + assert_eq!(from_pdf.symbol(), symbol); + assert_eq!(from_cdf.symbol(), symbol); + } + } + + #[test] + fn byte_prefix_from_log_probs_matches_from_pdf_for_both_orders() { + let mut pdf = [0.0f64; 256]; + for (idx, slot) in pdf.iter_mut().enumerate() { + *slot = ((idx * 23 + 5) % 211 + 1) as f64; + } + let pdf = normalize_pdf_for_test(pdf); + + let mut log_probs = [f64::NEG_INFINITY; 256]; + for (dst, &mass) in log_probs.iter_mut().zip(pdf.iter()) { + *dst = mass.ln() + 17.0; + } + + for order in [BitOrder::MsbFirst, BitOrder::LsbFirst] { + let symbol = 0b1010_0110u8; + let mut from_pdf = BytePrefixMass::from_pdf(&pdf, order); + let mut from_log_probs = BytePrefixMass::from_log_probs(&log_probs, order); + for bit_idx in 0..8u8 { + assert_binary_prediction_close(from_pdf.prediction(), from_log_probs.prediction()); + let bit = bit_at(symbol, order, bit_idx); + from_pdf.observe(bit); + from_log_probs.observe(bit); + } + assert_eq!(from_pdf.symbol(), symbol); + assert_eq!(from_log_probs.symbol(), symbol); + } + } + + #[test] + fn binary_prediction_from_probs_normalizes_and_floors() { + let pred = binary_prediction_from_probs(2.0, 6.0, 1e-6); + assert!((pred.p0 - 0.25).abs() < 1e-12); + assert!((pred.p1 - 0.75).abs() < 1e-12); + } + + #[test] + fn binary_prediction_from_probs_both_zero_returns_exact_half() { + let pred = binary_prediction_from_probs(0.0, 0.0, 1e-6); + assert!((pred.p1 - 0.5).abs() < 1e-12); + assert!((pred.p0 - 0.5).abs() < 1e-12); + } + + #[test] + fn binary_prediction_from_log_probs_both_neg_inf_returns_exact_half() { + let pred = binary_prediction_from_log_probs(f64::NEG_INFINITY, f64::NEG_INFINITY, 1e-6); + assert!((pred.p1 - 0.5).abs() < 1e-12); + } + + #[test] + #[should_panic(expected = "invalid probability")] + fn binary_prediction_from_probs_panics_on_nan() { + let _ = binary_prediction_from_probs(f64::NAN, 0.5, 1e-6); + } + + #[test] + #[should_panic(expected = "invalid probability")] + fn binary_prediction_from_probs_panics_on_negative() { + let _ = binary_prediction_from_probs(-1.0, 0.5, 1e-6); + } + + #[test] + #[should_panic(expected = "NaN log probability")] + fn binary_prediction_from_log_probs_panics_on_nan() { + let _ = binary_prediction_from_log_probs(f64::NAN, -1.0, 1e-6); + } + + #[test] + #[should_panic(expected = "+Inf log probability")] + fn binary_prediction_from_log_probs_panics_on_pos_inf() { + let _ = binary_prediction_from_log_probs(0.0, f64::INFINITY, 1e-6); + } + + #[test] + fn byte_prefix_partial_bits_reports_only_in_progress_prefixes() { + let pdf = [1.0 / 256.0; 256]; + let mut prefix = BytePrefixMass::from_pdf(&pdf, BitOrder::MsbFirst); + assert!(!prefix.has_partial_bits()); + + prefix.observe(true); + assert!(prefix.has_partial_bits()); + + for _ in 1..8u8 { + prefix.observe(false); + } + assert!(prefix.is_complete()); + assert!(!prefix.has_partial_bits()); + } +} diff --git a/crates/infotheory/src/rate_defaults.rs b/crates/infotheory/src/rate_defaults.rs new file mode 100644 index 00000000..0073cab5 --- /dev/null +++ b/crates/infotheory/src/rate_defaults.rs @@ -0,0 +1,196 @@ +//! Shared default-policy definitions for rate backend entrypoints. +//! +//! This module intentionally keeps distinct projections for: +//! - runtime implicit defaults, +//! - JSON parse defaults, +//! - shorthand parsing defaults. + +use crate::api::RateBackend; +use crate::runtime::RateBackendKind; +use std::sync::Arc; + +pub(crate) const JSON_DEFAULT_CTW_DEPTH: usize = 16; +pub(crate) const JSON_DEFAULT_FAC_CTW_BASE_DEPTH: usize = 16; +pub(crate) const JSON_DEFAULT_FAC_CTW_ENCODING_BITS: usize = 8; +pub(crate) const FAC_CTW_DEFAULT_NUM_PERCEPT_BITS: usize = 8; +pub(crate) const JSON_DEFAULT_MATCH_HASH_BITS: usize = 20; +pub(crate) const JSON_DEFAULT_MATCH_MIN_LEN: usize = 4; +pub(crate) const JSON_DEFAULT_MATCH_MAX_LEN: usize = 255; +pub(crate) const JSON_DEFAULT_MATCH_BASE_MIX: f64 = 0.02; +pub(crate) const JSON_DEFAULT_MATCH_CONFIDENCE_SCALE: f64 = 1.0; +pub(crate) const JSON_DEFAULT_SPARSE_MATCH_HASH_BITS: usize = 19; +pub(crate) const JSON_DEFAULT_SPARSE_MATCH_MIN_LEN: usize = 3; +pub(crate) const JSON_DEFAULT_SPARSE_MATCH_MAX_LEN: usize = 64; +pub(crate) const JSON_DEFAULT_SPARSE_MATCH_GAP_MIN: usize = 1; +pub(crate) const JSON_DEFAULT_SPARSE_MATCH_GAP_MAX: usize = 2; +pub(crate) const JSON_DEFAULT_SPARSE_MATCH_BASE_MIX: f64 = 0.05; +pub(crate) const JSON_DEFAULT_SPARSE_MATCH_CONFIDENCE_SCALE: f64 = 1.0; +pub(crate) const JSON_DEFAULT_PPMD_ORDER: usize = 10; +pub(crate) const JSON_DEFAULT_PPMD_MEMORY_MB: usize = 64; +pub(crate) const JSON_DEFAULT_SEQUITUR_CONTEXT_BYTES: usize = 64; +pub(crate) const JSON_DEFAULT_ZPAQ_RATE_METHOD: &str = "2"; + +pub(crate) const SHORTHAND_DEFAULT_CTW_DEPTH: usize = JSON_DEFAULT_CTW_DEPTH; +pub(crate) const SHORTHAND_DEFAULT_FAC_CTW_BASE_DEPTH: usize = JSON_DEFAULT_FAC_CTW_BASE_DEPTH; +pub(crate) const SHORTHAND_DEFAULT_FAC_CTW_NUM_PERCEPT_BITS: usize = + FAC_CTW_DEFAULT_NUM_PERCEPT_BITS; +pub(crate) const SHORTHAND_DEFAULT_FAC_CTW_ENCODING_BITS: usize = + JSON_DEFAULT_FAC_CTW_ENCODING_BITS; +pub(crate) const SHORTHAND_DEFAULT_PPMD_ORDER: usize = JSON_DEFAULT_PPMD_ORDER; +pub(crate) const SHORTHAND_DEFAULT_PPMD_MEMORY_MB: usize = JSON_DEFAULT_PPMD_MEMORY_MB; +pub(crate) const SHORTHAND_DEFAULT_SEQUITUR_CONTEXT_BYTES: usize = + JSON_DEFAULT_SEQUITUR_CONTEXT_BYTES; +pub(crate) const SHORTHAND_DEFAULT_ZPAQ_RATE_METHOD: &str = JSON_DEFAULT_ZPAQ_RATE_METHOD; + +/// Construct a [`RateBackend::FacCtw`] from explicit field values. +/// +/// `msb_first: None` defers to compile-time default (`encoding_bits == 8` → MSB-first). +pub fn fac_ctw_rate_backend( + base_depth: usize, + num_percept_bits: usize, + encoding_bits: usize, + msb_first: Option, +) -> RateBackend { + RateBackend::FacCtw { + base_depth, + num_percept_bits, + encoding_bits, + msb_first, + } +} + +/// JSON leaf object for a factorized CTW rate backend. +/// +/// Omits `msb_first` when `None` so compile-time defaults apply consistently. +/// +/// This is intentionally test-only; production code should construct +/// `RateBackend::FacCtw` via typed APIs and parse paths. +#[cfg(all(test, feature = "backend-ctw"))] +pub fn fac_ctw_spec_json( + base_depth: usize, + num_percept_bits: usize, + encoding_bits: usize, + msb_first: Option, +) -> serde_json::Value { + let mut object = serde_json::Map::new(); + object.insert( + "kind".to_string(), + serde_json::Value::String("fac-ctw".to_string()), + ); + object.insert( + "base_depth".to_string(), + serde_json::Value::Number(base_depth.into()), + ); + object.insert( + "num_percept_bits".to_string(), + serde_json::Value::Number(num_percept_bits.into()), + ); + object.insert( + "encoding_bits".to_string(), + serde_json::Value::Number(encoding_bits.into()), + ); + if let Some(msb_first) = msb_first { + object.insert("msb_first".to_string(), serde_json::Value::Bool(msb_first)); + } + serde_json::Value::Object(object) +} + +pub(crate) fn runtime_default_rate_backend_spec(kind: RateBackendKind) -> Option { + match kind { + RateBackendKind::RosaPlus => Some(RateBackend::RosaPlus { max_order: -1 }), + RateBackendKind::Match => Some(RateBackend::Match { + hash_bits: 18, + min_len: 4, + max_len: 96, + base_mix: 0.02, + confidence_scale: 1.0, + }), + RateBackendKind::SparseMatch => Some(RateBackend::SparseMatch { + hash_bits: 17, + min_len: 3, + max_len: 48, + gap_min: 1, + gap_max: 2, + base_mix: 0.05, + confidence_scale: 1.0, + }), + RateBackendKind::Ppmd => Some(RateBackend::Ppmd { + order: 6, + memory_mb: 16, + }), + RateBackendKind::Sequitur => Some(RateBackend::Sequitur { context_bytes: 32 }), + RateBackendKind::Ctw => Some(RateBackend::Ctw { depth: 8 }), + RateBackendKind::FacCtw => Some(fac_ctw_rate_backend( + 8, + FAC_CTW_DEFAULT_NUM_PERCEPT_BITS, + JSON_DEFAULT_FAC_CTW_ENCODING_BITS, + None, + )), + RateBackendKind::Zpaq => Some(RateBackend::Zpaq { + method: crate::api::ZpaqMethodSpec::literal("2"), + }), + RateBackendKind::Particle => Some(RateBackend::Particle { + spec: Arc::new(crate::api::ParticleSpec::default()), + }), + RateBackendKind::Mixture | RateBackendKind::Calibrated => None, + #[cfg(feature = "backend-mamba")] + RateBackendKind::Mamba => Some(RateBackend::MambaMethod { + method: crate::mambazip::MethodSpec::Online { + cfg: crate::mambazip::OnlineConfig { + hidden: 64, + layers: 1, + intermediate: 96, + state: 16, + conv: 4, + dt_rank: 16, + seed: 26, + train_mode: crate::mambazip::OnlineTrainMode::None, + lr: 0.0, + stride: 1, + }, + policy: Some(crate::backends::llm_policy::LlmPolicy { + load_from: None, + schedule: vec![crate::backends::llm_policy::ScheduleRule::Interval( + crate::backends::llm_policy::PolicyRule { + start: crate::backends::llm_policy::PositionExpr::Bytes(0), + end: crate::backends::llm_policy::PositionExpr::Bytes(100), + action: crate::backends::llm_policy::PolicyAction::Infer, + }, + )], + }), + }, + }), + #[cfg(not(feature = "backend-mamba"))] + RateBackendKind::Mamba => None, + #[cfg(feature = "backend-rwkv")] + RateBackendKind::Rwkv7 => Some(RateBackend::Rwkv7Method { + method: crate::rwkvzip::MethodSpec::Online { + cfg: crate::rwkvzip::OnlineConfig { + hidden: 64, + layers: 1, + intermediate: 64, + decay_rank: 32, + a_rank: 32, + v_rank: 32, + g_rank: 64, + seed: 0, + train_mode: crate::rwkvzip::OnlineTrainMode::Sgd, + lr: 0.01, + stride: 1, + }, + policy: Some(crate::backends::llm_policy::LlmPolicy { + load_from: None, + schedule: vec![crate::backends::llm_policy::ScheduleRule::Interval( + crate::backends::llm_policy::PolicyRule { + start: crate::backends::llm_policy::PositionExpr::Bytes(0), + end: crate::backends::llm_policy::PositionExpr::Bytes(100), + action: crate::backends::llm_policy::PolicyAction::Infer, + }, + )], + }), + }, + }), + #[cfg(not(feature = "backend-rwkv"))] + RateBackendKind::Rwkv7 => None, + } +} diff --git a/crates/infotheory/src/runtime/mod.rs b/crates/infotheory/src/runtime/mod.rs new file mode 100644 index 00000000..82ddb225 --- /dev/null +++ b/crates/infotheory/src/runtime/mod.rs @@ -0,0 +1,2779 @@ +//! Internal runtime builders and backend registry metadata. +//! +//! This module is the spec -> runtime boundary for predictor and compression +//! execution paths. + +use self::plan_macros::expect_plan_ref; +use crate::api::{CompressionBackend, RateBackend}; +#[cfg(feature = "backend-calibrated")] +use crate::backends::calibration::CalibratorCore; +#[cfg(feature = "backend-ctw")] +use crate::backends::ctw::{ContextTree, FacContextTree, ctw_symbol_bit_msb}; +#[cfg(feature = "backend-match")] +use crate::backends::match_model::MatchModel; +#[cfg(feature = "backend-particle")] +use crate::backends::particle::ParticleRuntime; +#[cfg(feature = "backend-ppmd")] +use crate::backends::ppmd::PpmdModel; +#[cfg(feature = "backend-rosa")] +use crate::backends::rosaplus::RosaPlus; +#[cfg(feature = "backend-sequitur")] +use crate::backends::sequitur::SequiturModel; +#[cfg(feature = "backend-match")] +use crate::backends::sparse_match::SparseMatchModel; +#[cfg(feature = "backend-zpaq")] +use crate::backends::zpaq_rate::ZpaqRateModel; +use crate::error::{InfotheoryError, InfotheoryResult}; +#[cfg(feature = "backend-mamba")] +use crate::mambazip; +#[cfg(feature = "backend-rwkv")] +use crate::rwkvzip; +use crate::spec::core::{ + CompressionBackendCapabilities, CompressionBackendPlan, MethodBackendFamily, + RateBackendCapabilities, RateBackendPlan, RateBackendTraceStrategy as PublicTraceStrategy, + SpecEnvironment, +}; +use crate::spec::{CompiledCompressionBackend, CompiledRateBackend, SpecResult}; +use std::sync::Arc; + +mod pdf_predictor_builders; +mod plan_macros; +mod predictor_builders; +mod registry; + +pub(crate) use registry::{ + describe_compression_backend_kind, describe_rate_backend_kind, + find_backend_descriptor_in_registry, +}; + +/// Stable internal identity for each rate-backend family. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] +pub enum RateBackendKind { + RosaPlus, + Ctw, + FacCtw, + Match, + SparseMatch, + Ppmd, + Sequitur, + Calibrated, + Zpaq, + Mixture, + Particle, + Mamba, + Rwkv7, +} + +/// Stable internal identity for each compression-backend family. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] +pub enum CompressionBackendKind { + Zpaq, + Rwkv7, + RateAc, + RateRans, +} + +/// Shared trace-model execution strategy used by VM glue. +#[cfg(feature = "vm")] +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] +pub enum TraceModelStrategy { + Rosa, + Ctw, + FacCtw, + PredictorBacked, + Zpaq, + Mamba, + Rwkv7, +} + +/// Canonical backend metadata entry shared by parsers and runtime builders. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct BackendDescriptor { + /// Stable internal backend identity. + pub kind: K, + /// Canonical backend name. + pub canonical: &'static str, + /// Accepted aliases for the backend, including the canonical spelling. + pub aliases: &'static [&'static str], + /// Required Cargo feature for availability, when the backend is feature-gated. + pub feature: Option<&'static str>, + /// Whether the backend is enabled in the current build. + pub enabled: bool, +} + +/// Canonical metadata for rate backends. +pub type RateBackendDescriptor = BackendDescriptor; +/// Canonical metadata for compression backends. +pub type CompressionBackendDescriptor = BackendDescriptor; + +macro_rules! backend_feature_option { + (none) => { + None + }; + ($feature:literal) => { + Some($feature) + }; +} + +macro_rules! backend_feature_enabled { + (none) => { + true + }; + ($feature:literal) => { + cfg!(feature = $feature) + }; +} + +// Feature-gated kernel pointers deliberately distinguish "anchor" from "drop". +// +// The runtime catalog contains one row per known backend even when a backend's +// implementation feature is disabled, so disabled builds still report stable +// names, aliases, canonical errors, and capability metadata without retaining +// heavyweight model code. Use `drop` for kernels whose real implementation may +// be compiled out completely; the catalog stores only the fallback pointer in +// disabled builds. Use `anchor` only for helpers that must remain available and +// type-checked when the backend feature is off, such as canonical spec helpers +// and cheap capability predicates. An anchored function must therefore be +// feature-independent and must not pull backend runtime state into disabled +// binaries. +macro_rules! feature_gated_kernel_ptr_anchor { + ($feature:literal, $ptr_type:ident, $func:path, fn fallback $args:tt $(-> $ret:ty)? $body:block) => { + { + #[cfg(feature = $feature)] + { $func as $ptr_type } + #[cfg(not(feature = $feature))] + { + #[allow(dead_code)] + fn reachability_anchor() { + let _ = $func as $ptr_type; + } + fn fallback $args $(-> $ret)? $body + fallback as $ptr_type + } + } + }; + (none, $ptr_type:ident, $func:path, fn fallback $args:tt $(-> $ret:ty)? $body:block) => { + $func as $ptr_type + }; +} + +macro_rules! feature_gated_kernel_ptr_drop { + ($feature:literal, $ptr_type:ident, $func:path, fn fallback $args:tt $(-> $ret:ty)? $body:block) => { + { + #[cfg(feature = $feature)] + { $func as $ptr_type } + #[cfg(not(feature = $feature))] + { + fn fallback $args $(-> $ret)? $body + fallback as $ptr_type + } + } + }; + (none, $ptr_type:ident, $func:path, fn fallback $args:tt $(-> $ret:ty)? $body:block) => { + $func as $ptr_type + }; +} + +macro_rules! feature_gated_kernel_ptr_mode { + (anchor, $feature:tt, $ptr_type:ident, $func:path, fn fallback $args:tt $(-> $ret:ty)? $body:block) => { + feature_gated_kernel_ptr_anchor!($feature, $ptr_type, $func, fn fallback $args $(-> $ret)? $body) + }; + (drop, $feature:tt, $ptr_type:ident, $func:path, fn fallback $args:tt $(-> $ret:ty)? $body:block) => { + feature_gated_kernel_ptr_drop!($feature, $ptr_type, $func, fn fallback $args $(-> $ret)? $body) + }; +} + +/// Shared declarative catalog for rate-backend metadata and runtime wiring. +/// +/// Adding a new rate backend should require: +/// 1. the backend-specific implementation code, +/// 2. one entry in this catalog. +/// +/// The catalog emits both the user-facing registry metadata and the runtime +/// kernel dispatch table so the family list cannot silently drift. +macro_rules! define_rate_backend_catalog { + ($( + backend { + kind: $kind:ident, + canonical: $canonical:literal, + aliases: [$($alias:literal),* $(,)?], + feature: $feature:tt, + spec_helper_refs: $spec_helper_refs:ident, + metric_helper_refs: $metric_helper_refs:ident, + compile_plan: $compile_plan:path, + to_wrapper: $to_wrapper:path, + encode_payload: $encode_payload:path, + display_label: $display_label:path, + default_name: $default_name:path, + trace_strategy: $trace_strategy:expr, + supports_biased_entropy: $supports_biased_entropy:expr, + supports_frozen_conditioning: $supports_frozen_conditioning:expr, + supports_rate_coded_compression: $supports_rate_coded_compression:expr, + supports_native_bit_prediction: $supports_native_bit_prediction:path, + supports_byte_prefix_mass: $supports_byte_prefix_mass:path, + supports_efficient_byte_packed_bit_sessions: $supports_efficient_byte_packed_bit_sessions:path, + supports_reversible_bit_updates: $supports_reversible_bit_updates:path, + method_family: $method_family:expr, + contains_zpaq: $contains_zpaq:path, + build_predictor: $build_predictor:path, + build_binary_token_predictor: $build_binary_token_predictor:path, + build_pdf_predictor: $build_pdf_predictor:path, + entropy_rate: $entropy_rate:path, + joint_entropy_rate: $joint_entropy_rate:path, + conditional_chain_rate: $conditional_chain_rate:path, + } + ),* $(,)?) => { + /// Registry of rate-backend metadata. + pub const RATE_BACKEND_REGISTRY: &[RateBackendDescriptor] = &[ + $( + BackendDescriptor { + kind: RateBackendKind::$kind, + canonical: $canonical, + aliases: &[$($alias),*], + feature: backend_feature_option!($feature), + enabled: backend_feature_enabled!($feature), + }, + )* + ]; + + pub(crate) const RATE_BACKEND_KERNELS: &[RateBackendKernel] = &[ + $( + RateBackendKernel { + kind: RateBackendKind::$kind, + compile_plan: feature_gated_kernel_ptr_mode!( + $spec_helper_refs, + $feature, RatePlanCompiler, $compile_plan, + fn fallback(_b: &RateBackend, _e: &SpecEnvironment, _d: usize) -> SpecResult { + unreachable!("kernel should never compile without its feature") + } + ), + to_wrapper: feature_gated_kernel_ptr_mode!( + $spec_helper_refs, + $feature, RateWrapperBuilder, $to_wrapper, + fn fallback(_p: &RateBackendPlan) -> RateBackend { + unreachable!("kernel should never emit a wrapper without its feature") + } + ), + encode_payload: feature_gated_kernel_ptr_mode!( + $spec_helper_refs, + $feature, RatePayloadEncoder, $encode_payload, + fn fallback(_p: &RateBackendPlan, _o: &mut Vec) { + unreachable!("kernel should never encode payload without its feature") + } + ), + display_label: feature_gated_kernel_ptr_mode!( + $spec_helper_refs, + $feature, RateDisplayLabelFn, $display_label, + fn fallback(_p: &RateBackendPlan) -> String { + unreachable!("kernel should never format a label without its feature") + } + ), + default_name: feature_gated_kernel_ptr_mode!( + $spec_helper_refs, + $feature, RateDefaultNameFn, $default_name, + fn fallback(_p: &RateBackendPlan) -> String { + unreachable!("kernel should never format a default name without its feature") + } + ), + trace_strategy: $trace_strategy, + supports_biased_entropy: $supports_biased_entropy, + supports_frozen_conditioning: $supports_frozen_conditioning, + supports_rate_coded_compression: $supports_rate_coded_compression, + supports_native_bit_prediction: feature_gated_kernel_ptr_anchor!( + $feature, RateCapabilityFn, $supports_native_bit_prediction, + fn fallback(_p: &RateBackendPlan) -> bool { + false + } + ), + supports_byte_prefix_mass: feature_gated_kernel_ptr_anchor!( + $feature, RateCapabilityFn, $supports_byte_prefix_mass, + fn fallback(_p: &RateBackendPlan) -> bool { + false + } + ), + supports_efficient_byte_packed_bit_sessions: feature_gated_kernel_ptr_anchor!( + $feature, RateCapabilityFn, $supports_efficient_byte_packed_bit_sessions, + fn fallback(_p: &RateBackendPlan) -> bool { + false + } + ), + supports_reversible_bit_updates: feature_gated_kernel_ptr_anchor!( + $feature, RateCapabilityFn, $supports_reversible_bit_updates, + fn fallback(_p: &RateBackendPlan) -> bool { + false + } + ), + method_family: $method_family, + contains_zpaq: $contains_zpaq, + build_predictor: feature_gated_kernel_ptr_drop!( + $feature, RateBackendPredictorBuilder, $build_predictor, + fn fallback(_b: &CompiledRateBackend, _p: f64) -> Result { + Err(registry::rate_backend_feature_error(RateBackendKind::$kind)) + } + ), + build_binary_token_predictor: feature_gated_kernel_ptr_drop!( + $feature, RateBackendBinaryTokenPredictorBuilder, $build_binary_token_predictor, + fn fallback(_b: &CompiledRateBackend, _p: f64) -> Result { + Err(registry::rate_backend_feature_error(RateBackendKind::$kind)) + } + ), + build_pdf_predictor: feature_gated_kernel_ptr_drop!( + $feature, RatePdfPredictorBuilder, $build_pdf_predictor, + fn fallback(_b: &CompiledRateBackend) -> anyhow::Result { + Err(anyhow::anyhow!(registry::rate_backend_feature_error(RateBackendKind::$kind))) + } + ), + entropy_rate: feature_gated_kernel_ptr_mode!( + $metric_helper_refs, + $feature, RateEntropyFn, $entropy_rate, + fn fallback(_d: &[u8], _b: &CompiledRateBackend) -> InfotheoryResult { + Err(InfotheoryError::unsupported(registry::rate_backend_feature_error(RateBackendKind::$kind))) + } + ), + joint_entropy_rate: feature_gated_kernel_ptr_mode!( + $metric_helper_refs, + $feature, RateJointEntropyFn, $joint_entropy_rate, + fn fallback(_x: &[u8], _y: &[u8], _b: &CompiledRateBackend) -> InfotheoryResult { + Err(InfotheoryError::unsupported(registry::rate_backend_feature_error(RateBackendKind::$kind))) + } + ), + conditional_chain_rate: feature_gated_kernel_ptr_mode!( + $metric_helper_refs, + $feature, RateConditionalChainFn, $conditional_chain_rate, + fn fallback(_p: &[&[u8]], _d: &[u8], _b: &CompiledRateBackend) -> InfotheoryResult { + Err(InfotheoryError::unsupported(registry::rate_backend_feature_error(RateBackendKind::$kind))) + } + ), + }, + )* + ]; + }; +} + +/// Shared declarative catalog for compression-backend metadata and runtime wiring. +macro_rules! define_compression_backend_catalog { + ($( + backend { + kind: $kind:ident, + canonical: $canonical:literal, + aliases: [$($alias:literal),* $(,)?], + feature: $feature:tt, + helper_refs: $helper_refs:ident, + compile_plan: $compile_plan:path, + to_wrapper: $to_wrapper:path, + encode_payload: $encode_payload:path, + display_label: $display_label:path, + uses_rate_backend: $uses_rate_backend:expr, + supports_decompression: $supports_decompression:expr, + build_runtime: $build_runtime:path, + } + ),* $(,)?) => { + /// Registry of compression-backend metadata. + pub const COMPRESSION_BACKEND_REGISTRY: &[CompressionBackendDescriptor] = &[ + $( + BackendDescriptor { + kind: CompressionBackendKind::$kind, + canonical: $canonical, + aliases: &[$($alias),*], + feature: backend_feature_option!($feature), + enabled: backend_feature_enabled!($feature), + }, + )* + ]; + + pub(crate) const COMPRESSION_BACKEND_KERNELS: &[CompressionBackendKernel] = &[ + $( + CompressionBackendKernel { + kind: CompressionBackendKind::$kind, + compile_plan: feature_gated_kernel_ptr_mode!( + $helper_refs, + $feature, CompressionPlanCompiler, $compile_plan, + fn fallback(_b: &CompressionBackend, _e: &SpecEnvironment) -> SpecResult { + unreachable!("compression kernel should never compile without its feature") + } + ), + to_wrapper: feature_gated_kernel_ptr_mode!( + $helper_refs, + $feature, CompressionWrapperBuilder, $to_wrapper, + fn fallback(_p: &CompressionBackendPlan) -> CompressionBackend { + unreachable!("compression kernel should never emit a wrapper without its feature") + } + ), + encode_payload: feature_gated_kernel_ptr_mode!( + $helper_refs, + $feature, CompressionPayloadEncoder, $encode_payload, + fn fallback(_p: &CompressionBackendPlan, _o: &mut Vec) { + unreachable!("compression kernel should never encode payload without its feature") + } + ), + display_label: feature_gated_kernel_ptr_mode!( + $helper_refs, + $feature, CompressionDisplayLabelFn, $display_label, + fn fallback(_p: &CompressionBackendPlan) -> String { + unreachable!("compression kernel should never format a label without its feature") + } + ), + uses_rate_backend: $uses_rate_backend, + supports_decompression: $supports_decompression, + build_runtime: feature_gated_kernel_ptr_mode!( + $helper_refs, + $feature, CompressionRuntimeBuilder, $build_runtime, + fn fallback(_b: &CompiledCompressionBackend) -> Result { + Err(registry::compression_backend_feature_error(CompressionBackendKind::$kind)) + } + ), + }, + )* + ]; + }; +} + +type RateBackendPredictorBuilder = + fn(&CompiledRateBackend, f64) -> Result; +type RateBackendBinaryTokenPredictorBuilder = + fn(&CompiledRateBackend, f64) -> Result; +type RatePdfPredictorBuilder = + fn(&CompiledRateBackend) -> anyhow::Result; +type RateEntropyFn = fn(&[u8], &CompiledRateBackend) -> InfotheoryResult; +type RateJointEntropyFn = fn(&[u8], &[u8], &CompiledRateBackend) -> InfotheoryResult; +type RateConditionalChainFn = fn(&[&[u8]], &[u8], &CompiledRateBackend) -> InfotheoryResult; +type RatePlanCompiler = fn(&RateBackend, &SpecEnvironment, usize) -> SpecResult; +type RateWrapperBuilder = fn(&RateBackendPlan) -> RateBackend; +type RatePayloadEncoder = fn(&RateBackendPlan, &mut Vec); +type RateDisplayLabelFn = fn(&RateBackendPlan) -> String; +type RateDefaultNameFn = fn(&RateBackendPlan) -> String; +type RateCapabilityFn = fn(&RateBackendPlan) -> bool; +type RateContainsZpaqFn = fn(&RateBackendPlan) -> bool; +type CompressionRuntimeBuilder = + fn(&CompiledCompressionBackend) -> Result; +type CompressionPlanCompiler = + fn(&CompressionBackend, &SpecEnvironment) -> SpecResult; +type CompressionWrapperBuilder = fn(&CompressionBackendPlan) -> CompressionBackend; +type CompressionPayloadEncoder = fn(&CompressionBackendPlan, &mut Vec); +type CompressionDisplayLabelFn = fn(&CompressionBackendPlan) -> String; + +#[derive(Clone, Copy)] +pub(crate) struct RateBackendKernel { + pub kind: RateBackendKind, + pub compile_plan: RatePlanCompiler, + pub to_wrapper: RateWrapperBuilder, + pub encode_payload: RatePayloadEncoder, + pub display_label: RateDisplayLabelFn, + pub default_name: RateDefaultNameFn, + pub trace_strategy: PublicTraceStrategy, + pub supports_biased_entropy: bool, + pub supports_frozen_conditioning: bool, + pub supports_rate_coded_compression: bool, + pub supports_native_bit_prediction: RateCapabilityFn, + pub supports_byte_prefix_mass: RateCapabilityFn, + pub supports_efficient_byte_packed_bit_sessions: RateCapabilityFn, + pub supports_reversible_bit_updates: RateCapabilityFn, + pub method_family: Option, + pub contains_zpaq: RateContainsZpaqFn, + pub build_predictor: RateBackendPredictorBuilder, + pub build_binary_token_predictor: RateBackendBinaryTokenPredictorBuilder, + pub build_pdf_predictor: RatePdfPredictorBuilder, + pub entropy_rate: RateEntropyFn, + pub joint_entropy_rate: RateJointEntropyFn, + pub conditional_chain_rate: RateConditionalChainFn, +} + +#[derive(Clone, Copy)] +pub(crate) struct CompressionBackendKernel { + pub kind: CompressionBackendKind, + pub compile_plan: CompressionPlanCompiler, + pub to_wrapper: CompressionWrapperBuilder, + pub encode_payload: CompressionPayloadEncoder, + pub display_label: CompressionDisplayLabelFn, + pub uses_rate_backend: bool, + pub supports_decompression: bool, + pub build_runtime: CompressionRuntimeBuilder, +} + +define_rate_backend_catalog! { + backend { + kind: RosaPlus, + canonical: "rosaplus", + aliases: ["rosaplus", "rosa"], + feature: "backend-rosa", + spec_helper_refs: anchor, + metric_helper_refs: drop, + compile_plan: crate::spec::core::compile_rate_plan_rosa, + to_wrapper: crate::spec::core::rate_plan_to_wrapper_rosa, + encode_payload: crate::spec::core::encode_rate_payload_rosa, + display_label: crate::spec::core::rate_plan_display_label_rosa, + default_name: crate::spec::core::rate_plan_default_name_rosa, + trace_strategy: PublicTraceStrategy::Rosa, + supports_biased_entropy: true, + supports_frozen_conditioning: true, + supports_rate_coded_compression: true, + supports_native_bit_prediction: crate::runtime::capability_always_false, + supports_byte_prefix_mass: crate::runtime::capability_always_true, + supports_efficient_byte_packed_bit_sessions: crate::runtime::capability_always_true, + supports_reversible_bit_updates: crate::runtime::capability_always_false, + method_family: None, + contains_zpaq: crate::spec::core::rate_plan_contains_zpaq_false, + build_predictor: predictor_builders::build_predictor_rosa, + build_binary_token_predictor: predictor_builders::build_predictor_rosa, + build_pdf_predictor: pdf_predictor_builders::build_pdf_predictor_rosa, + entropy_rate: entropy_rosa, + joint_entropy_rate: joint_entropy_rosa, + conditional_chain_rate: conditional_chain_rosa, + }, + backend { + kind: Match, + canonical: "match", + aliases: ["match"], + feature: "backend-match", + spec_helper_refs: anchor, + metric_helper_refs: anchor, + compile_plan: crate::spec::core::compile_rate_plan_match, + to_wrapper: crate::spec::core::rate_plan_to_wrapper_match, + encode_payload: crate::spec::core::encode_rate_payload_match, + display_label: crate::spec::core::rate_plan_display_label_match, + default_name: crate::spec::core::rate_plan_default_name_match, + trace_strategy: PublicTraceStrategy::PredictorBacked, + supports_biased_entropy: true, + supports_frozen_conditioning: true, + supports_rate_coded_compression: true, + supports_native_bit_prediction: crate::runtime::capability_always_false, + supports_byte_prefix_mass: crate::runtime::capability_always_true, + supports_efficient_byte_packed_bit_sessions: crate::runtime::capability_always_true, + supports_reversible_bit_updates: crate::runtime::capability_always_false, + method_family: None, + contains_zpaq: crate::spec::core::rate_plan_contains_zpaq_false, + build_predictor: predictor_builders::build_predictor_match, + build_binary_token_predictor: predictor_builders::build_predictor_match, + build_pdf_predictor: pdf_predictor_builders::build_pdf_predictor_match, + entropy_rate: entropy_prequential, + joint_entropy_rate: joint_entropy_prequential, + conditional_chain_rate: conditional_chain_prequential, + }, + backend { + kind: SparseMatch, + canonical: "sparse-match", + aliases: ["sparse-match"], + feature: "backend-match", + spec_helper_refs: anchor, + metric_helper_refs: anchor, + compile_plan: crate::spec::core::compile_rate_plan_sparse_match, + to_wrapper: crate::spec::core::rate_plan_to_wrapper_sparse_match, + encode_payload: crate::spec::core::encode_rate_payload_sparse_match, + display_label: crate::spec::core::rate_plan_display_label_sparse_match, + default_name: crate::spec::core::rate_plan_default_name_sparse_match, + trace_strategy: PublicTraceStrategy::PredictorBacked, + supports_biased_entropy: true, + supports_frozen_conditioning: true, + supports_rate_coded_compression: true, + supports_native_bit_prediction: crate::runtime::capability_always_false, + supports_byte_prefix_mass: crate::runtime::capability_always_true, + supports_efficient_byte_packed_bit_sessions: crate::runtime::capability_always_true, + supports_reversible_bit_updates: crate::runtime::capability_always_false, + method_family: None, + contains_zpaq: crate::spec::core::rate_plan_contains_zpaq_false, + build_predictor: predictor_builders::build_predictor_sparse_match, + build_binary_token_predictor: predictor_builders::build_predictor_sparse_match, + build_pdf_predictor: pdf_predictor_builders::build_pdf_predictor_sparse_match, + entropy_rate: entropy_prequential, + joint_entropy_rate: joint_entropy_prequential, + conditional_chain_rate: conditional_chain_prequential, + }, + backend { + kind: Ppmd, + canonical: "ppmd", + aliases: ["ppmd"], + feature: "backend-ppmd", + spec_helper_refs: anchor, + metric_helper_refs: anchor, + compile_plan: crate::spec::core::compile_rate_plan_ppmd, + to_wrapper: crate::spec::core::rate_plan_to_wrapper_ppmd, + encode_payload: crate::spec::core::encode_rate_payload_ppmd, + display_label: crate::spec::core::rate_plan_display_label_ppmd, + default_name: crate::spec::core::rate_plan_default_name_ppmd, + trace_strategy: PublicTraceStrategy::PredictorBacked, + supports_biased_entropy: true, + supports_frozen_conditioning: true, + supports_rate_coded_compression: true, + supports_native_bit_prediction: crate::runtime::capability_always_false, + supports_byte_prefix_mass: crate::runtime::capability_always_true, + supports_efficient_byte_packed_bit_sessions: crate::runtime::capability_always_true, + supports_reversible_bit_updates: crate::runtime::capability_always_false, + method_family: None, + contains_zpaq: crate::spec::core::rate_plan_contains_zpaq_false, + build_predictor: predictor_builders::build_predictor_ppmd, + build_binary_token_predictor: predictor_builders::build_predictor_ppmd, + build_pdf_predictor: pdf_predictor_builders::build_pdf_predictor_ppmd, + entropy_rate: entropy_prequential, + joint_entropy_rate: joint_entropy_prequential, + conditional_chain_rate: conditional_chain_prequential, + }, + backend { + kind: Sequitur, + canonical: "sequitur", + aliases: ["sequitur"], + feature: "backend-sequitur", + spec_helper_refs: anchor, + metric_helper_refs: anchor, + compile_plan: crate::spec::core::compile_rate_plan_sequitur, + to_wrapper: crate::spec::core::rate_plan_to_wrapper_sequitur, + encode_payload: crate::spec::core::encode_rate_payload_sequitur, + display_label: crate::spec::core::rate_plan_display_label_sequitur, + default_name: crate::spec::core::rate_plan_default_name_sequitur, + trace_strategy: PublicTraceStrategy::PredictorBacked, + supports_biased_entropy: true, + supports_frozen_conditioning: true, + supports_rate_coded_compression: true, + supports_native_bit_prediction: crate::runtime::capability_always_false, + supports_byte_prefix_mass: crate::runtime::capability_always_true, + supports_efficient_byte_packed_bit_sessions: crate::runtime::capability_always_true, + supports_reversible_bit_updates: crate::runtime::capability_always_false, + method_family: None, + contains_zpaq: crate::spec::core::rate_plan_contains_zpaq_false, + build_predictor: predictor_builders::build_predictor_sequitur, + build_binary_token_predictor: predictor_builders::build_predictor_sequitur, + build_pdf_predictor: pdf_predictor_builders::build_pdf_predictor_sequitur, + entropy_rate: entropy_prequential, + joint_entropy_rate: joint_entropy_prequential, + conditional_chain_rate: conditional_chain_prequential, + }, + backend { + kind: Ctw, + canonical: "ctw", + aliases: ["ctw", "ac-ctw"], + feature: "backend-ctw", + spec_helper_refs: anchor, + metric_helper_refs: drop, + compile_plan: crate::spec::core::compile_rate_plan_ctw, + to_wrapper: crate::spec::core::rate_plan_to_wrapper_ctw, + encode_payload: crate::spec::core::encode_rate_payload_ctw, + display_label: crate::spec::core::rate_plan_display_label_ctw, + default_name: crate::spec::core::rate_plan_default_name_ctw, + trace_strategy: PublicTraceStrategy::Ctw, + supports_biased_entropy: true, + supports_frozen_conditioning: true, + supports_rate_coded_compression: true, + supports_native_bit_prediction: crate::runtime::capability_always_true, + supports_byte_prefix_mass: crate::runtime::capability_always_true, + supports_efficient_byte_packed_bit_sessions: crate::runtime::capability_always_true, + supports_reversible_bit_updates: crate::runtime::capability_always_true, + method_family: None, + contains_zpaq: crate::spec::core::rate_plan_contains_zpaq_false, + build_predictor: predictor_builders::build_predictor_ctw, + build_binary_token_predictor: predictor_builders::build_predictor_binary_tokens_ctw, + build_pdf_predictor: pdf_predictor_builders::build_pdf_predictor_ctw, + entropy_rate: entropy_ctw, + joint_entropy_rate: joint_entropy_ctw, + conditional_chain_rate: conditional_chain_ctw, + }, + backend { + kind: FacCtw, + canonical: "fac-ctw", + aliases: ["fac-ctw"], + feature: "backend-ctw", + spec_helper_refs: anchor, + metric_helper_refs: drop, + compile_plan: crate::spec::core::compile_rate_plan_fac_ctw, + to_wrapper: crate::spec::core::rate_plan_to_wrapper_fac_ctw, + encode_payload: crate::spec::core::encode_rate_payload_fac_ctw, + display_label: crate::spec::core::rate_plan_display_label_fac_ctw, + default_name: crate::spec::core::rate_plan_default_name_fac_ctw, + trace_strategy: PublicTraceStrategy::FacCtw, + supports_biased_entropy: true, + supports_frozen_conditioning: true, + supports_rate_coded_compression: true, + supports_native_bit_prediction: crate::runtime::capability_always_true, + supports_byte_prefix_mass: crate::runtime::capability_always_true, + supports_efficient_byte_packed_bit_sessions: crate::runtime::capability_always_true, + supports_reversible_bit_updates: crate::runtime::capability_always_true, + method_family: None, + contains_zpaq: crate::spec::core::rate_plan_contains_zpaq_false, + build_predictor: predictor_builders::build_predictor_fac_ctw, + build_binary_token_predictor: predictor_builders::build_predictor_binary_tokens_fac_ctw, + build_pdf_predictor: pdf_predictor_builders::build_pdf_predictor_fac_ctw, + entropy_rate: entropy_fac_ctw, + joint_entropy_rate: joint_entropy_fac_ctw, + conditional_chain_rate: conditional_chain_fac_ctw, + }, + backend { + kind: Zpaq, + canonical: "zpaq", + aliases: ["zpaq"], + feature: "backend-zpaq", + spec_helper_refs: anchor, + metric_helper_refs: drop, + compile_plan: crate::spec::core::compile_rate_plan_zpaq, + to_wrapper: crate::spec::core::rate_plan_to_wrapper_zpaq, + encode_payload: crate::spec::core::encode_rate_payload_zpaq, + display_label: crate::spec::core::rate_plan_display_label_zpaq, + default_name: crate::spec::core::rate_plan_default_name_zpaq, + trace_strategy: PublicTraceStrategy::Zpaq, + supports_biased_entropy: false, + supports_frozen_conditioning: false, + supports_rate_coded_compression: true, + supports_native_bit_prediction: crate::runtime::capability_always_false, + supports_byte_prefix_mass: crate::runtime::capability_always_true, + supports_efficient_byte_packed_bit_sessions: crate::runtime::capability_always_false, + supports_reversible_bit_updates: crate::runtime::capability_always_false, + method_family: None, + contains_zpaq: crate::spec::core::rate_plan_contains_zpaq_true, + build_predictor: predictor_builders::build_predictor_zpaq, + build_binary_token_predictor: predictor_builders::build_predictor_zpaq, + build_pdf_predictor: pdf_predictor_builders::build_pdf_predictor_zpaq, + entropy_rate: entropy_zpaq, + joint_entropy_rate: joint_entropy_zpaq, + conditional_chain_rate: conditional_chain_zpaq, + }, + backend { + kind: Mixture, + canonical: "mixture", + aliases: ["mixture"], + feature: "backend-mixture", + spec_helper_refs: anchor, + metric_helper_refs: drop, + compile_plan: crate::spec::core::compile_rate_plan_mixture, + to_wrapper: crate::spec::core::rate_plan_to_wrapper_mixture, + encode_payload: crate::spec::core::encode_rate_payload_mixture, + display_label: crate::spec::core::rate_plan_display_label_mixture, + default_name: crate::spec::core::rate_plan_default_name_mixture, + trace_strategy: PublicTraceStrategy::PredictorBacked, + supports_biased_entropy: true, + supports_frozen_conditioning: true, + supports_rate_coded_compression: true, + supports_native_bit_prediction: crate::runtime::mixture_supports_native_bit_prediction, + supports_byte_prefix_mass: crate::runtime::mixture_supports_byte_prefix_mass, + supports_efficient_byte_packed_bit_sessions: + crate::runtime::mixture_supports_efficient_byte_packed_bit_sessions, + supports_reversible_bit_updates: crate::runtime::mixture_supports_reversible_bit_updates, + method_family: None, + contains_zpaq: crate::spec::core::rate_plan_contains_zpaq_mixture, + build_predictor: predictor_builders::build_predictor_mixture, + build_binary_token_predictor: predictor_builders::build_predictor_binary_tokens_mixture, + build_pdf_predictor: pdf_predictor_builders::build_pdf_predictor_mixture, + entropy_rate: entropy_mixture, + joint_entropy_rate: joint_entropy_mixture, + conditional_chain_rate: conditional_chain_mixture, + }, + backend { + kind: Particle, + canonical: "particle", + aliases: ["particle"], + feature: "backend-particle", + spec_helper_refs: anchor, + metric_helper_refs: drop, + compile_plan: crate::spec::core::compile_rate_plan_particle, + to_wrapper: crate::spec::core::rate_plan_to_wrapper_particle, + encode_payload: crate::spec::core::encode_rate_payload_particle, + display_label: crate::spec::core::rate_plan_display_label_particle, + default_name: crate::spec::core::rate_plan_default_name_particle, + trace_strategy: PublicTraceStrategy::PredictorBacked, + supports_biased_entropy: true, + supports_frozen_conditioning: true, + supports_rate_coded_compression: true, + supports_native_bit_prediction: crate::runtime::capability_always_false, + supports_byte_prefix_mass: crate::runtime::capability_always_true, + supports_efficient_byte_packed_bit_sessions: crate::runtime::capability_always_true, + supports_reversible_bit_updates: crate::runtime::capability_always_false, + method_family: None, + contains_zpaq: crate::spec::core::rate_plan_contains_zpaq_false, + build_predictor: predictor_builders::build_predictor_particle, + build_binary_token_predictor: predictor_builders::build_predictor_particle, + build_pdf_predictor: pdf_predictor_builders::build_pdf_predictor_particle, + entropy_rate: entropy_particle, + joint_entropy_rate: joint_entropy_particle, + conditional_chain_rate: conditional_chain_particle, + }, + backend { + kind: Calibrated, + canonical: "calibrated", + aliases: ["calibrated"], + feature: "backend-calibrated", + spec_helper_refs: anchor, + metric_helper_refs: anchor, + compile_plan: crate::spec::core::compile_rate_plan_calibrated, + to_wrapper: crate::spec::core::rate_plan_to_wrapper_calibrated, + encode_payload: crate::spec::core::encode_rate_payload_calibrated, + display_label: crate::spec::core::rate_plan_display_label_calibrated, + default_name: crate::spec::core::rate_plan_default_name_calibrated, + trace_strategy: PublicTraceStrategy::PredictorBacked, + supports_biased_entropy: true, + supports_frozen_conditioning: true, + supports_rate_coded_compression: true, + supports_native_bit_prediction: crate::runtime::calibrated_supports_native_bit_prediction, + supports_byte_prefix_mass: crate::runtime::calibrated_supports_byte_prefix_mass, + supports_efficient_byte_packed_bit_sessions: + crate::runtime::calibrated_supports_efficient_byte_packed_bit_sessions, + supports_reversible_bit_updates: crate::runtime::calibrated_supports_reversible_bit_updates, + method_family: None, + contains_zpaq: crate::spec::core::rate_plan_contains_zpaq_calibrated, + build_predictor: predictor_builders::build_predictor_calibrated, + build_binary_token_predictor: predictor_builders::build_predictor_binary_tokens_calibrated, + build_pdf_predictor: pdf_predictor_builders::build_pdf_predictor_calibrated, + entropy_rate: entropy_prequential, + joint_entropy_rate: joint_entropy_prequential, + conditional_chain_rate: conditional_chain_prequential, + }, + backend { + kind: Mamba, + canonical: "mamba", + aliases: ["mamba"], + feature: "backend-mamba", + spec_helper_refs: drop, + metric_helper_refs: drop, + compile_plan: crate::spec::core::compile_rate_plan_mamba, + to_wrapper: crate::spec::core::rate_plan_to_wrapper_mamba, + encode_payload: crate::spec::core::encode_rate_payload_mamba, + display_label: crate::spec::core::rate_plan_display_label_mamba, + default_name: crate::spec::core::rate_plan_default_name_mamba, + trace_strategy: PublicTraceStrategy::Mamba, + supports_biased_entropy: true, + supports_frozen_conditioning: true, + supports_rate_coded_compression: true, + supports_native_bit_prediction: crate::runtime::capability_always_false, + supports_byte_prefix_mass: crate::runtime::capability_always_true, + supports_efficient_byte_packed_bit_sessions: crate::runtime::capability_always_true, + supports_reversible_bit_updates: crate::runtime::capability_always_false, + method_family: Some(MethodBackendFamily::Mamba), + contains_zpaq: crate::spec::core::rate_plan_contains_zpaq_false, + build_predictor: predictor_builders::build_predictor_mamba, + build_binary_token_predictor: predictor_builders::build_predictor_mamba, + build_pdf_predictor: pdf_predictor_builders::build_pdf_predictor_mamba, + entropy_rate: entropy_mamba, + joint_entropy_rate: joint_entropy_mamba, + conditional_chain_rate: conditional_chain_mamba, + }, + backend { + kind: Rwkv7, + canonical: "rwkv7", + aliases: ["rwkv7"], + feature: "backend-rwkv", + spec_helper_refs: drop, + metric_helper_refs: drop, + compile_plan: crate::spec::core::compile_rate_plan_rwkv7, + to_wrapper: crate::spec::core::rate_plan_to_wrapper_rwkv7, + encode_payload: crate::spec::core::encode_rate_payload_rwkv7, + display_label: crate::spec::core::rate_plan_display_label_rwkv7, + default_name: crate::spec::core::rate_plan_default_name_rwkv7, + trace_strategy: PublicTraceStrategy::Rwkv7, + supports_biased_entropy: true, + supports_frozen_conditioning: true, + supports_rate_coded_compression: true, + supports_native_bit_prediction: crate::runtime::capability_always_false, + supports_byte_prefix_mass: crate::runtime::capability_always_true, + supports_efficient_byte_packed_bit_sessions: crate::runtime::capability_always_true, + supports_reversible_bit_updates: crate::runtime::capability_always_false, + method_family: Some(MethodBackendFamily::Rwkv7), + contains_zpaq: crate::spec::core::rate_plan_contains_zpaq_false, + build_predictor: predictor_builders::build_predictor_rwkv, + build_binary_token_predictor: predictor_builders::build_predictor_rwkv, + build_pdf_predictor: pdf_predictor_builders::build_pdf_predictor_rwkv, + entropy_rate: entropy_rwkv, + joint_entropy_rate: joint_entropy_rwkv, + conditional_chain_rate: conditional_chain_rwkv, + }, +} + +define_compression_backend_catalog! { + backend { + kind: Zpaq, + canonical: "zpaq", + aliases: ["zpaq"], + feature: "backend-zpaq", + helper_refs: anchor, + compile_plan: crate::spec::core::compile_compression_plan_zpaq, + to_wrapper: crate::spec::core::compression_plan_to_wrapper_zpaq, + encode_payload: crate::spec::core::encode_compression_payload_zpaq, + display_label: crate::spec::core::compression_plan_display_label_zpaq, + uses_rate_backend: false, + supports_decompression: true, + build_runtime: build_compression_runtime_zpaq, + }, + backend { + kind: Rwkv7, + canonical: "rwkv7", + aliases: ["rwkv7"], + feature: "backend-rwkv", + helper_refs: drop, + compile_plan: crate::spec::core::compile_compression_plan_rwkv7, + to_wrapper: crate::spec::core::compression_plan_to_wrapper_rwkv7, + encode_payload: crate::spec::core::encode_compression_payload_rwkv7, + display_label: crate::spec::core::compression_plan_display_label_rwkv7, + uses_rate_backend: false, + supports_decompression: true, + build_runtime: build_compression_runtime_rwkv, + }, + backend { + kind: RateAc, + canonical: "rate-ac", + aliases: ["rate-ac"], + feature: none, + helper_refs: anchor, + compile_plan: crate::spec::core::compile_compression_plan_rate, + to_wrapper: crate::spec::core::compression_plan_to_wrapper_rate, + encode_payload: crate::spec::core::encode_compression_payload_rate, + display_label: crate::spec::core::compression_plan_display_label_rate, + uses_rate_backend: true, + supports_decompression: true, + build_runtime: build_compression_runtime_rate, + }, + backend { + kind: RateRans, + canonical: "rate-rans", + aliases: ["rate-rans"], + feature: none, + helper_refs: anchor, + compile_plan: crate::spec::core::compile_compression_plan_rate, + to_wrapper: crate::spec::core::compression_plan_to_wrapper_rate, + encode_payload: crate::spec::core::encode_compression_payload_rate, + display_label: crate::spec::core::compression_plan_display_label_rate, + uses_rate_backend: true, + supports_decompression: true, + build_runtime: build_compression_runtime_rate, + }, +} + +pub(crate) fn default_rate_backend_spec(kind: RateBackendKind) -> Option { + crate::rate_defaults::runtime_default_rate_backend_spec(kind) +} + +pub(crate) fn first_enabled_default_rate_backend_spec() -> Option { + RATE_BACKEND_REGISTRY + .iter() + .filter(|descriptor| descriptor.enabled) + .find_map(|descriptor| default_rate_backend_spec(descriptor.kind)) +} + +pub(crate) fn rate_backend_kernel(kind: RateBackendKind) -> &'static RateBackendKernel { + RATE_BACKEND_KERNELS + .iter() + .find(|kernel| kernel.kind == kind) + .unwrap_or_else(|| panic!("missing RATE_BACKEND_KERNELS entry for {kind:?}")) +} + +pub(crate) fn compression_backend_kernel( + kind: CompressionBackendKind, +) -> &'static CompressionBackendKernel { + COMPRESSION_BACKEND_KERNELS + .iter() + .find(|kernel| kernel.kind == kind) + .unwrap_or_else(|| panic!("missing COMPRESSION_BACKEND_KERNELS entry for {kind:?}")) +} + +pub(crate) fn rate_backend_canonical_name(kind: RateBackendKind) -> &'static str { + describe_rate_backend_kind(kind) + .unwrap_or_else(|err| panic!("{err}")) + .canonical +} + +pub(crate) fn compression_backend_canonical_name(kind: CompressionBackendKind) -> &'static str { + describe_compression_backend_kind(kind) + .unwrap_or_else(|err| panic!("{err}")) + .canonical +} + +pub(crate) fn compile_rate_backend_plan_via_kernel( + kind: RateBackendKind, + backend: &RateBackend, + env: &SpecEnvironment, + depth: usize, +) -> SpecResult { + (rate_backend_kernel(kind).compile_plan)(backend, env, depth) +} + +pub(crate) fn compile_compression_backend_plan_via_kernel( + kind: CompressionBackendKind, + backend: &CompressionBackend, + env: &SpecEnvironment, +) -> SpecResult { + (compression_backend_kernel(kind).compile_plan)(backend, env) +} + +pub(crate) fn rate_backend_wrapper_via_kernel(plan: &RateBackendPlan) -> RateBackend { + (rate_backend_kernel(plan.kind()).to_wrapper)(plan) +} + +pub(crate) fn compression_backend_wrapper_via_kernel( + plan: &CompressionBackendPlan, +) -> CompressionBackend { + (compression_backend_kernel(plan.kind()).to_wrapper)(plan) +} + +pub(crate) fn encode_rate_backend_payload_via_kernel(plan: &RateBackendPlan, out: &mut Vec) { + (rate_backend_kernel(plan.kind()).encode_payload)(plan, out); +} + +pub(crate) fn encode_compression_backend_payload_via_kernel( + plan: &CompressionBackendPlan, + out: &mut Vec, +) { + (compression_backend_kernel(plan.kind()).encode_payload)(plan, out); +} + +pub(crate) fn rate_backend_display_label_via_kernel(plan: &RateBackendPlan) -> String { + (rate_backend_kernel(plan.kind()).display_label)(plan) +} + +pub(crate) fn rate_backend_default_name_via_kernel(plan: &RateBackendPlan) -> String { + (rate_backend_kernel(plan.kind()).default_name)(plan) +} + +pub(crate) fn compression_backend_display_label_via_kernel( + plan: &CompressionBackendPlan, +) -> String { + (compression_backend_kernel(plan.kind()).display_label)(plan) +} + +pub(crate) fn rate_backend_capabilities_via_kernel( + plan: &RateBackendPlan, +) -> RateBackendCapabilities { + let kernel = rate_backend_kernel(plan.kind()); + RateBackendCapabilities { + canonical_name: rate_backend_canonical_name(plan.kind()), + display_label: Arc::::from((kernel.display_label)(plan)), + trace_strategy: kernel.trace_strategy, + supports_biased_entropy: kernel.supports_biased_entropy, + supports_frozen_conditioning: kernel.supports_frozen_conditioning, + supports_rate_coded_compression: kernel.supports_rate_coded_compression, + supports_native_bit_prediction: (kernel.supports_native_bit_prediction)(plan), + supports_byte_prefix_mass: (kernel.supports_byte_prefix_mass)(plan), + supports_efficient_byte_packed_bit_sessions: (kernel + .supports_efficient_byte_packed_bit_sessions)( + plan + ), + supports_reversible_bit_updates: (kernel.supports_reversible_bit_updates)(plan), + contains_zpaq: (kernel.contains_zpaq)(plan), + method_family: kernel.method_family, + } +} + +pub(crate) fn compression_backend_capabilities_via_kernel( + plan: &CompressionBackendPlan, +) -> CompressionBackendCapabilities { + let kernel = compression_backend_kernel(plan.kind()); + CompressionBackendCapabilities { + canonical_name: compression_backend_canonical_name(plan.kind()), + display_label: Arc::::from(compression_backend_display_label_via_kernel(plan)), + uses_rate_backend: kernel.uses_rate_backend, + supports_decompression: kernel.supports_decompression, + } +} + +#[cfg(feature = "vm")] +pub(crate) fn rate_backend_trace_model_strategy( + backend: &CompiledRateBackend, +) -> TraceModelStrategy { + match backend.capabilities().trace_strategy { + PublicTraceStrategy::Rosa => TraceModelStrategy::Rosa, + PublicTraceStrategy::Ctw => TraceModelStrategy::Ctw, + PublicTraceStrategy::FacCtw => TraceModelStrategy::FacCtw, + PublicTraceStrategy::PredictorBacked => TraceModelStrategy::PredictorBacked, + PublicTraceStrategy::Zpaq => TraceModelStrategy::Zpaq, + PublicTraceStrategy::Mamba => TraceModelStrategy::Mamba, + PublicTraceStrategy::Rwkv7 => TraceModelStrategy::Rwkv7, + } +} + +#[cfg(feature = "backend-rwkv")] +/// Execute `f` with the RWKV method string + parsed spec from a compiled backend. +fn with_rwkv_backend_plan( + backend: &CompiledRateBackend, + f: impl FnOnce(&str, &crate::rwkvzip::MethodSpec) -> InfotheoryResult, +) -> InfotheoryResult { + expect_plan_ref!( + backend.plan(), + crate::spec::core::RateBackendPlan::Rwkv7 { + method, + parsed_method, + .. + }, + "rwkv kernel used with non-rwkv plan" + ); + f(method, parsed_method) +} + +#[cfg(feature = "backend-mamba")] +/// Execute `f` with the Mamba method string + parsed spec from a compiled backend. +fn with_mamba_backend_plan( + backend: &CompiledRateBackend, + f: impl FnOnce(&str, &crate::mambazip::MethodSpec) -> InfotheoryResult, +) -> InfotheoryResult { + expect_plan_ref!( + backend.plan(), + crate::spec::core::RateBackendPlan::Mamba { + method, + parsed_method, + .. + }, + "mamba kernel used with non-mamba plan" + ); + f(method, parsed_method) +} + +#[cfg(feature = "backend-zpaq")] +/// Execute `f` with the ZPAQ method string from a compiled backend. +fn with_zpaq_backend_plan( + backend: &CompiledRateBackend, + f: impl FnOnce(&str) -> InfotheoryResult, +) -> InfotheoryResult { + expect_plan_ref!( + backend.plan(), + crate::spec::core::RateBackendPlan::Zpaq { method }, + "zpaq kernel used with non-zpaq plan" + ); + f(method) +} + +#[cfg(feature = "backend-particle")] +/// Execute `f` with the particle spec from a compiled backend. +fn with_particle_backend_plan( + backend: &CompiledRateBackend, + f: impl FnOnce(&crate::api::ParticleSpec) -> InfotheoryResult, +) -> InfotheoryResult { + expect_plan_ref!( + backend.plan(), + crate::spec::core::RateBackendPlan::Particle { spec }, + "particle kernel used with non-particle plan" + ); + f(spec) +} + +#[cfg(feature = "backend-ctw")] +/// Execute `f` with CTW depth from a compiled backend. +fn with_ctw_backend_plan( + backend: &CompiledRateBackend, + f: impl FnOnce(usize) -> InfotheoryResult, +) -> InfotheoryResult { + expect_plan_ref!( + backend.plan(), + crate::spec::core::RateBackendPlan::Ctw { depth }, + "ctw kernel used with non-ctw plan" + ); + f(*depth) +} + +#[cfg(feature = "backend-ctw")] +/// Execute `f` with FAC-CTW `(base_depth, encoding_bits)` from a compiled backend. +fn with_fac_ctw_backend_plan( + backend: &CompiledRateBackend, + f: impl FnOnce(usize, usize, bool) -> InfotheoryResult, +) -> InfotheoryResult { + expect_plan_ref!( + backend.plan(), + crate::spec::core::RateBackendPlan::FacCtw { + base_depth, + num_percept_bits: _, + encoding_bits, + msb_first, + }, + "fac-ctw kernel used with non-fac-ctw plan" + ); + f(*base_depth, *encoding_bits, *msb_first) +} + +/// Execute `f` with the ZPAQ compression method from a compiled compression backend. +fn with_zpaq_compression_plan( + backend: &CompiledCompressionBackend, + f: impl FnOnce(&str, usize) -> Result, +) -> Result { + expect_plan_ref!( + backend.plan(), + crate::spec::core::CompressionBackendPlan::Zpaq { method, threads }, + "zpaq compression kernel used with non-zpaq plan" + ); + f(method, *threads) +} + +/// Execute `f` with `(rate_backend, coder, framing)` from a rate-coded compression backend. +fn with_rate_compression_plan( + backend: &CompiledCompressionBackend, + f: impl FnOnce( + &Arc, + crate::coders::CoderType, + crate::compression::FramingMode, + ) -> Result, +) -> Result { + expect_plan_ref!( + backend.plan(), + crate::spec::core::CompressionBackendPlan::Rate { + rate_backend, + coder, + framing, + }, + "rate compression kernel used with non-rate compression plan" + ); + f(rate_backend, *coder, *framing) +} + +#[cfg(feature = "backend-rwkv")] +/// Execute `f` with RWKV compression `(method, parsed_method, coder)` from a compiled backend. +fn with_rwkv_compression_plan( + backend: &CompiledCompressionBackend, + f: impl FnOnce(&str, &crate::rwkvzip::MethodSpec, crate::coders::CoderType) -> Result, +) -> Result { + expect_plan_ref!( + backend.plan(), + crate::spec::core::CompressionBackendPlan::Rwkv7 { + method, + parsed_method, + coder, + .. + }, + "rwkv compression kernel used with non-rwkv compression plan" + ); + f(method, parsed_method, *coder) +} + +fn entropy_prequential(data: &[u8], backend: &CompiledRateBackend) -> InfotheoryResult { + crate::try_prequential_rate_backend(data, &[], backend) +} + +fn joint_entropy_prequential( + x: &[u8], + y: &[u8], + backend: &CompiledRateBackend, +) -> InfotheoryResult { + if x.is_empty() || y.is_empty() { + return Ok(0.0); + } + let joint = interleave_aligned_bytes(x, y); + entropy_prequential(&joint, backend).map(|bits| bits * 2.0) +} + +fn conditional_chain_prequential( + prefix_parts: &[&[u8]], + data: &[u8], + backend: &CompiledRateBackend, +) -> InfotheoryResult { + crate::try_prequential_rate_backend(data, prefix_parts, backend) +} + +#[cfg(feature = "backend-rosa")] +fn entropy_rosa(data: &[u8], backend: &CompiledRateBackend) -> InfotheoryResult { + if data.is_empty() { + return Ok(0.0); + } + let crate::spec::core::RateBackendPlan::RosaPlus { max_order } = backend.plan() else { + unreachable!("rosa kernel used with non-rosa plan") + }; + let mut model = RosaPlus::new(*max_order, false, 0, 42); + Ok(model.predictive_entropy_rate(data)) +} + +#[cfg(feature = "backend-rosa")] +fn joint_entropy_rosa(x: &[u8], y: &[u8], backend: &CompiledRateBackend) -> InfotheoryResult { + if x.is_empty() || y.is_empty() { + return Ok(0.0); + } + let crate::spec::core::RateBackendPlan::RosaPlus { max_order } = backend.plan() else { + unreachable!("rosa kernel used with non-rosa plan") + }; + let joint_symbols: Vec = (0..x.len()) + .map(|idx| (x[idx] as u32) * 256 + (y[idx] as u32)) + .collect(); + let mut model = RosaPlus::new(*max_order, false, 0, 42); + Ok(model.entropy_rate_cps(&joint_symbols)) +} + +#[cfg(feature = "backend-rosa")] +fn conditional_chain_rosa( + prefix_parts: &[&[u8]], + data: &[u8], + backend: &CompiledRateBackend, +) -> InfotheoryResult { + crate::try_frozen_plugin_rate_backend(data, prefix_parts, backend) +} + +#[cfg(feature = "backend-rwkv")] +fn entropy_rwkv(data: &[u8], backend: &CompiledRateBackend) -> InfotheoryResult { + with_rwkv_backend_plan(backend, |method, parsed_method| { + crate::with_rwkv_method_spec_tls(method, parsed_method, |c| { + c.cross_entropy(data).map_err(|err| { + InfotheoryError::runtime(format!("rwkv method entropy scoring failed: {err:#}")) + }) + }) + }) +} + +#[cfg(feature = "backend-rwkv")] +fn joint_entropy_rwkv(x: &[u8], y: &[u8], backend: &CompiledRateBackend) -> InfotheoryResult { + with_rwkv_backend_plan(backend, |method, parsed_method| { + crate::with_rwkv_method_spec_tls(method, parsed_method, |c| { + c.joint_cross_entropy_aligned_min(x, y).map_err(|err| { + InfotheoryError::runtime(format!( + "rwkv method joint-entropy scoring failed: {err:#}" + )) + }) + }) + }) +} + +#[cfg(feature = "backend-rwkv")] +fn conditional_chain_rwkv( + prefix_parts: &[&[u8]], + data: &[u8], + backend: &CompiledRateBackend, +) -> InfotheoryResult { + with_rwkv_backend_plan(backend, |method, parsed_method| { + crate::with_rwkv_method_spec_tls(method, parsed_method, |c| { + c.cross_entropy_conditional_chain(prefix_parts, data) + .map_err(|err| { + InfotheoryError::runtime(format!( + "rwkv method conditional-chain scoring failed: {err:#}" + )) + }) + }) + }) +} + +#[cfg(feature = "backend-mamba")] +fn entropy_mamba(data: &[u8], backend: &CompiledRateBackend) -> InfotheoryResult { + with_mamba_backend_plan(backend, |method, parsed_method| { + crate::with_mamba_method_spec_tls(method, parsed_method, |c| { + c.cross_entropy(data).map_err(|err| { + InfotheoryError::runtime(format!("mamba method entropy scoring failed: {err:#}")) + }) + }) + }) +} + +#[cfg(feature = "backend-mamba")] +fn joint_entropy_mamba(x: &[u8], y: &[u8], backend: &CompiledRateBackend) -> InfotheoryResult { + with_mamba_backend_plan(backend, |method, parsed_method| { + crate::with_mamba_method_spec_tls(method, parsed_method, |c| { + c.joint_cross_entropy_aligned_min(x, y).map_err(|err| { + InfotheoryError::runtime(format!( + "mamba method joint-entropy scoring failed: {err:#}" + )) + }) + }) + }) +} + +#[cfg(feature = "backend-mamba")] +fn conditional_chain_mamba( + prefix_parts: &[&[u8]], + data: &[u8], + backend: &CompiledRateBackend, +) -> InfotheoryResult { + with_mamba_backend_plan(backend, |method, parsed_method| { + crate::with_mamba_method_spec_tls(method, parsed_method, |c| { + c.cross_entropy_conditional_chain(prefix_parts, data) + .map_err(|err| { + InfotheoryError::runtime(format!( + "mamba method conditional-chain scoring failed: {err:#}" + )) + }) + }) + }) +} + +#[cfg(feature = "backend-zpaq")] +fn entropy_zpaq(data: &[u8], backend: &CompiledRateBackend) -> InfotheoryResult { + with_zpaq_backend_plan(backend, |method| { + zpaq_conditional_chain_rate_bits(method, &[], data) + }) +} + +#[cfg(feature = "backend-zpaq")] +fn joint_entropy_zpaq(x: &[u8], y: &[u8], backend: &CompiledRateBackend) -> InfotheoryResult { + with_zpaq_backend_plan(backend, |method| zpaq_joint_entropy_rate_bits(method, x, y)) +} + +#[cfg(feature = "backend-zpaq")] +fn conditional_chain_zpaq( + prefix_parts: &[&[u8]], + data: &[u8], + backend: &CompiledRateBackend, +) -> InfotheoryResult { + with_zpaq_backend_plan(backend, |method| { + zpaq_conditional_chain_rate_bits(method, prefix_parts, data) + }) +} + +#[cfg(feature = "backend-mixture")] +fn entropy_mixture(data: &[u8], backend: &CompiledRateBackend) -> InfotheoryResult { + mixture_entropy_rate_bits(data, backend) +} + +#[cfg(feature = "backend-mixture")] +fn joint_entropy_mixture( + x: &[u8], + y: &[u8], + backend: &CompiledRateBackend, +) -> InfotheoryResult { + mixture_joint_entropy_rate_bits(x, y, backend) +} + +#[cfg(feature = "backend-mixture")] +fn conditional_chain_mixture( + prefix_parts: &[&[u8]], + data: &[u8], + backend: &CompiledRateBackend, +) -> InfotheoryResult { + mixture_conditional_chain_rate_bits(prefix_parts, data, backend) +} + +#[cfg(feature = "backend-particle")] +fn entropy_particle(data: &[u8], backend: &CompiledRateBackend) -> InfotheoryResult { + with_particle_backend_plan(backend, |spec| { + particle_stream_entropy_rate_bits(data, spec) + }) +} + +#[cfg(feature = "backend-particle")] +fn joint_entropy_particle( + x: &[u8], + y: &[u8], + backend: &CompiledRateBackend, +) -> InfotheoryResult { + with_particle_backend_plan(backend, |spec| particle_joint_entropy_rate_bits(x, y, spec)) +} + +#[cfg(feature = "backend-particle")] +fn conditional_chain_particle( + prefix_parts: &[&[u8]], + data: &[u8], + backend: &CompiledRateBackend, +) -> InfotheoryResult { + with_particle_backend_plan(backend, |spec| { + particle_conditional_chain_rate_bits(prefix_parts, data, spec) + }) +} + +#[cfg(feature = "backend-ctw")] +fn entropy_ctw(data: &[u8], backend: &CompiledRateBackend) -> InfotheoryResult { + with_ctw_backend_plan(backend, |depth| ctw_entropy_rate_bits(depth, data)) +} + +#[cfg(feature = "backend-ctw")] +fn joint_entropy_ctw(x: &[u8], y: &[u8], backend: &CompiledRateBackend) -> InfotheoryResult { + with_ctw_backend_plan(backend, |depth| ctw_joint_entropy_rate_bits(depth, x, y)) +} + +#[cfg(feature = "backend-ctw")] +fn conditional_chain_ctw( + prefix_parts: &[&[u8]], + data: &[u8], + backend: &CompiledRateBackend, +) -> InfotheoryResult { + with_ctw_backend_plan(backend, |depth| { + ctw_conditional_chain_rate_bits(depth, prefix_parts, data) + }) +} + +#[cfg(feature = "backend-ctw")] +fn entropy_fac_ctw(data: &[u8], backend: &CompiledRateBackend) -> InfotheoryResult { + with_fac_ctw_backend_plan(backend, |base_depth, encoding_bits, msb_first| { + fac_ctw_entropy_rate_bits(base_depth, encoding_bits, msb_first, data) + }) +} + +#[cfg(feature = "backend-ctw")] +fn joint_entropy_fac_ctw( + x: &[u8], + y: &[u8], + backend: &CompiledRateBackend, +) -> InfotheoryResult { + with_fac_ctw_backend_plan(backend, |base_depth, encoding_bits, msb_first| { + fac_ctw_joint_entropy_rate_bits(base_depth, encoding_bits, msb_first, x, y) + }) +} + +#[cfg(feature = "backend-ctw")] +fn conditional_chain_fac_ctw( + prefix_parts: &[&[u8]], + data: &[u8], + backend: &CompiledRateBackend, +) -> InfotheoryResult { + with_fac_ctw_backend_plan(backend, |base_depth, encoding_bits, msb_first| { + fac_ctw_conditional_chain_rate_bits( + base_depth, + encoding_bits, + msb_first, + prefix_parts, + data, + ) + }) +} + +fn build_compression_runtime_zpaq( + backend: &CompiledCompressionBackend, +) -> Result { + with_zpaq_compression_plan(backend, |method, threads| { + Ok(CompressionRuntimeHandle::Zpaq { + method: method.to_string(), + threads, + }) + }) +} + +#[cfg(feature = "backend-rwkv")] +fn build_compression_runtime_rwkv( + backend: &CompiledCompressionBackend, +) -> Result { + with_rwkv_compression_plan(backend, |method, parsed_method, coder| { + Ok(CompressionRuntimeHandle::Rwkv7 { + method: method.to_string(), + parsed_method: parsed_method.clone(), + coder, + }) + }) +} + +fn build_compression_runtime_rate( + backend: &CompiledCompressionBackend, +) -> Result { + with_rate_compression_plan(backend, |rate_backend, coder, framing| { + Ok(CompressionRuntimeHandle::Rate { + rate_backend: crate::spec::core::compiled_rate_backend_from_plan(rate_backend.clone()) + .map_err(|err| format!("failed to compile rate compression backend plan: {err}"))?, + coder, + framing, + }) + }) +} + +fn build_rate_backend_predictor_via_kernel( + backend: &CompiledRateBackend, + min_prob: f64, +) -> Result { + (rate_backend_kernel(backend.plan().kind()).build_predictor)(backend, min_prob) +} + +fn build_rate_backend_binary_token_predictor_via_kernel( + backend: &CompiledRateBackend, + min_prob: f64, +) -> Result { + (rate_backend_kernel(backend.plan().kind()).build_binary_token_predictor)(backend, min_prob) +} + +fn build_rate_pdf_predictor_via_kernel( + backend: &CompiledRateBackend, +) -> anyhow::Result { + (rate_backend_kernel(backend.plan().kind()).build_pdf_predictor)(backend) +} + +#[cfg(feature = "backend-calibrated")] +pub(super) fn compile_calibrated_base_backend( + base: &Arc, +) -> Result { + crate::spec::core::compiled_rate_backend_from_plan(base.clone()) + .map_err(|err| format!("failed to compile calibrated base backend plan: {err}")) +} + +/// Shared runtime trait for compression-capable backends. +pub trait CompressionRuntime { + /// Compressed size of a single byte slice. + fn compress_size(&mut self, data: &[u8]) -> InfotheoryResult; + + /// Compressed size of chained slices encoded as one stream. + fn compress_size_chain(&mut self, parts: &[&[u8]]) -> InfotheoryResult; + + /// Encode raw bytes. + fn compress_bytes(&mut self, data: &[u8]) -> InfotheoryResult>; + + /// Decode previously encoded bytes. + fn decompress_bytes(&mut self, input: &[u8]) -> InfotheoryResult>; +} + +/// Shared runtime factory trait for compression backends. +pub trait CompressionFactory { + /// Runtime type produced by this factory. + type Runtime: CompressionRuntime; + + /// Build a compression runtime from a spec object. + fn build_compression_runtime(&self) -> Result; +} + +struct SliceChainReader<'a> { + parts: &'a [&'a [u8]], + i: usize, + off: usize, +} + +impl<'a> SliceChainReader<'a> { + fn new(parts: &'a [&'a [u8]]) -> Self { + Self { + parts, + i: 0, + off: 0, + } + } +} + +impl<'a> std::io::Read for SliceChainReader<'a> { + fn read(&mut self, mut buf: &mut [u8]) -> std::io::Result { + let mut total = 0; + if buf.is_empty() { + return Ok(0); + } + while self.i < self.parts.len() { + let p = self.parts[self.i]; + if self.off >= p.len() { + self.i += 1; + self.off = 0; + continue; + } + let n = (p.len() - self.off).min(buf.len()); + buf[..n].copy_from_slice(&p[self.off..self.off + n]); + self.off += n; + total += n; + let tmp = buf; + buf = &mut tmp[n..]; + if buf.is_empty() { + break; + } + } + Ok(total) + } +} + +/// Concrete compression runtime handle built from a [`CompressionBackend`] spec. +pub enum CompressionRuntimeHandle { + Zpaq { + method: String, + threads: usize, + }, + #[cfg(feature = "backend-rwkv")] + Rwkv7 { + method: String, + parsed_method: crate::rwkvzip::MethodSpec, + coder: crate::coders::CoderType, + }, + Rate { + rate_backend: CompiledRateBackend, + coder: crate::coders::CoderType, + framing: crate::compression::FramingMode, + }, +} + +impl CompressionRuntime for CompressionRuntimeHandle { + fn compress_size(&mut self, data: &[u8]) -> InfotheoryResult { + match self { + CompressionRuntimeHandle::Zpaq { method, threads } => { + if *threads <= 1 { + crate::try_zpaq_compress_size_bytes(data, method.as_str()) + } else { + crate::try_zpaq_compress_size_parallel_bytes(data, method.as_str(), *threads) + } + } + #[cfg(feature = "backend-rwkv")] + CompressionRuntimeHandle::Rwkv7 { + method, + parsed_method, + coder, + } => crate::with_rwkv_method_spec_tls(method, parsed_method, |c| { + c.compress_size(data, *coder).map_err(|err| { + InfotheoryError::runtime(format!("rwkv7 compression failed: {err:#}")) + }) + }), + CompressionRuntimeHandle::Rate { + rate_backend, + coder, + framing, + } => crate::compression::compress_rate_size(data, rate_backend, *coder, *framing) + .map_err(|err| { + InfotheoryError::runtime(format!("rate-coded compression failed: {err:#}")) + }), + } + } + + fn compress_size_chain(&mut self, parts: &[&[u8]]) -> InfotheoryResult { + match self { + CompressionRuntimeHandle::Zpaq { method, threads } => { + let reader = SliceChainReader::new(parts); + if *threads <= 1 { + crate::try_zpaq_compress_size_stream(reader, method.as_str()) + } else { + crate::try_zpaq_compress_size_stream_parallel(reader, method.as_str(), *threads) + } + } + #[cfg(feature = "backend-rwkv")] + CompressionRuntimeHandle::Rwkv7 { + method, + parsed_method, + coder, + } => crate::with_rwkv_method_spec_tls(method, parsed_method, |c| { + c.compress_size_chain(parts, *coder).map_err(|err| { + InfotheoryError::runtime(format!("rwkv7 chain compression failed: {err:#}")) + }) + }), + CompressionRuntimeHandle::Rate { + rate_backend, + coder, + framing, + } => { + crate::compression::compress_rate_size_chain(parts, rate_backend, *coder, *framing) + .map_err(|err| { + InfotheoryError::runtime(format!( + "rate-coded chain compression failed: {err:#}" + )) + }) + } + } + } + + fn compress_bytes(&mut self, data: &[u8]) -> InfotheoryResult> { + match self { + CompressionRuntimeHandle::Zpaq { method, .. } => { + crate::zpaq_compress_to_vec(data, method).map_err(|err| { + InfotheoryError::runtime(format!("zpaq byte compression failed: {err:#}")) + }) + } + #[cfg(feature = "backend-rwkv")] + CompressionRuntimeHandle::Rwkv7 { + method, + parsed_method, + coder, + } => crate::with_rwkv_method_spec_tls(method, parsed_method, |c| { + c.compress(data, *coder) + }) + .map_err(|err| { + InfotheoryError::runtime(format!("rwkv7 byte compression failed: {err:#}")) + }), + CompressionRuntimeHandle::Rate { + rate_backend, + coder, + framing, + } => crate::compression::compress_rate_bytes(data, rate_backend, *coder, *framing) + .map_err(|err| { + InfotheoryError::runtime(format!("rate-coded byte compression failed: {err:#}")) + }), + } + } + + fn decompress_bytes(&mut self, input: &[u8]) -> InfotheoryResult> { + match self { + CompressionRuntimeHandle::Zpaq { .. } => { + crate::zpaq_decompress_to_vec(input).map_err(|err| { + InfotheoryError::runtime(format!("zpaq decompression failed: {err:#}")) + }) + } + #[cfg(feature = "backend-rwkv")] + CompressionRuntimeHandle::Rwkv7 { + method, + parsed_method, + .. + } => crate::with_rwkv_method_spec_tls(method, parsed_method, |c| c.decompress(input)) + .map_err(|err| { + InfotheoryError::runtime(format!("rwkv7 decompression failed: {err:#}")) + }), + CompressionRuntimeHandle::Rate { + rate_backend, + coder, + framing, + } => crate::compression::decompress_rate_bytes(input, rate_backend, *coder, *framing) + .map_err(|err| { + InfotheoryError::runtime(format!("rate-coded decompression failed: {err:#}")) + }), + } + } +} + +impl CompressionFactory for CompiledCompressionBackend { + type Runtime = CompressionRuntimeHandle; + + fn build_compression_runtime(&self) -> Result { + (compression_backend_kernel(self.plan().kind()).build_runtime)(self) + } +} + +pub(crate) fn try_describe_rate_backend( + backend: &RateBackend, +) -> Result<&'static RateBackendDescriptor, String> { + describe_rate_backend_kind(backend.kind()) +} + +pub(crate) fn try_describe_compression_backend( + backend: &CompressionBackend, +) -> Result<&'static CompressionBackendDescriptor, String> { + describe_compression_backend_kind(backend.kind()) +} + +/// Shared spec -> predictor runtime builder using the default probability floor. +pub(crate) fn build_rate_backend_predictor( + backend: &CompiledRateBackend, + min_prob: f64, +) -> Result { + build_rate_backend_predictor_via_kernel(backend, min_prob) +} + +/// Shared spec -> predictor runtime builder using the library's default probability floor. +pub(crate) fn build_rate_backend_predictor_default( + backend: &CompiledRateBackend, +) -> Result { + build_rate_backend_predictor(backend, crate::mixture::DEFAULT_MIN_PROB) +} + +/// Shared spec -> predictor runtime builder for binary-token applications. +/// +/// Native bit backends expose their true bit-token predictors here. Byte-native +/// backends are adapted by observing the literal byte symbols `0` and `1`, +/// with downstream callers normalizing those two log-probabilities into a +/// binary prediction. +pub(crate) fn build_rate_backend_binary_token_predictor( + backend: &CompiledRateBackend, + min_prob: f64, +) -> Result { + build_rate_backend_binary_token_predictor_via_kernel(backend, min_prob) +} + +/// Shared spec -> compression predictor runtime builder. +pub(crate) fn build_rate_pdf_predictor( + backend: &CompiledRateBackend, +) -> anyhow::Result { + build_rate_pdf_predictor_via_kernel(backend) +} + +/// Shared spec -> compression runtime builder. +pub(crate) fn build_compression_runtime( + backend: &CompiledCompressionBackend, +) -> Result { + backend.build_compression_runtime() +} + +fn interleave_aligned_bytes(x: &[u8], y: &[u8]) -> Vec { + let mut joint = Vec::with_capacity(x.len() * 2); + for (&xb, &yb) in x.iter().zip(y.iter()) { + joint.push(xb); + joint.push(yb); + } + joint +} + +#[cfg(feature = "backend-zpaq")] +fn zpaq_conditional_chain_rate_bits( + method: &str, + prefix_parts: &[&[u8]], + data: &[u8], +) -> InfotheoryResult { + if data.is_empty() { + return Ok(0.0); + } + let mut model = ZpaqRateModel::new(method.to_owned(), 2f64.powi(-24)); + for &part in prefix_parts { + model.update_and_score(part); + } + let bits = model.update_and_score(data); + Ok(bits / (data.len() as f64)) +} + +#[cfg(feature = "backend-zpaq")] +fn zpaq_joint_entropy_rate_bits(method: &str, x: &[u8], y: &[u8]) -> InfotheoryResult { + if x.is_empty() || y.is_empty() { + return Ok(0.0); + } + let joint = interleave_aligned_bytes(x, y); + let mut model = ZpaqRateModel::new(method.to_owned(), 2f64.powi(-24)); + let bits = model.update_and_score(&joint); + Ok(bits / (x.len() as f64)) +} + +#[cfg(feature = "backend-mixture")] +fn build_compiled_mixture_runtime( + backend: &CompiledRateBackend, +) -> Result { + let experts = crate::mixture::expert_configs_from_compiled_mixture(backend)?; + crate::mixture::build_mixture_runtime_from_compiled(backend, &experts).map_err(|err| { + InfotheoryError::invalid_backend_config(format!("MixtureSpec invalid: {err}")) + }) +} + +#[cfg(feature = "backend-mixture")] +fn mixture_entropy_rate_bits(data: &[u8], backend: &CompiledRateBackend) -> InfotheoryResult { + if data.is_empty() { + return Ok(0.0); + } + let mut mix = build_compiled_mixture_runtime(backend)?; + mix.begin_stream(Some(data.len() as u64)) + .map_err(|err| InfotheoryError::runtime(format!("Mixture stream init failed: {err}")))?; + let mut bits = 0.0; + for &byte in data { + bits -= mix.step(byte) / std::f64::consts::LN_2; + } + mix.finish_stream().map_err(|err| { + InfotheoryError::runtime(format!("Mixture stream finalize failed: {err}")) + })?; + Ok(bits / (data.len() as f64)) +} + +#[cfg(feature = "backend-mixture")] +fn mixture_joint_entropy_rate_bits( + x: &[u8], + y: &[u8], + backend: &CompiledRateBackend, +) -> InfotheoryResult { + if x.is_empty() || y.is_empty() { + return Ok(0.0); + } + let joint = interleave_aligned_bytes(x, y); + let mut mix = build_compiled_mixture_runtime(backend)?; + mix.begin_stream(Some(joint.len() as u64)) + .map_err(|err| InfotheoryError::runtime(format!("Mixture stream init failed: {err}")))?; + let mut bits = 0.0; + for &byte in &joint { + bits -= mix.step(byte) / std::f64::consts::LN_2; + } + mix.finish_stream().map_err(|err| { + InfotheoryError::runtime(format!("Mixture stream finalize failed: {err}")) + })?; + Ok(bits / (x.len() as f64)) +} + +#[cfg(feature = "backend-mixture")] +fn mixture_conditional_chain_rate_bits( + prefix_parts: &[&[u8]], + data: &[u8], + backend: &CompiledRateBackend, +) -> InfotheoryResult { + if data.is_empty() { + return Ok(0.0); + } + let mut mix = build_compiled_mixture_runtime(backend)?; + let total = prefix_parts + .iter() + .map(|part| part.len() as u64) + .sum::() + .saturating_add(data.len() as u64); + mix.begin_stream(Some(total)) + .map_err(|err| InfotheoryError::runtime(format!("Mixture stream init failed: {err}")))?; + for &part in prefix_parts { + for &byte in part { + mix.step(byte); + } + } + let mut bits = 0.0; + for &byte in data { + bits -= mix.step(byte) / std::f64::consts::LN_2; + } + Ok(bits / (data.len() as f64)) +} + +#[cfg(feature = "backend-particle")] +fn particle_stream_entropy_rate_bits( + data: &[u8], + spec: &crate::api::ParticleSpec, +) -> InfotheoryResult { + if data.is_empty() { + return Ok(0.0); + } + let mut runtime = ParticleRuntime::new(spec); + let mut bits = 0.0; + for &byte in data { + bits -= runtime.step(byte) / std::f64::consts::LN_2; + } + Ok(bits / (data.len() as f64)) +} + +#[cfg(feature = "backend-particle")] +fn particle_conditional_chain_rate_bits( + prefix_parts: &[&[u8]], + data: &[u8], + spec: &crate::api::ParticleSpec, +) -> InfotheoryResult { + if data.is_empty() { + return Ok(0.0); + } + let mut runtime = ParticleRuntime::new(spec); + for &part in prefix_parts { + for &byte in part { + runtime.step(byte); + } + } + let mut bits = 0.0; + for &byte in data { + bits -= runtime.step(byte) / std::f64::consts::LN_2; + } + Ok(bits / (data.len() as f64)) +} + +#[cfg(feature = "backend-particle")] +fn particle_joint_entropy_rate_bits( + x: &[u8], + y: &[u8], + spec: &crate::api::ParticleSpec, +) -> InfotheoryResult { + if x.is_empty() || y.is_empty() { + return Ok(0.0); + } + let joint = interleave_aligned_bytes(x, y); + let mut runtime = ParticleRuntime::new(spec); + let mut bits = 0.0; + for &byte in &joint { + bits -= runtime.step(byte) / std::f64::consts::LN_2; + } + Ok(bits / (x.len() as f64)) +} + +#[cfg(feature = "backend-ctw")] +#[inline] +fn ctw_byte_bit_msb(byte: u8, bit_idx: usize) -> bool { + ctw_symbol_bit_msb(byte, 8, bit_idx) +} + +#[cfg(feature = "backend-ctw")] +#[inline] +fn ctw_update_byte_msb(tree: &mut ContextTree, byte: u8) { + for bit_idx in 0..8usize { + tree.update(ctw_byte_bit_msb(byte, bit_idx)); + } +} + +#[cfg(feature = "backend-ctw")] +fn ctw_entropy_rate_bits(depth: usize, data: &[u8]) -> InfotheoryResult { + if data.is_empty() { + return Ok(0.0); + } + let mut tree = ContextTree::new(depth); + for &byte in data { + ctw_update_byte_msb(&mut tree, byte); + } + let ln_p = tree.get_log_block_probability(); + Ok((-ln_p / std::f64::consts::LN_2) / (data.len() as f64)) +} + +#[cfg(feature = "backend-ctw")] +fn fac_ctw_entropy_rate_bits( + base_depth: usize, + encoding_bits: usize, + msb_first: bool, + data: &[u8], +) -> InfotheoryResult { + if data.is_empty() { + return Ok(0.0); + } + let bits_per_byte = encoding_bits; + let mut fac = FacContextTree::new(base_depth, bits_per_byte); + fac.reserve_for_symbols(data.len()); + for &byte in data { + if msb_first { + fac.update_byte_msb(byte); + } else { + fac.update_byte_lsb(byte); + } + } + let ln_p = fac.get_log_block_probability(); + Ok((-ln_p / std::f64::consts::LN_2) / (data.len() as f64)) +} + +#[cfg(feature = "backend-ctw")] +fn ctw_joint_entropy_rate_bits(depth: usize, x: &[u8], y: &[u8]) -> InfotheoryResult { + let mut tree = ContextTree::new(depth); + for (&xb, &yb) in x.iter().zip(y.iter()) { + ctw_update_byte_msb(&mut tree, xb); + ctw_update_byte_msb(&mut tree, yb); + } + let ln_p = tree.get_log_block_probability(); + Ok((-ln_p / std::f64::consts::LN_2) / (x.len() as f64)) +} + +#[cfg(feature = "backend-ctw")] +fn fac_ctw_joint_entropy_rate_bits( + base_depth: usize, + encoding_bits: usize, + msb_first: bool, + x: &[u8], + y: &[u8], +) -> InfotheoryResult { + let bits_per_byte = encoding_bits; + let mut fac = FacContextTree::new(base_depth, bits_per_byte * 2); + for (&xb, &yb) in x.iter().zip(y.iter()) { + for idx in 0..bits_per_byte { + let bit_idx_x = idx * 2; + let bit_idx_y = bit_idx_x + 1; + let x_bit = if msb_first { + ctw_symbol_bit_msb(xb, bits_per_byte, idx) + } else { + ((xb >> idx) & 1) == 1 + }; + let y_bit = if msb_first { + ctw_symbol_bit_msb(yb, bits_per_byte, idx) + } else { + ((yb >> idx) & 1) == 1 + }; + fac.update(x_bit, bit_idx_x); + fac.update(y_bit, bit_idx_y); + } + } + let ln_p = fac.get_log_block_probability(); + Ok((-ln_p / std::f64::consts::LN_2) / (x.len() as f64)) +} + +#[cfg(feature = "backend-ctw")] +fn ctw_conditional_chain_rate_bits( + depth: usize, + prefix_parts: &[&[u8]], + data: &[u8], +) -> InfotheoryResult { + if data.is_empty() { + return Ok(0.0); + } + let mut tree = ContextTree::new(depth); + for &part in prefix_parts { + for &byte in part { + ctw_update_byte_msb(&mut tree, byte); + } + } + let log_p_prefix = tree.get_log_block_probability(); + for &byte in data { + ctw_update_byte_msb(&mut tree, byte); + } + let log_p_joint = tree.get_log_block_probability(); + let bits = -(log_p_joint - log_p_prefix) / std::f64::consts::LN_2; + Ok(bits / (data.len() as f64)) +} + +#[cfg(feature = "backend-ctw")] +fn fac_ctw_conditional_chain_rate_bits( + base_depth: usize, + encoding_bits: usize, + msb_first: bool, + prefix_parts: &[&[u8]], + data: &[u8], +) -> InfotheoryResult { + if data.is_empty() { + return Ok(0.0); + } + let bits_per_byte = encoding_bits; + let mut fac = FacContextTree::new(base_depth, bits_per_byte); + for &part in prefix_parts { + for &byte in part { + for idx in 0..bits_per_byte { + let bit = if msb_first { + ctw_symbol_bit_msb(byte, bits_per_byte, idx) + } else { + ((byte >> idx) & 1) == 1 + }; + fac.update(bit, idx); + } + } + } + let log_p_prefix = fac.get_log_block_probability(); + for &byte in data { + for idx in 0..bits_per_byte { + let bit = if msb_first { + ctw_symbol_bit_msb(byte, bits_per_byte, idx) + } else { + ((byte >> idx) & 1) == 1 + }; + fac.update(bit, idx); + } + } + let log_p_joint = fac.get_log_block_probability(); + let bits = -(log_p_joint - log_p_prefix) / std::f64::consts::LN_2; + Ok(bits / (data.len() as f64)) +} + +fn execute_entropy_rate_backend( + data: &[u8], + backend: &CompiledRateBackend, +) -> InfotheoryResult { + (rate_backend_kernel(backend.plan().kind()).entropy_rate)(data, backend) +} + +pub(crate) fn try_entropy_rate_backend_direct( + data: &[u8], + backend: &CompiledRateBackend, +) -> InfotheoryResult { + execute_entropy_rate_backend(data, backend) +} + +pub(crate) fn try_cross_entropy_rate_backend_direct( + test_data: &[u8], + train_data: &[u8], + backend: &CompiledRateBackend, +) -> InfotheoryResult { + if backend.plan().kind() == RateBackendKind::Zpaq { + return (rate_backend_kernel(backend.plan().kind()).conditional_chain_rate)( + &[train_data], + test_data, + backend, + ); + } + crate::try_frozen_plugin_rate_backend(test_data, &[train_data], backend) +} + +pub(crate) fn try_joint_entropy_rate_backend_direct( + x: &[u8], + y: &[u8], + backend: &CompiledRateBackend, +) -> InfotheoryResult { + if x.is_empty() || y.is_empty() { + return Ok(0.0); + } + let n = x.len().min(y.len()); + let x = &x[..n]; + let y = &y[..n]; + + (rate_backend_kernel(backend.plan().kind()).joint_entropy_rate)(x, y, backend) +} + +pub(crate) fn try_cross_entropy_conditional_chain_backend( + prefix_parts: &[&[u8]], + data: &[u8], + backend: &CompiledRateBackend, +) -> InfotheoryResult { + (rate_backend_kernel(backend.plan().kind()).conditional_chain_rate)(prefix_parts, data, backend) +} + +pub(crate) fn capability_always_true(_plan: &crate::spec::core::RateBackendPlan) -> bool { + true +} + +pub(crate) fn capability_always_false(_plan: &crate::spec::core::RateBackendPlan) -> bool { + false +} + +pub(crate) fn mixture_supports_native_bit_prediction( + plan: &crate::spec::core::RateBackendPlan, +) -> bool { + let crate::spec::core::RateBackendPlan::Mixture { experts, .. } = plan else { + unreachable!() + }; + experts.iter().all(|e| { + (crate::runtime::rate_backend_kernel(e.backend.kind()).supports_native_bit_prediction)( + &e.backend, + ) + }) +} + +pub(crate) fn mixture_supports_byte_prefix_mass(plan: &crate::spec::core::RateBackendPlan) -> bool { + let crate::spec::core::RateBackendPlan::Mixture { experts, .. } = plan else { + unreachable!() + }; + experts.iter().all(|e| { + (crate::runtime::rate_backend_kernel(e.backend.kind()).supports_byte_prefix_mass)( + &e.backend, + ) + }) +} + +pub(crate) fn mixture_supports_efficient_byte_packed_bit_sessions( + plan: &crate::spec::core::RateBackendPlan, +) -> bool { + let crate::spec::core::RateBackendPlan::Mixture { experts, .. } = plan else { + unreachable!() + }; + experts.iter().all(|e| { + (crate::runtime::rate_backend_kernel(e.backend.kind()) + .supports_efficient_byte_packed_bit_sessions)(&e.backend) + }) +} + +pub(crate) fn mixture_supports_reversible_bit_updates( + plan: &crate::spec::core::RateBackendPlan, +) -> bool { + let crate::spec::core::RateBackendPlan::Mixture { experts, .. } = plan else { + unreachable!() + }; + experts.iter().all(|e| { + (crate::runtime::rate_backend_kernel(e.backend.kind()).supports_reversible_bit_updates)( + &e.backend, + ) + }) +} + +pub(crate) fn calibrated_supports_native_bit_prediction( + plan: &crate::spec::core::RateBackendPlan, +) -> bool { + let crate::spec::core::RateBackendPlan::Calibrated { base, .. } = plan else { + unreachable!() + }; + (crate::runtime::rate_backend_kernel(base.kind()).supports_native_bit_prediction)(base) +} + +pub(crate) fn calibrated_supports_byte_prefix_mass( + plan: &crate::spec::core::RateBackendPlan, +) -> bool { + let crate::spec::core::RateBackendPlan::Calibrated { base, .. } = plan else { + unreachable!() + }; + (crate::runtime::rate_backend_kernel(base.kind()).supports_byte_prefix_mass)(base) +} + +pub(crate) fn calibrated_supports_efficient_byte_packed_bit_sessions( + plan: &crate::spec::core::RateBackendPlan, +) -> bool { + let crate::spec::core::RateBackendPlan::Calibrated { base, .. } = plan else { + unreachable!() + }; + (crate::runtime::rate_backend_kernel(base.kind()).supports_efficient_byte_packed_bit_sessions)( + base, + ) +} + +pub(crate) fn calibrated_supports_reversible_bit_updates( + plan: &crate::spec::core::RateBackendPlan, +) -> bool { + let crate::spec::core::RateBackendPlan::Calibrated { base, .. } = plan else { + unreachable!() + }; + (crate::runtime::rate_backend_kernel(base.kind()).supports_reversible_bit_updates)(base) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::HashSet; + use std::io::Read; + + #[cfg(feature = "backend-calibrated")] + use crate::api::{CalibratedSpec, CalibrationContextKind}; + use crate::api::{CompressionBackend, RateBackend}; + #[cfg(feature = "backend-mixture")] + use crate::api::{MixtureExpertSpec, MixtureKind, MixtureSpec}; + use crate::mixture::OnlineBytePredictor; + #[cfg(any(feature = "backend-mixture", feature = "backend-calibrated"))] + use std::sync::Arc; + + #[cfg(any( + feature = "backend-ctw", + feature = "backend-zpaq", + feature = "backend-mixture", + feature = "backend-rwkv", + feature = "backend-mamba", + feature = "all-backends" + ))] + fn compiled_rate_backend(backend: &RateBackend) -> CompiledRateBackend { + backend.compile().expect("compiled rate backend") + } + + #[cfg(feature = "backend-ctw")] + fn compiled_compression_backend(backend: &CompressionBackend) -> CompiledCompressionBackend { + backend.compile().expect("compiled compression backend") + } + + fn sample_rate_backend_for_kind(kind: RateBackendKind) -> Option { + match kind { + RateBackendKind::Mixture => { + #[cfg(feature = "backend-mixture")] + { + let base = first_enabled_default_rate_backend_spec()?; + Some(RateBackend::Mixture { + spec: Arc::new(MixtureSpec::new( + MixtureKind::Bayes, + vec![MixtureExpertSpec::new(base)], + )), + }) + } + #[cfg(not(feature = "backend-mixture"))] + { + None + } + } + RateBackendKind::Calibrated => { + #[cfg(feature = "backend-calibrated")] + { + let base = first_enabled_default_rate_backend_spec()?; + Some(RateBackend::Calibrated { + spec: Arc::new(CalibratedSpec::new(base, CalibrationContextKind::Global)), + }) + } + #[cfg(not(feature = "backend-calibrated"))] + { + None + } + } + _ => default_rate_backend_spec(kind), + } + } + + fn assert_registry_is_injective(registry: &[BackendDescriptor], label: &str) + where + K: Copy + Eq + std::fmt::Debug + std::hash::Hash, + { + let mut kinds = HashSet::new(); + let mut aliases = HashSet::new(); + + for descriptor in registry { + assert!( + kinds.insert(descriptor.kind), + "{label} duplicates backend kind {:?}", + descriptor.kind + ); + assert!( + descriptor.aliases.contains(&descriptor.canonical), + "{label} descriptor '{}' must list its canonical alias", + descriptor.canonical + ); + for alias in descriptor.aliases { + assert!( + aliases.insert(*alias), + "{label} alias '{alias}' is assigned to multiple backends" + ); + } + } + } + + #[test] + fn rate_backend_registry_resolves_aliases() { + let rosa = find_backend_descriptor_in_registry(RATE_BACKEND_REGISTRY, "rosa") + .expect("rosa descriptor"); + assert_eq!(rosa.canonical, "rosaplus"); + + let mixture = find_backend_descriptor_in_registry(RATE_BACKEND_REGISTRY, "mixture") + .expect("mixture descriptor"); + assert_eq!(mixture.canonical, "mixture"); + + let missing = find_backend_descriptor_in_registry(RATE_BACKEND_REGISTRY, "mix"); + assert!(missing.is_none(), "legacy alias 'mix' must be rejected"); + } + + #[test] + fn compression_registry_resolves_aliases() { + let ac = find_backend_descriptor_in_registry(COMPRESSION_BACKEND_REGISTRY, "rate-ac") + .expect("rate-ac descriptor"); + assert_eq!(ac.canonical, "rate-ac"); + + let missing = find_backend_descriptor_in_registry(COMPRESSION_BACKEND_REGISTRY, "rate_ac"); + assert!(missing.is_none(), "legacy alias 'rate_ac' must be rejected"); + } + + #[test] + fn rate_backend_registry_has_unique_kinds_and_aliases() { + assert_registry_is_injective(RATE_BACKEND_REGISTRY, "RATE_BACKEND_REGISTRY"); + } + + #[test] + fn compression_backend_registry_has_unique_kinds_and_aliases() { + assert_registry_is_injective(COMPRESSION_BACKEND_REGISTRY, "COMPRESSION_BACKEND_REGISTRY"); + } + + #[test] + fn rate_backend_registry_kinds_have_runtime_kernels() { + let kernel_kinds: HashSet<_> = RATE_BACKEND_KERNELS + .iter() + .map(|kernel| kernel.kind) + .collect(); + for descriptor in RATE_BACKEND_REGISTRY { + assert!( + kernel_kinds.contains(&descriptor.kind), + "missing runtime kernel for rate backend kind {:?}", + descriptor.kind + ); + } + } + + #[test] + fn enabled_rate_backend_catalog_entries_compile_and_build_consistently() { + for descriptor in RATE_BACKEND_REGISTRY + .iter() + .filter(|descriptor| descriptor.enabled) + { + let backend = sample_rate_backend_for_kind(descriptor.kind) + .unwrap_or_else(|| panic!("missing sample backend for {:?}", descriptor.kind)); + let compiled = backend.compile().unwrap_or_else(|err| { + panic!( + "sample backend {:?} failed to compile: {err}", + descriptor.kind + ) + }); + + assert_eq!(compiled.canonical_name(), descriptor.canonical); + assert_eq!(compiled.capabilities().canonical_name, descriptor.canonical); + + let mut predictor = + build_rate_backend_predictor_default(&compiled).unwrap_or_else(|err| { + panic!("failed to build predictor for {:?}: {err}", descriptor.kind) + }); + predictor.begin_stream(Some(1)).unwrap_or_else(|err| { + panic!( + "failed to begin predictor stream for {:?}: {err}", + descriptor.kind + ) + }); + predictor.finish_stream().unwrap_or_else(|err| { + panic!( + "failed to finish predictor stream for {:?}: {err}", + descriptor.kind + ) + }); + + if compiled.supports_rate_coded_compression() { + let _pdf = build_rate_pdf_predictor(&compiled).unwrap_or_else(|err| { + panic!( + "failed to build pdf predictor for {:?}: {err}", + descriptor.kind + ) + }); + } + + let mut bit_predictor = build_rate_backend_binary_token_predictor( + &compiled, + crate::mixture::DEFAULT_MIN_PROB, + ) + .unwrap_or_else(|err| { + panic!( + "failed to build binary-token predictor for {:?}: {err}", + descriptor.kind + ) + }); + let prediction = crate::prediction::binary_prediction_from_log_probs( + bit_predictor.log_prob(0), + bit_predictor.log_prob(1), + crate::mixture::DEFAULT_MIN_PROB, + ); + assert!( + (prediction.p0 + prediction.p1 - 1.0).abs() < 1e-12, + "binary-token predictor for {:?} must normalize to 1.0, got p0={} p1={}", + descriptor.kind, + prediction.p0, + prediction.p1 + ); + bit_predictor.begin_stream(Some(1)).unwrap_or_else(|err| { + panic!( + "failed to begin binary-token stream for {:?}: {err}", + descriptor.kind + ) + }); + bit_predictor.finish_stream().unwrap_or_else(|err| { + panic!( + "failed to finish binary-token stream for {:?}: {err}", + descriptor.kind + ) + }); + } + } + + #[test] + fn compression_backend_registry_kinds_have_runtime_kernels() { + let kernel_kinds: HashSet<_> = COMPRESSION_BACKEND_KERNELS + .iter() + .map(|kernel| kernel.kind) + .collect(); + for descriptor in COMPRESSION_BACKEND_REGISTRY { + assert!( + kernel_kinds.contains(&descriptor.kind), + "missing runtime kernel for compression backend kind {:?}", + descriptor.kind + ); + } + } + + #[test] + fn describe_compression_backend_uses_canonical_lookup_not_positional_indices() { + let ac = try_describe_compression_backend(&CompressionBackend::Rate { + rate_backend: RateBackend::RosaPlus { max_order: -1 }, + coder: crate::coders::CoderType::AC, + framing: crate::compression::FramingMode::Framed, + }) + .expect("descriptor for rate-ac"); + assert_eq!(ac.canonical, "rate-ac"); + + let rans = try_describe_compression_backend(&CompressionBackend::Rate { + rate_backend: RateBackend::RosaPlus { max_order: -1 }, + coder: crate::coders::CoderType::RANS, + framing: crate::compression::FramingMode::Framed, + }) + .expect("descriptor for rate-rans"); + assert_eq!(rans.canonical, "rate-rans"); + } + + #[test] + fn missing_descriptor_reports_registry_mismatch_error() { + let err = registry::backend_descriptor_by_kind_checked( + &[], + RateBackendKind::RosaPlus, + "RATE_BACKEND_REGISTRY", + ) + .expect_err("missing descriptor should return an error"); + assert!(err.contains("internal backend registry mismatch")); + assert!(err.contains("RosaPlus")); + } + + #[test] + fn registry_descriptors_round_trip_and_feature_messages_are_stable() { + let rosa = find_backend_descriptor_in_registry(RATE_BACKEND_REGISTRY, " ROSA ") + .expect("trimmed case-insensitive alias should resolve"); + assert_eq!(rosa.canonical, "rosaplus"); + + for descriptor in RATE_BACKEND_REGISTRY { + let described = describe_rate_backend_kind(descriptor.kind) + .expect("rate descriptor kind lookup must succeed"); + assert_eq!(described.canonical, descriptor.canonical); + } + + for descriptor in COMPRESSION_BACKEND_REGISTRY { + let described = describe_compression_backend_kind(descriptor.kind) + .expect("compression descriptor kind lookup must succeed"); + assert_eq!(described.canonical, descriptor.canonical); + } + + let rate_with_feature = RATE_BACKEND_REGISTRY + .iter() + .find(|descriptor| descriptor.feature.is_some()) + .expect("at least one feature-gated rate backend"); + let rate_feature_message = registry::rate_backend_feature_error(rate_with_feature.kind); + assert!(rate_feature_message.contains(rate_with_feature.canonical)); + assert!(rate_feature_message.contains("requires infotheory feature")); + + if let Some(rate_without_feature) = RATE_BACKEND_REGISTRY + .iter() + .find(|descriptor| descriptor.feature.is_none()) + { + let rate_unavailable_message = + registry::rate_backend_feature_error(rate_without_feature.kind); + assert!(rate_unavailable_message.contains(rate_without_feature.canonical)); + assert!(rate_unavailable_message.contains("is unavailable")); + } + + let compression_with_feature = COMPRESSION_BACKEND_REGISTRY + .iter() + .find(|descriptor| descriptor.feature.is_some()) + .expect("at least one feature-gated compression backend"); + let compression_feature_message = + registry::compression_backend_feature_error(compression_with_feature.kind); + assert!(compression_feature_message.contains(compression_with_feature.canonical)); + assert!(compression_feature_message.contains("requires infotheory feature")); + + let compression_without_feature = COMPRESSION_BACKEND_REGISTRY + .iter() + .find(|descriptor| descriptor.feature.is_none()) + .expect("at least one always-enabled compression backend"); + let compression_unavailable_message = + registry::compression_backend_feature_error(compression_without_feature.kind); + assert!(compression_unavailable_message.contains(compression_without_feature.canonical)); + assert!(compression_unavailable_message.contains("is unavailable")); + } + + #[test] + fn interleave_aligned_bytes_uses_shorter_input_and_preserves_pair_order() { + let interleaved = interleave_aligned_bytes(&[1, 2, 3], &[9, 8]); + assert_eq!(interleaved, vec![1, 9, 2, 8]); + } + + #[test] + fn slice_chain_reader_reads_across_empty_and_nonempty_parts() { + let parts: [&[u8]; 4] = [b"ab", b"", b"c", b"def"]; + let mut reader = SliceChainReader::new(&parts); + let mut out = [0u8; 6]; + + let first = reader.read(&mut out[..3]).expect("first read"); + let second = reader.read(&mut out[3..]).expect("second read"); + let eof = reader.read(&mut out[0..1]).expect("eof read"); + + assert_eq!(first, 3); + assert_eq!(second, 3); + assert_eq!(eof, 0); + assert_eq!(&out, b"abcdef"); + } + + #[cfg(feature = "backend-ctw")] + #[test] + fn rate_runtime_chain_matches_concatenated_stream_and_roundtrips() { + let backend = compiled_compression_backend(&CompressionBackend::Rate { + rate_backend: RateBackend::Ctw { depth: 5 }, + coder: crate::coders::CoderType::AC, + framing: crate::compression::FramingMode::Framed, + }); + let mut runtime = build_compression_runtime(&backend).expect("rate runtime"); + let parts: [&[u8]; 3] = [b"alpha", b"-", b"beta"]; + let joined = b"alpha-beta"; + + let chain_size = runtime.compress_size_chain(&parts).expect("chain size"); + let joined_size = runtime.compress_size(joined).expect("joined size"); + assert_eq!(chain_size, joined_size); + + let encoded = runtime.compress_bytes(joined).expect("compress bytes"); + let decoded = runtime + .decompress_bytes(&encoded) + .expect("decompress bytes"); + assert_eq!(decoded, joined); + } + + #[cfg(feature = "backend-zpaq")] + #[test] + fn zpaq_runtime_helpers_cover_empty_and_conditioned_paths() { + let backend = compiled_rate_backend(&RateBackend::Zpaq { + method: crate::api::ZpaqMethodSpec::literal("2"), + }); + assert_eq!( + zpaq_conditional_chain_rate_bits("2", &[b"prefix"], b"").expect("empty zpaq chain"), + 0.0 + ); + + let entropy = entropy_zpaq(b"banana", &backend).expect("zpaq entropy"); + let joint = joint_entropy_zpaq(b"banana", b"bandit", &backend).expect("zpaq joint"); + let cond = conditional_chain_zpaq(&[b"ban"], b"ana", &backend).expect("zpaq conditional"); + assert!(entropy.is_finite() && entropy >= 0.0); + assert!(joint.is_finite() && joint >= 0.0); + assert!(cond.is_finite() && cond >= 0.0); + } + + #[cfg(feature = "backend-ctw")] + #[test] + fn ctw_family_runtime_entropy_helpers_are_finite() { + let ctw = compiled_rate_backend(&RateBackend::Ctw { depth: 5 }); + let fac = compiled_rate_backend(&RateBackend::FacCtw { + base_depth: 5, + num_percept_bits: 8, + encoding_bits: 8, + msb_first: None, + }); + + let ctw_entropy = entropy_ctw(b"abracadabra", &ctw).expect("ctw entropy"); + let ctw_joint = joint_entropy_ctw(b"aaaa", b"bbbb", &ctw).expect("ctw joint"); + let ctw_cond = conditional_chain_ctw(&[b"abra"], b"cad", &ctw).expect("ctw conditional"); + assert!(ctw_entropy.is_finite() && ctw_entropy >= 0.0); + assert!(ctw_joint.is_finite() && ctw_joint >= 0.0); + assert!(ctw_cond.is_finite() && ctw_cond >= 0.0); + + let fac_entropy = entropy_fac_ctw(b"abracadabra", &fac).expect("fac entropy"); + let fac_joint = joint_entropy_fac_ctw(b"aaaa", b"bbbb", &fac).expect("fac joint"); + let fac_cond = + conditional_chain_fac_ctw(&[b"abra"], b"cad", &fac).expect("fac conditional"); + assert!(fac_entropy.is_finite() && fac_entropy >= 0.0); + assert!(fac_joint.is_finite() && fac_joint >= 0.0); + assert!(fac_cond.is_finite() && fac_cond >= 0.0); + } + + #[cfg(feature = "backend-mixture")] + #[test] + fn mixture_runtime_entropy_helpers_are_finite() { + let mixture = RateBackend::Mixture { + spec: Arc::new(MixtureSpec::new( + MixtureKind::Bayes, + vec![MixtureExpertSpec::new(RateBackend::Ctw { depth: 4 })], + )), + }; + let backend = compiled_rate_backend(&mixture); + + let entropy = entropy_mixture(b"mixture bytes", &backend).expect("mixture entropy"); + let joint = joint_entropy_mixture(b"abcd", b"wxyz", &backend).expect("mixture joint"); + let cond = + conditional_chain_mixture(&[b"mix"], b"ture", &backend).expect("mixture conditional"); + assert!(entropy.is_finite() && entropy >= 0.0); + assert!(joint.is_finite() && joint >= 0.0); + assert!(cond.is_finite() && cond >= 0.0); + } + + #[cfg(feature = "backend-rwkv")] + #[test] + fn rwkv_runtime_entropy_helpers_are_finite() { + let backend = compiled_rate_backend( + &default_rate_backend_spec(RateBackendKind::Rwkv7).expect("default rwkv spec"), + ); + + let entropy = entropy_rwkv(b"rwkv bytes", &backend).expect("rwkv entropy"); + let joint = joint_entropy_rwkv(b"abc", b"xyz", &backend).expect("rwkv joint"); + let cond = conditional_chain_rwkv(&[b"seed"], b"more", &backend).expect("rwkv conditional"); + assert!(entropy.is_finite() && entropy >= 0.0); + assert!(joint.is_finite() && joint >= 0.0); + assert!(cond.is_finite() && cond >= 0.0); + } + + #[cfg(feature = "backend-mamba")] + #[test] + fn mamba_runtime_entropy_helpers_are_finite() { + let backend = compiled_rate_backend( + &default_rate_backend_spec(RateBackendKind::Mamba).expect("default mamba spec"), + ); + + let entropy = entropy_mamba(b"mamba bytes", &backend).expect("mamba entropy"); + let joint = joint_entropy_mamba(b"abc", b"xyz", &backend).expect("mamba joint"); + let cond = + conditional_chain_mamba(&[b"seed"], b"more", &backend).expect("mamba conditional"); + assert!(entropy.is_finite() && entropy >= 0.0); + assert!(joint.is_finite() && joint >= 0.0); + assert!(cond.is_finite() && cond >= 0.0); + } +} diff --git a/crates/infotheory/src/runtime/pdf_predictor_builders.rs b/crates/infotheory/src/runtime/pdf_predictor_builders.rs new file mode 100644 index 00000000..caa2d229 --- /dev/null +++ b/crates/infotheory/src/runtime/pdf_predictor_builders.rs @@ -0,0 +1,251 @@ +#[cfg(any( + feature = "backend-rosa", + feature = "backend-match", + feature = "backend-ppmd", + feature = "backend-sequitur", + feature = "backend-ctw", + feature = "backend-zpaq", + feature = "backend-mixture", + feature = "backend-particle", + feature = "backend-calibrated", + feature = "backend-mamba", + feature = "backend-rwkv" +))] +use super::*; + +macro_rules! feature_gated_rate_pdf_predictor_builder { + ( + feature: $feature:literal, + fn $name:ident($backend:ident) $body:block + ) => { + #[cfg(feature = $feature)] + pub(super) fn $name( + $backend: &CompiledRateBackend, + ) -> anyhow::Result $body + }; +} + +feature_gated_rate_pdf_predictor_builder! { + feature: "backend-rosa", + fn build_pdf_predictor_rosa(backend) { + expect_plan_ref!( + backend.plan(), + crate::spec::core::RateBackendPlan::RosaPlus { max_order }, + "rosa kernel used with non-rosa plan" + ); + Ok(crate::compression::RatePdfPredictor::Rosa( + crate::compression::RosaPredictor::new(*max_order), + )) + } +} + +feature_gated_rate_pdf_predictor_builder! { + feature: "backend-match", + fn build_pdf_predictor_match(backend) { + expect_plan_ref!( + backend.plan(), + crate::spec::core::RateBackendPlan::Match { + hash_bits, + min_len, + max_len, + base_mix, + confidence_scale, + }, + "match kernel used with non-match plan" + ); + Ok(crate::compression::RatePdfPredictor::Match { + model: MatchModel::new_contiguous( + *hash_bits, + *min_len, + *max_len, + *base_mix, + *confidence_scale, + ), + }) + } +} + +feature_gated_rate_pdf_predictor_builder! { + feature: "backend-match", + fn build_pdf_predictor_sparse_match(backend) { + expect_plan_ref!( + backend.plan(), + crate::spec::core::RateBackendPlan::SparseMatch { + hash_bits, + min_len, + max_len, + gap_min, + gap_max, + base_mix, + confidence_scale, + }, + "sparse-match kernel used with non-sparse-match plan" + ); + Ok(crate::compression::RatePdfPredictor::SparseMatch { + model: SparseMatchModel::new( + *hash_bits, + *min_len, + *max_len, + *gap_min, + *gap_max, + *base_mix, + *confidence_scale, + ), + }) + } +} + +feature_gated_rate_pdf_predictor_builder! { + feature: "backend-ppmd", + fn build_pdf_predictor_ppmd(backend) { + expect_plan_ref!( + backend.plan(), + crate::spec::core::RateBackendPlan::Ppmd { order, memory_mb }, + "ppmd kernel used with non-ppmd plan" + ); + Ok(crate::compression::RatePdfPredictor::Ppmd { + model: PpmdModel::new(*order, *memory_mb), + }) + } +} + +feature_gated_rate_pdf_predictor_builder! { + feature: "backend-sequitur", + fn build_pdf_predictor_sequitur(backend) { + expect_plan_ref!( + backend.plan(), + crate::spec::core::RateBackendPlan::Sequitur { context_bytes }, + "sequitur kernel used with non-sequitur plan" + ); + Ok(crate::compression::RatePdfPredictor::Sequitur { + model: SequiturModel::new(*context_bytes), + }) + } +} + +feature_gated_rate_pdf_predictor_builder! { + feature: "backend-ctw", + fn build_pdf_predictor_ctw(backend) { + expect_plan_ref!( + backend.plan(), + crate::spec::core::RateBackendPlan::Ctw { depth }, + "ctw kernel used with non-ctw plan" + ); + Ok(crate::compression::RatePdfPredictor::Ctw( + crate::compression::CtwPredictor::new_ctw(*depth), + )) + } +} + +feature_gated_rate_pdf_predictor_builder! { + feature: "backend-ctw", + fn build_pdf_predictor_fac_ctw(backend) { + expect_plan_ref!( + backend.plan(), + crate::spec::core::RateBackendPlan::FacCtw { + base_depth, + num_percept_bits: _, + encoding_bits, + msb_first, + }, + "fac-ctw kernel used with non-fac-ctw plan" + ); + Ok(crate::compression::RatePdfPredictor::FacCtw( + crate::compression::CtwPredictor::new_fac( + *base_depth, + *encoding_bits, + Some(*msb_first), + ), + )) + } +} + +feature_gated_rate_pdf_predictor_builder! { + feature: "backend-mamba", + fn build_pdf_predictor_mamba(backend) { + expect_plan_ref!( + backend.plan(), + crate::spec::core::RateBackendPlan::Mamba { parsed_method, .. }, + "mamba kernel used with non-mamba plan" + ); + Ok(crate::compression::RatePdfPredictor::Mamba( + crate::compression::MambaPredictor::from_method_spec(parsed_method)?, + )) + } +} + +feature_gated_rate_pdf_predictor_builder! { + feature: "backend-rwkv", + fn build_pdf_predictor_rwkv(backend) { + expect_plan_ref!( + backend.plan(), + crate::spec::core::RateBackendPlan::Rwkv7 { parsed_method, .. }, + "rwkv kernel used with non-rwkv plan" + ); + Ok(crate::compression::RatePdfPredictor::Rwkv( + crate::compression::RwkvPredictor::from_method_spec(parsed_method)?, + )) + } +} + +feature_gated_rate_pdf_predictor_builder! { + feature: "backend-zpaq", + fn build_pdf_predictor_zpaq(backend) { + expect_plan_ref!( + backend.plan(), + crate::spec::core::RateBackendPlan::Zpaq { method }, + "zpaq kernel used with non-zpaq plan" + ); + Ok(crate::compression::RatePdfPredictor::Zpaq( + crate::compression::ZpaqPredictor::new(method.clone()), + )) + } +} + +feature_gated_rate_pdf_predictor_builder! { + feature: "backend-mixture", + fn build_pdf_predictor_mixture(backend) { + Ok(crate::compression::RatePdfPredictor::Mixture( + crate::compression::MixturePredictor::new_from_compiled(backend)?, + )) + } +} + +feature_gated_rate_pdf_predictor_builder! { + feature: "backend-particle", + fn build_pdf_predictor_particle(backend) { + expect_plan_ref!( + backend.plan(), + crate::spec::core::RateBackendPlan::Particle { spec }, + "particle kernel used with non-particle plan" + ); + Ok(crate::compression::RatePdfPredictor::Particle( + ParticleRuntime::new(spec), + )) + } +} + +feature_gated_rate_pdf_predictor_builder! { + feature: "backend-calibrated", + fn build_pdf_predictor_calibrated(backend) { + expect_plan_ref!( + backend.plan(), + crate::spec::core::RateBackendPlan::Calibrated { + context, + bins, + learning_rate, + bias_clip, + base, + }, + "calibrated kernel used with non-calibrated plan" + ); + let base_backend = + compile_calibrated_base_backend(base).map_err(|err| anyhow::anyhow!("{err}"))?; + Ok(crate::compression::RatePdfPredictor::Calibrated { + base: Box::new(build_rate_pdf_predictor_via_kernel(&base_backend)?), + core: CalibratorCore::new(*context, *bins, *learning_rate, *bias_clip), + pdf: vec![1.0 / 256.0; 256], + valid: false, + }) + } +} diff --git a/crates/infotheory/src/runtime/plan_macros.rs b/crates/infotheory/src/runtime/plan_macros.rs new file mode 100644 index 00000000..266c1243 --- /dev/null +++ b/crates/infotheory/src/runtime/plan_macros.rs @@ -0,0 +1,9 @@ +macro_rules! expect_plan_ref { + ($plan_expr:expr, $pattern:pat, $message:literal) => { + let $pattern = $plan_expr else { + unreachable!($message) + }; + }; +} + +pub(super) use expect_plan_ref; diff --git a/crates/infotheory/src/runtime/predictor_builders.rs b/crates/infotheory/src/runtime/predictor_builders.rs new file mode 100644 index 00000000..fe855d55 --- /dev/null +++ b/crates/infotheory/src/runtime/predictor_builders.rs @@ -0,0 +1,370 @@ +#[cfg(any( + feature = "backend-rosa", + feature = "backend-match", + feature = "backend-ppmd", + feature = "backend-sequitur", + feature = "backend-ctw", + feature = "backend-zpaq", + feature = "backend-mixture", + feature = "backend-particle", + feature = "backend-calibrated", + feature = "backend-mamba", + feature = "backend-rwkv" +))] +use super::*; + +macro_rules! feature_gated_rate_predictor_builder { + ( + feature: $feature:literal, + fn $name:ident($backend:ident, $min_prob:ident) $body:block + ) => { + #[cfg(feature = $feature)] + pub(super) fn $name( + $backend: &CompiledRateBackend, + $min_prob: f64, + ) -> Result $body + }; +} + +feature_gated_rate_predictor_builder! { + feature: "backend-rosa", + fn build_predictor_rosa(backend, min_prob) { + expect_plan_ref!( + backend.plan(), + crate::spec::core::RateBackendPlan::RosaPlus { max_order }, + "rosa kernel used with non-rosa plan" + ); + let mut model = RosaPlus::new(*max_order, false, 0, 42); + model.build_lm_full_bytes_no_finalize_endpos(); + Ok(crate::mixture::RateBackendPredictor::Rosa { + model, + min_prob, + checkpoint_journal: Vec::new(), + checkpoint_depth: 0, + }) + } +} + +feature_gated_rate_predictor_builder! { + feature: "backend-match", + fn build_predictor_match(backend, min_prob) { + expect_plan_ref!( + backend.plan(), + crate::spec::core::RateBackendPlan::Match { + hash_bits, + min_len, + max_len, + base_mix, + confidence_scale, + }, + "match kernel used with non-match plan" + ); + Ok(crate::mixture::RateBackendPredictor::Match { + model: MatchModel::new_contiguous( + *hash_bits, + *min_len, + *max_len, + *base_mix, + *confidence_scale, + ), + min_prob, + }) + } +} + +feature_gated_rate_predictor_builder! { + feature: "backend-match", + fn build_predictor_sparse_match(backend, min_prob) { + expect_plan_ref!( + backend.plan(), + crate::spec::core::RateBackendPlan::SparseMatch { + hash_bits, + min_len, + max_len, + gap_min, + gap_max, + base_mix, + confidence_scale, + }, + "sparse-match kernel used with non-sparse-match plan" + ); + Ok(crate::mixture::RateBackendPredictor::SparseMatch { + model: SparseMatchModel::new( + *hash_bits, + *min_len, + *max_len, + *gap_min, + *gap_max, + *base_mix, + *confidence_scale, + ), + min_prob, + }) + } +} + +feature_gated_rate_predictor_builder! { + feature: "backend-ppmd", + fn build_predictor_ppmd(backend, min_prob) { + expect_plan_ref!( + backend.plan(), + crate::spec::core::RateBackendPlan::Ppmd { order, memory_mb }, + "ppmd kernel used with non-ppmd plan" + ); + Ok(crate::mixture::RateBackendPredictor::Ppmd { + model: PpmdModel::new(*order, *memory_mb), + min_prob, + }) + } +} + +feature_gated_rate_predictor_builder! { + feature: "backend-sequitur", + fn build_predictor_sequitur(backend, min_prob) { + expect_plan_ref!( + backend.plan(), + crate::spec::core::RateBackendPlan::Sequitur { context_bytes }, + "sequitur kernel used with non-sequitur plan" + ); + Ok(crate::mixture::RateBackendPredictor::Sequitur { + model: SequiturModel::new(*context_bytes), + min_prob, + }) + } +} + +feature_gated_rate_predictor_builder! { + feature: "backend-ctw", + fn build_predictor_ctw(backend, min_prob) { + expect_plan_ref!( + backend.plan(), + crate::spec::core::RateBackendPlan::Ctw { depth }, + "ctw kernel used with non-ctw plan" + ); + Ok(crate::mixture::RateBackendPredictor::Ctw { + tree: ContextTree::new(*depth), + bits_per_symbol: 8, + min_prob, + checkpoint_journal: Vec::new(), + checkpoint_depth: 0, + native_prefix_progress: None, + }) + } +} + +feature_gated_rate_predictor_builder! { + feature: "backend-ctw", + fn build_predictor_binary_tokens_ctw(backend, min_prob) { + expect_plan_ref!( + backend.plan(), + crate::spec::core::RateBackendPlan::Ctw { depth }, + "ctw binary-token kernel used with non-ctw plan" + ); + Ok(crate::mixture::RateBackendPredictor::Ctw { + tree: ContextTree::new(*depth), + bits_per_symbol: 1, + min_prob, + checkpoint_journal: Vec::new(), + checkpoint_depth: 0, + native_prefix_progress: None, + }) + } +} + +feature_gated_rate_predictor_builder! { + feature: "backend-ctw", + fn build_predictor_fac_ctw(backend, min_prob) { + expect_plan_ref!( + backend.plan(), + crate::spec::core::RateBackendPlan::FacCtw { + base_depth, + num_percept_bits: _, + encoding_bits, + msb_first, + }, + "fac-ctw kernel used with non-fac-ctw plan" + ); + let bits_per_symbol = *encoding_bits; + Ok(crate::mixture::RateBackendPredictor::FacCtw { + tree: FacContextTree::new(*base_depth, bits_per_symbol), + bits_per_symbol, + msb_first: *msb_first, + min_prob, + checkpoint_journal: Vec::new(), + checkpoint_depth: 0, + native_prefix_progress: None, + }) + } +} + +feature_gated_rate_predictor_builder! { + feature: "backend-ctw", + fn build_predictor_binary_tokens_fac_ctw(backend, min_prob) { + expect_plan_ref!( + backend.plan(), + crate::spec::core::RateBackendPlan::FacCtw { base_depth, .. }, + "fac-ctw binary-token kernel used with non-fac-ctw plan" + ); + Ok(crate::mixture::RateBackendPredictor::FacCtw { + tree: FacContextTree::new(*base_depth, 1), + bits_per_symbol: 1, + msb_first: false, + min_prob, + checkpoint_journal: Vec::new(), + checkpoint_depth: 0, + native_prefix_progress: None, + }) + } +} + +feature_gated_rate_predictor_builder! { + feature: "backend-rwkv", + fn build_predictor_rwkv(backend, min_prob) { + expect_plan_ref!( + backend.plan(), + crate::spec::core::RateBackendPlan::Rwkv7 { parsed_method, .. }, + "rwkv kernel used with non-rwkv plan" + ); + let mut compressor = rwkvzip::Compressor::new_from_method_spec(parsed_method) + .map_err(|e| format!("invalid rwkv method: {e}"))?; + compressor.reset_and_prime(); + Ok(crate::mixture::RateBackendPredictor::Rwkv7 { + pdf_scratch: vec![0.0; compressor.pdf_buffer.len()], + compressor, + primed: true, + min_prob, + }) + } +} + +feature_gated_rate_predictor_builder! { + feature: "backend-mamba", + fn build_predictor_mamba(backend, min_prob) { + expect_plan_ref!( + backend.plan(), + crate::spec::core::RateBackendPlan::Mamba { parsed_method, .. }, + "mamba kernel used with non-mamba plan" + ); + let mut compressor = mambazip::Compressor::new_from_method_spec(parsed_method) + .map_err(|e| format!("invalid mamba method: {e}"))?; + let bias = compressor.online_bias_snapshot(); + let logits = compressor + .model + .forward(&mut compressor.scratch, 0, &mut compressor.state); + mambazip::Compressor::logits_to_pdf(logits, bias.as_deref(), &mut compressor.pdf_buffer); + Ok(crate::mixture::RateBackendPredictor::Mamba { + pdf_scratch: vec![0.0; compressor.pdf_buffer.len()], + compressor, + primed: true, + min_prob, + }) + } +} + +feature_gated_rate_predictor_builder! { + feature: "backend-zpaq", + fn build_predictor_zpaq(backend, min_prob) { + expect_plan_ref!( + backend.plan(), + crate::spec::core::RateBackendPlan::Zpaq { method }, + "zpaq kernel used with non-zpaq plan" + ); + Ok(crate::mixture::RateBackendPredictor::Zpaq { + model: ZpaqRateModel::new(method.clone(), min_prob), + }) + } +} + +feature_gated_rate_predictor_builder! { + feature: "backend-mixture", + fn build_predictor_mixture(backend, _min_prob) { + let experts = crate::mixture::expert_configs_from_compiled_mixture(backend)?; + let runtime = crate::mixture::build_mixture_runtime_from_compiled(backend, &experts) + .map_err(|e| format!("MixtureSpec invalid: {e}"))?; + Ok(crate::mixture::RateBackendPredictor::Mixture { runtime }) + } +} + +feature_gated_rate_predictor_builder! { + feature: "backend-mixture", + fn build_predictor_binary_tokens_mixture(backend, min_prob) { + let experts = crate::mixture::expert_configs_from_compiled_mixture_with_builder( + backend, + crate::runtime::build_rate_backend_binary_token_predictor, + min_prob, + )?; + let runtime = crate::mixture::build_mixture_runtime_from_compiled(backend, &experts) + .map_err(|e| format!("MixtureSpec invalid: {e}"))?; + Ok(crate::mixture::RateBackendPredictor::Mixture { runtime }) + } +} + +feature_gated_rate_predictor_builder! { + feature: "backend-particle", + fn build_predictor_particle(backend, _min_prob) { + expect_plan_ref!( + backend.plan(), + crate::spec::core::RateBackendPlan::Particle { spec }, + "particle kernel used with non-particle plan" + ); + Ok(crate::mixture::RateBackendPredictor::Particle { + runtime: ParticleRuntime::new(spec), + }) + } +} + +feature_gated_rate_predictor_builder! { + feature: "backend-calibrated", + fn build_predictor_calibrated(backend, min_prob) { + expect_plan_ref!( + backend.plan(), + crate::spec::core::RateBackendPlan::Calibrated { + context, + bins, + learning_rate, + bias_clip, + base, + }, + "calibrated kernel used with non-calibrated plan" + ); + let base_backend = compile_calibrated_base_backend(base)?; + Ok(crate::mixture::RateBackendPredictor::Calibrated { + base: Box::new(build_rate_backend_predictor_via_kernel( + &base_backend, + min_prob, + )?), + core: CalibratorCore::new(*context, *bins, *learning_rate, *bias_clip), + pdf: [1.0 / 256.0; 256], + valid: false, + min_prob, + }) + } +} + +feature_gated_rate_predictor_builder! { + feature: "backend-calibrated", + fn build_predictor_binary_tokens_calibrated(backend, min_prob) { + expect_plan_ref!( + backend.plan(), + crate::spec::core::RateBackendPlan::Calibrated { + context, + bins, + learning_rate, + bias_clip, + base, + }, + "calibrated binary-token kernel used with non-calibrated plan" + ); + let base_backend = compile_calibrated_base_backend(base)?; + Ok(crate::mixture::RateBackendPredictor::Calibrated { + base: Box::new(crate::runtime::build_rate_backend_binary_token_predictor( + &base_backend, + min_prob, + )?), + core: CalibratorCore::new(*context, *bins, *learning_rate, *bias_clip), + pdf: [1.0 / 256.0; 256], + valid: false, + min_prob, + }) + } +} diff --git a/crates/infotheory/src/runtime/registry.rs b/crates/infotheory/src/runtime/registry.rs new file mode 100644 index 00000000..fab38608 --- /dev/null +++ b/crates/infotheory/src/runtime/registry.rs @@ -0,0 +1,93 @@ +//! Backend registry descriptor lookups and feature-gate error helpers. + +use super::{ + BackendDescriptor, COMPRESSION_BACKEND_REGISTRY, CompressionBackendDescriptor, + CompressionBackendKind, RATE_BACKEND_REGISTRY, RateBackendDescriptor, RateBackendKind, +}; + +pub(crate) fn find_backend_descriptor_in_registry( + registry: &'static [BackendDescriptor], + input: &str, +) -> Option<&'static BackendDescriptor> { + let key = input.trim().to_ascii_lowercase(); + registry + .iter() + .find(|descriptor| descriptor.aliases.iter().any(|alias| *alias == key)) +} + +fn backend_descriptor_by_kind( + registry: &'static [BackendDescriptor], + kind: K, +) -> Option<&'static BackendDescriptor> { + registry.iter().find(|descriptor| descriptor.kind == kind) +} + +pub(crate) fn backend_descriptor_by_kind_checked( + registry: &'static [BackendDescriptor], + kind: K, + registry_name: &'static str, +) -> Result<&'static BackendDescriptor, String> { + backend_descriptor_by_kind(registry, kind).ok_or_else(|| { + format!( + "internal backend registry mismatch: backend '{kind:?}' is missing from {registry_name}" + ) + }) +} + +pub(crate) fn describe_rate_backend_kind( + kind: RateBackendKind, +) -> Result<&'static RateBackendDescriptor, String> { + backend_descriptor_by_kind_checked(RATE_BACKEND_REGISTRY, kind, "RATE_BACKEND_REGISTRY") +} + +pub(crate) fn describe_compression_backend_kind( + kind: CompressionBackendKind, +) -> Result<&'static CompressionBackendDescriptor, String> { + backend_descriptor_by_kind_checked( + COMPRESSION_BACKEND_REGISTRY, + kind, + "COMPRESSION_BACKEND_REGISTRY", + ) +} + +#[cfg(any( + test, + not(feature = "backend-rosa"), + not(feature = "backend-ctw"), + not(feature = "backend-match"), + not(feature = "backend-ppmd"), + not(feature = "backend-sequitur"), + not(feature = "backend-zpaq"), + not(feature = "backend-mixture"), + not(feature = "backend-particle"), + not(feature = "backend-calibrated"), + not(feature = "backend-mamba"), + not(feature = "backend-rwkv") +))] +pub(super) fn rate_backend_feature_error(kind: RateBackendKind) -> String { + describe_rate_backend_kind(kind) + .map(|descriptor| match descriptor.feature { + Some(feature) => format!( + "backend '{}' requires infotheory feature '{}'", + descriptor.canonical, feature + ), + None => format!("backend '{}' is unavailable", descriptor.canonical), + }) + .unwrap_or_else(|err| err) +} + +#[cfg(any(test, not(feature = "backend-zpaq"), not(feature = "backend-rwkv")))] +pub(super) fn compression_backend_feature_error(kind: CompressionBackendKind) -> String { + describe_compression_backend_kind(kind) + .map(|descriptor| match descriptor.feature { + Some(feature) => format!( + "compression backend '{}' requires infotheory feature '{}'", + descriptor.canonical, feature + ), + None => format!( + "compression backend '{}' is unavailable", + descriptor.canonical + ), + }) + .unwrap_or_else(|err| err) +} diff --git a/crates/infotheory/src/search.rs b/crates/infotheory/src/search.rs new file mode 100644 index 00000000..f33c5dd9 --- /dev/null +++ b/crates/infotheory/src/search.rs @@ -0,0 +1,1472 @@ +use crate::api::{InfotheoryCtx, empirical_cross_entropy_bytes, empirical_entropy_bytes}; +use crate::backends::rosaplus::RosaPlus; +use crate::error::{InfotheoryError, InfotheoryResult}; +#[cfg(feature = "backend-rwkv")] +use crate::spec::MethodBackendFamily; +use crate::spec::RateBackendTraceStrategy; +use rayon::prelude::*; +use std::collections::hash_map::DefaultHasher; +use std::fs; +use std::hash::{Hash, Hasher}; +use std::path::{Path, PathBuf}; + +#[derive(Debug, Clone)] +/// One scored retrieval unit returned by code search. +pub struct Snippet { + /// Source file containing the match/candidate. + pub path: PathBuf, + /// 1-based inclusive start line for snippet display. + pub start_line: usize, + /// 1-based inclusive end line for snippet display. + pub end_line: usize, + /// Raw candidate bytes used for entropy/rerank scoring. + pub content: Vec, + /// Final ranking score (larger is better). + pub score: f64, +} + +fn stage0_prefilter( + query_bytes: &[u8], + mut candidates: Vec, + opts: &SearchOptions, + debug: bool, +) -> InfotheoryResult> { + let n = candidates.len(); + if n == 0 { + return Ok(candidates); + } + + let frac = opts.stage0_keep_frac.clamp(0.0, 1.0); + if frac >= 1.0 { + return Ok(candidates); + } + + // Option A: Unigram (i.i.d.) likelihood-gain proxy. + // score0(x) = H0(Q) - H0(Q|X) + // where H0(Q|X) is the empirical cross-entropy of Q under X's unigram model. + let h0_q = empirical_entropy_bytes(query_bytes); + candidates.par_iter_mut().try_for_each(|s| { + let h0_q_x = empirical_cross_entropy_bytes(query_bytes, &s.content); + s.score = h0_q - h0_q_x; + Ok::<(), InfotheoryError>(()) + })?; + + let mut keep = ((n as f64) * frac).ceil() as usize; + keep = keep.max(opts.top_k).min(n); + if keep < n { + let nth = keep.saturating_sub(1); + candidates.select_nth_unstable_by(nth, |a, b| { + b.score + .partial_cmp(&a.score) + .unwrap_or(std::cmp::Ordering::Equal) + }); + candidates.truncate(keep); + } + + if debug { + println!( + "Stage-0 prefilter kept {}/{} candidates (frac={:.4})", + candidates.len(), + n, + frac + ); + } + + Ok(candidates) +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +/// Candidate unit granularity for stage-0/1 collection. +pub enum SearchGranularity { + /// Split files into hashed line windows/snippets. + Snippet, + /// Treat each file as a single candidate. + File, +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +/// Stage-2 prior handling strategy for KMI reranking. +pub enum Stage2PriorMode { + /// Use the (full or summarized) universal prior as a prefix for compression metrics. + Use, + /// Do NOT use the universal prior in Stage 2 (pure NCD/KMI rerank on Stage-1-filtered set). + Disable, + /// Summarize the universal prior via an inner prior-less search over the prior corpus. + Summarize, +} + +#[derive(Clone)] +/// Tunables for the three-stage information-theoretic search pipeline. +pub struct SearchOptions { + /// Candidate granularity at collection time. + pub granularity: SearchGranularity, + /// Universal prior corpus path (file or directory). If set: + /// - Stage 1 always uses it. + /// - Stage 2 uses it by default (unless Stage2PriorMode::Disable). + pub universal_prior: Option, + /// Whether/how Stage-2 reranking uses universal prior context. + pub stage2_prior_mode: Stage2PriorMode, + /// Number of final results to keep. + pub top_k: usize, + /// Fraction of candidates retained by the unigram prefilter. + pub stage0_keep_frac: f64, + /// Fully configured information-theory context/backend bundle. + /// + /// Algorithm-specific configuration (such as ROSA's `max_order`) lives + /// inside the rate backend's variant and is read from it when needed. + pub ctx: InfotheoryCtx, +} + +/// Default rate backend name used by CLI search when no backend flags are supplied. +pub const DEFAULT_SEARCH_RATE_BACKEND_NAME: &str = "rosaplus"; +/// Default compression backend name used by CLI search when no backend flags are supplied. +#[cfg(feature = "backend-zpaq")] +pub const DEFAULT_SEARCH_COMPRESSION_BACKEND_NAME: &str = "zpaq"; +/// Default compression backend name used by CLI search when no backend flags are supplied. +#[cfg(not(feature = "backend-zpaq"))] +pub const DEFAULT_SEARCH_COMPRESSION_BACKEND_NAME: &str = "rate-ac"; + +fn default_search_ctx() -> InfotheoryResult { + InfotheoryCtx::try_default() +} + +impl SearchOptions { + /// Build the default search configuration for the current feature slice. + pub fn try_default() -> InfotheoryResult { + Ok(Self { + granularity: SearchGranularity::Snippet, + universal_prior: None, + stage2_prior_mode: Stage2PriorMode::Use, + top_k: 50, + stage0_keep_frac: 0.2, + ctx: default_search_ctx()?, + }) + } +} + +/// Run search with default options and print top shell extraction commands. +pub fn run_search(query: &str, target_path: &str) -> InfotheoryResult<()> { + let opts = SearchOptions::try_default()?; + run_search_with_options(query, target_path, &opts) +} + +/// Run search with explicit options and print top shell extraction commands. +pub fn run_search_with_options( + query: &str, + target_path: &str, + opts: &SearchOptions, +) -> InfotheoryResult<()> { + let debug = std::env::var("DEBUG_SEARCH").is_ok(); + let results = search_with_options(query, target_path, opts)?; + for (i, snippet) in results.iter().take(5).enumerate() { + if debug { + println!( + "Rank {}: Score={:.6}, Path={}", + i + 1, + snippet.score, + snippet.path.display() + ); + } + println!( + "sed -n '{},{}p' {}", + snippet.start_line, + snippet.end_line, + snippet.path.display() + ); + } + Ok(()) +} + +/// Run the full 3-stage search pipeline and return ranked results. +/// +/// The returned `Vec` is sorted by descending score, truncated +/// to `opts.top_k` entries. Each snippet carries its file path, line +/// range, content bytes, and final KMI-reranked score. +pub fn search_with_options( + query: &str, + target_path: &str, + opts: &SearchOptions, +) -> InfotheoryResult> { + let debug = std::env::var("DEBUG_SEARCH").is_ok(); + let query_bytes = resolve_query_bytes(query); + if query_bytes.is_empty() { + return Err(InfotheoryError::runtime("search query is empty")); + } + + if debug { + println!( + "Scanning target: {} (granularity={:?}, prior={}, stage2_prior_mode={:?})", + target_path, + opts.granularity, + opts.universal_prior.as_deref().unwrap_or(""), + opts.stage2_prior_mode + ); + } + + let candidates = collect_candidates(target_path, opts.granularity); + if candidates.is_empty() { + return Err(InfotheoryError::runtime(format!( + "no accessible files found in target '{target_path}'" + ))); + } + + let candidates = stage0_prefilter(query_bytes.as_slice(), candidates, opts, debug)?; + if candidates.is_empty() { + return Err(InfotheoryError::runtime( + "no candidates remain after the stage-0 prefilter", + )); + } + if debug { + println!("Found {} candidates. Filtering...", candidates.len()); + } + + // Stage 1: Filter + let mut scored_candidates = if let Some(prior_path) = opts.universal_prior.as_deref() { + stage1_filter_with_universal_prior(&query_bytes, prior_path, candidates, opts)? + } else { + stage1_filter_no_prior(&query_bytes, candidates, opts)? + }; + + let top_k_size = opts.top_k.min(scored_candidates.len()); + if top_k_size < scored_candidates.len() { + let nth = top_k_size.saturating_sub(1); + scored_candidates.select_nth_unstable_by(nth, |a, b| { + b.score + .partial_cmp(&a.score) + .unwrap_or(std::cmp::Ordering::Equal) + }); + scored_candidates.truncate(top_k_size); + } + + scored_candidates.sort_by(|a, b| { + b.score + .partial_cmp(&a.score) + .unwrap_or(std::cmp::Ordering::Equal) + }); + + let top_candidates = &mut scored_candidates[..top_k_size]; + if debug { + println!( + "Reranking top {} candidates with Kolmogorov Mutual Information...", + top_k_size + ); + } + + // Stage 2: Rerank + stage2_rerank_kmi(&query_bytes, top_candidates, opts)?; + top_candidates.sort_by(|a, b| { + b.score + .partial_cmp(&a.score) + .unwrap_or(std::cmp::Ordering::Equal) + }); + + Ok(scored_candidates) +} + +fn resolve_query_bytes(query: &str) -> Vec { + let p = Path::new(query); + if p.exists() && fs::metadata(p).map(|m| m.is_file()).unwrap_or(false) { + fs::read(p).unwrap_or_else(|_| query.as_bytes().to_vec()) + } else { + query.as_bytes().to_vec() + } +} + +fn stage1_filter_no_prior( + query_bytes: &[u8], + candidates: Vec, + opts: &SearchOptions, +) -> InfotheoryResult> { + let h_q = opts + .ctx + .try_entropy_rate_bytes(query_bytes) + .map_err(|err| InfotheoryError::runtime(format!("stage-1 search entropy failed: {err}")))?; + + let scored: InfotheoryResult> = candidates + .into_par_iter() + .map(|mut snippet| { + let h_q_x = opts + .ctx + .try_cross_entropy_rate_bytes(query_bytes, &snippet.content) + .map_err(|err| { + InfotheoryError::runtime(format!("stage-1 search cross entropy failed: {err}")) + })?; + snippet.score = h_q - h_q_x; + Ok(snippet) + }) + .collect(); + + // Keep equivalence with old behavior by not clamping. + scored +} + +fn stage1_filter_with_universal_prior( + query_bytes: &[u8], + prior_path: &str, + candidates: Vec, + opts: &SearchOptions, +) -> InfotheoryResult> { + #[cfg(feature = "backend-rwkv")] + if let Some((mut base, prior_snapshot)) = rwkv_prior_snapshot(opts, prior_path) { + let h_u_q = { + base.restore_runtime(&prior_snapshot); + base.cross_entropy_from_current(query_bytes) + .map_err(|err| { + InfotheoryError::runtime(format!("stage-1 rwkv prior scoring failed: {err}")) + })? + }; + return candidates + .into_par_iter() + .map_init( + || base.clone(), + |m: &mut crate::rwkvzip::Compressor, mut snippet| { + m.restore_runtime(&prior_snapshot); + m.absorb_chain(&[snippet.content.as_slice()]) + .map_err(|err| { + InfotheoryError::runtime(format!( + "stage-1 rwkv candidate absorption failed: {err}" + )) + })?; + let h_ux_q = m.cross_entropy_from_current(query_bytes).map_err(|err| { + InfotheoryError::runtime(format!( + "stage-1 rwkv candidate scoring failed: {err}" + )) + })?; + snippet.score = h_u_q - h_ux_q; + Ok(snippet) + }, + ) + .collect(); + } + + if opts.ctx.rate_backend.capabilities().trace_strategy != RateBackendTraceStrategy::Rosa { + let prior_prefix = corpus_bytes(prior_path, SearchGranularity::File); + let h_u_q = opts + .ctx + .try_cross_entropy_conditional_chain(&[prior_prefix.as_slice()], query_bytes) + .map_err(|err| { + InfotheoryError::runtime(format!( + "stage-1 conditional-chain prior scoring failed: {err}" + )) + })?; + return candidates + .into_par_iter() + .map(|mut snippet| { + let h_ux_q = opts + .ctx + .try_cross_entropy_conditional_chain( + &[prior_prefix.as_slice(), snippet.content.as_slice()], + query_bytes, + ) + .map_err(|err| { + InfotheoryError::runtime(format!( + "stage-1 conditional-chain candidate scoring failed: {err}" + )) + })?; + snippet.score = h_u_q - h_ux_q; + Ok(snippet) + }) + .collect(); + } + + // PERFORMANCE NOTE: + // Training the prior using snippet-level windows would duplicate overlapping content + // and explode runtime. We *always* train/load the prior at file granularity. + let mut base = load_or_train_prior_model(prior_path, opts); + // For true conditional updates we require the fixed 256-byte alphabet LM. + // This ensures symbol indices remain stable across incremental updates. + base.ensure_lm_built_no_finalize_endpos(); + // Reduce the cost of cloning `base` per worker. + base.shrink_aux_buffers(); + + // Precompute query codepoints once (cross_entropy() would allocate this per call). + let query_cps: Vec = query_bytes.iter().map(|&b| b as u32).collect(); + let h_u_q = base.cross_entropy_cps(&query_cps); + + // True conditional update: + // score(x) = H_U(q) - H_{U+x}(q) + // by applying a reversible candidate update to the *full* prior model. + // + // MEMORY NOTE: + // `map_init(|| base.clone(), ...)` clones the model once per Rayon worker. + // For large priors this can blow up RSS. We cap worker count based on an estimate + // of model bytes and best-effort available memory (Linux). + let model_bytes = base.estimated_size_bytes().max(1); + let threads = memory_aware_threads(model_bytes); + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(threads) + .build() + .map_err(|err| InfotheoryError::runtime(format!("failed to build rayon pool: {err}")))?; + + pool.install(|| { + candidates + .into_par_iter() + .map_init( + || base.clone(), + |m, mut snippet| { + let mut tx = m.begin_tx(); + m.train_example_tx(&mut tx, &snippet.content); + let h_ux_q = m.cross_entropy_cps(&query_cps); + m.rollback_tx(tx); + snippet.score = h_u_q - h_ux_q; + Ok(snippet) + }, + ) + .collect() + }) +} + +#[cfg(feature = "backend-rwkv")] +fn rwkv_prior_snapshot( + opts: &SearchOptions, + prior_path: &str, +) -> Option<(crate::rwkvzip::Compressor, crate::rwkvzip::RuntimeSnapshot)> { + if opts.ctx.rate_backend.capabilities().method_family != Some(MethodBackendFamily::Rwkv7) { + return None; + } + let method = opts.ctx.rate_backend.method_string()?; + let mut compressor = crate::rwkvzip::Compressor::new_from_method(method).ok()?; + + let prior_prefix = corpus_bytes(prior_path, SearchGranularity::File); + compressor.reset_and_prime(); + let _ = compressor.absorb_chain(&[prior_prefix.as_slice()]); + let snapshot = compressor.snapshot_runtime(); + Some((compressor, snapshot)) +} + +fn memory_aware_threads(model_bytes: usize) -> usize { + let hw = num_cpus::get().max(1); + let avail = linux_mem_available_bytes().unwrap_or(0); + if avail == 0 { + return hw; + } + + // Heuristic: allow up to 25% of available memory for (worker clones + overhead). + let budget = (avail / 4).max(model_bytes as u64); + let max_by_mem = (budget / (model_bytes as u64)).max(1) as usize; + hw.min(max_by_mem).max(1) +} + +fn linux_mem_available_bytes() -> Option { + // Linux-only best-effort. If parsing fails, fall back to unconstrained. + let s = std::fs::read_to_string("/proc/meminfo").ok()?; + for line in s.lines() { + if let Some(rest) = line.strip_prefix("MemAvailable:") { + let parts: Vec<&str> = rest.split_whitespace().collect(); + if parts.is_empty() { + return None; + } + let kb: u64 = parts[0].parse().ok()?; + return Some(kb.saturating_mul(1024)); + } + } + None +} + +fn stage2_rerank_kmi( + query_bytes: &[u8], + top_candidates: &mut [Snippet], + opts: &SearchOptions, +) -> InfotheoryResult<()> { + let prior_prefix: Option> = + match (opts.universal_prior.as_deref(), opts.stage2_prior_mode) { + (None, _) => None, + (Some(_), Stage2PriorMode::Disable) => None, + (Some(prior_path), Stage2PriorMode::Use) => { + Some(corpus_bytes(prior_path, SearchGranularity::File)) + } + (Some(prior_path), Stage2PriorMode::Summarize) => { + Some(summarize_prior_for_query(query_bytes, prior_path, opts)?) + } + }; + + let cq = if let Some(prefix) = prior_prefix.as_deref() { + opts.ctx + .try_compress_size_chain(&[prefix, query_bytes]) + .map_err(|err| { + InfotheoryError::runtime(format!("stage-2 query compression failed: {err}")) + })? + } else { + opts.ctx + .try_compress_size_chain(&[query_bytes]) + .map_err(|err| { + InfotheoryError::runtime(format!("stage-2 query compression failed: {err}")) + })? + }; + + top_candidates.par_iter_mut().try_for_each(|snippet| { + let cx = if let Some(prefix) = prior_prefix.as_deref() { + opts.ctx + .try_compress_size_chain(&[prefix, snippet.content.as_slice()]) + .map_err(|err| { + InfotheoryError::runtime(format!("stage-2 candidate compression failed: {err}")) + })? + } else { + opts.ctx + .try_compress_size_chain(&[snippet.content.as_slice()]) + .map_err(|err| { + InfotheoryError::runtime(format!("stage-2 candidate compression failed: {err}")) + })? + }; + + let c1 = if let Some(prefix) = prior_prefix.as_deref() { + opts.ctx + .try_compress_size_chain(&[prefix, snippet.content.as_slice(), query_bytes]) + .map_err(|err| { + InfotheoryError::runtime(format!("stage-2 joint compression failed: {err}")) + })? + } else { + opts.ctx + .try_compress_size_chain(&[snippet.content.as_slice(), query_bytes]) + .map_err(|err| { + InfotheoryError::runtime(format!("stage-2 joint compression failed: {err}")) + })? + }; + + let c2 = if let Some(prefix) = prior_prefix.as_deref() { + opts.ctx + .try_compress_size_chain(&[prefix, query_bytes, snippet.content.as_slice()]) + .map_err(|err| { + InfotheoryError::runtime(format!("stage-2 joint compression failed: {err}")) + })? + } else { + opts.ctx + .try_compress_size_chain(&[query_bytes, snippet.content.as_slice()]) + .map_err(|err| { + InfotheoryError::runtime(format!("stage-2 joint compression failed: {err}")) + })? + }; + + let c_joint = c1.min(c2); + snippet.score = if c_joint == u64::MAX { + 0.0 + } else { + (cq as f64 + cx as f64 - c_joint as f64).max(0.0) + }; + Ok::<(), InfotheoryError>(()) + })?; + Ok(()) +} + +fn summarize_prior_for_query( + query_bytes: &[u8], + prior_path: &str, + opts: &SearchOptions, +) -> InfotheoryResult> { + // Prior-less search inside the prior corpus itself. + // We approximate K(q|x) via conditional compression: min(C(xq),C(qx)) - C(x), and select the MIN. + let candidates = collect_candidates(prior_path, opts.granularity); + if candidates.is_empty() { + return Ok(Vec::new()); + } + + let cq = opts + .ctx + .try_compress_size_chain(&[query_bytes]) + .map_err(|err| { + InfotheoryError::runtime(format!( + "prior summarization query compression failed: {err}" + )) + })?; + + let mut best: Option<(f64, Vec)> = None; + for c in candidates { + let cx = opts + .ctx + .try_compress_size_chain(&[c.content.as_slice()]) + .map_err(|err| { + InfotheoryError::runtime(format!( + "prior summarization candidate compression failed: {err}" + )) + })?; + + let cxq = opts + .ctx + .try_compress_size_chain(&[c.content.as_slice(), query_bytes]) + .map_err(|err| { + InfotheoryError::runtime(format!( + "prior summarization joint compression failed: {err}" + )) + })?; + let cqx = opts + .ctx + .try_compress_size_chain(&[query_bytes, c.content.as_slice()]) + .map_err(|err| { + InfotheoryError::runtime(format!( + "prior summarization joint compression failed: {err}" + )) + })?; + let c_joint = cxq.min(cqx); + if c_joint == u64::MAX { + continue; + } + // Conditional complexity proxy. + let k_q_given_x = (c_joint as f64 - cx as f64).max(0.0); + // Tie-breaker: if equal, prefer smaller candidate. + let candidate_key = (k_q_given_x, cx as f64, cq as f64); + let is_better = match &best { + None => true, + Some((best_k, best_bytes)) => { + let best_cx = opts.ctx.try_compress_size(best_bytes).map_err(|err| { + InfotheoryError::runtime(format!( + "prior summarization tie-break compression failed: {err}" + )) + })? as f64; + (candidate_key.0, candidate_key.1) < (*best_k, best_cx) + } + }; + if is_better { + best = Some((k_q_given_x, c.content)); + } + } + + Ok(best.map(|(_, b)| b).unwrap_or_default()) +} + +fn train_rosa_on_corpus(m: &mut RosaPlus, corpus_path: &str, granularity: SearchGranularity) { + // Train incrementally on each candidate to avoid giant concatenations. + for c in collect_candidates(corpus_path, granularity) { + if !c.content.is_empty() { + m.train_example(&c.content); + } + } +} + +fn prior_cache_path(prior_path: &str, max_order: i64) -> Option { + let home = std::env::var("XDG_CACHE_HOME") + .ok() + .or_else(|| std::env::var("HOME").ok().map(|h| format!("{}/.cache", h))); + let cache_root = match home { + Some(h) => PathBuf::from(h).join("infotheory").join("rosa_prior"), + None => return None, + }; + + let mut hasher = DefaultHasher::new(); + // Cache format/version (bump when training or serialization semantics change). + (5u32).hash(&mut hasher); + prior_path.hash(&mut hasher); + max_order.hash(&mut hasher); + // file-granularity is baked into the cache key (we always use it for prior training) + ("file" as &str).hash(&mut hasher); + let key = hasher.finish(); + Some(cache_root.join(format!("prior_{:016x}.rosa", key))) +} + +fn load_or_train_prior_model(prior_path: &str, opts: &SearchOptions) -> RosaPlus { + // This path is only reached when the backend's trace strategy is Rosa, + // so the plan is guaranteed to be RosaPlus. + let crate::spec::core::RateBackendPlan::RosaPlus { max_order } = opts.ctx.rate_backend.plan() + else { + unreachable!("load_or_train_prior_model called with non-ROSA backend") + }; + let max_order: i64 = *max_order; + + // Load cached prior model if present. + if let Some(cache_path) = prior_cache_path(prior_path, max_order) { + if let Some(parent) = cache_path.parent() { + let _ = fs::create_dir_all(parent); + } + if cache_path.exists() + && let Ok(mut m) = RosaPlus::load(cache_path.to_string_lossy().as_ref()) + { + // Ensure fixed 256-byte alphabet LM for incremental conditional updates. + if m.lm_alpha_n() != 256 { + m.build_lm_full_bytes_no_finalize_endpos(); + let _ = m.save(cache_path.to_string_lossy().as_ref()); + } + return m; + } + + // Train + save. + let mut m = RosaPlus::new(max_order, false, 0, 42); + train_rosa_on_corpus(&mut m, prior_path, SearchGranularity::File); + // Build a fixed-byte alphabet LM once so the saved model is the full state. + m.build_lm_full_bytes_no_finalize_endpos(); + let _ = m.save(cache_path.to_string_lossy().as_ref()); + return m; + } + + // Fallback: no cache location available. + let mut m = RosaPlus::new(max_order, false, 0, 42); + train_rosa_on_corpus(&mut m, prior_path, SearchGranularity::File); + m +} + +fn corpus_bytes(corpus_path: &str, granularity: SearchGranularity) -> Vec { + // Compression prior prefix requires a concrete byte buffer. + // We join candidates with a simple delimiter to preserve boundaries. + let mut out = Vec::new(); + for c in collect_candidates(corpus_path, granularity) { + if c.content.is_empty() { + continue; + } + out.extend_from_slice(&c.content); + out.extend_from_slice(b"\n\n"); + } + out +} + +fn collect_candidates(target: &str, granularity: SearchGranularity) -> Vec { + let mut snippets = Vec::new(); + let path = Path::new(target); + + if path.exists() { + if path.is_file() { + snippets.extend(file_to_candidates(path, granularity)); + } else if path.is_dir() { + visit_dirs(path, &mut snippets, granularity); + } + } + + snippets +} + +fn visit_dirs(dir: &Path, snippets: &mut Vec, granularity: SearchGranularity) { + if let Ok(entries) = fs::read_dir(dir) { + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() { + if let Some(name_str) = path.file_name().and_then(|n| n.to_str()) + && !name_str.starts_with('.') + { + visit_dirs(&path, snippets, granularity); + } + } else { + snippets.extend(file_to_candidates(&path, granularity)); + } + } + } +} + +fn file_to_candidates(path: &Path, granularity: SearchGranularity) -> Vec { + let mut snippets = Vec::new(); + + // Only process text files + if let Some(ext) = path.extension() { + let ext_str = ext.to_string_lossy(); + if matches!( + ext_str.as_ref(), + "o" | "a" | "so" | "dll" | "exe" | "bin" | "png" | "jpg" | "zip" | "gz" + ) { + return snippets; + } + } + + match granularity { + SearchGranularity::File => { + if let Ok(bytes) = fs::read(path) + && !bytes.is_empty() + { + // Best-effort line count for `sed` output. + let lines = bytes.iter().filter(|&&b| b == b'\n').count() + 1; + snippets.push(Snippet { + path: path.to_path_buf(), + start_line: 1, + end_line: lines.max(1), + content: bytes, + score: 0.0, + }); + } + } + SearchGranularity::Snippet => { + if let Ok(bytes) = fs::read(path) { + if bytes.is_empty() { + return snippets; + } + + let window = 50usize; + let stride = 20usize; + + let mut line_starts: Vec = Vec::new(); + line_starts.push(0); + for (i, &b) in bytes.iter().enumerate() { + if b == b'\n' { + let next = i + 1; + if next < bytes.len() { + line_starts.push(next); + } + } + } + + if line_starts.is_empty() { + return snippets; + } + + let mut i = 0usize; + while i < line_starts.len() { + let end = (i + window).min(line_starts.len()); + let start_b = line_starts[i]; + let end_b = if end >= line_starts.len() { + bytes.len() + } else { + line_starts[end] + }; + + if end_b > start_b { + let content = bytes[start_b..end_b].to_vec(); + if content.len() > 50 { + snippets.push(Snippet { + path: path.to_path_buf(), + start_line: i + 1, + end_line: end, + content, + score: 0.0, + }); + } + } + + if end == line_starts.len() { + break; + } + i += stride; + } + } + } + } + snippets +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::api::{CompressionBackend, InfotheoryCtx, RateBackend}; + #[cfg(feature = "backend-zpaq")] + use crate::error::InfotheoryError; + use std::time::{SystemTime, UNIX_EPOCH}; + + fn temp_path(prefix: &str) -> PathBuf { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("clock before epoch") + .as_nanos(); + std::env::temp_dir().join(format!("infotheory-search-{prefix}-{nanos}")) + } + + fn write_text(path: &Path, text: &str) { + fs::write(path, text.as_bytes()).expect("write temp text fixture"); + } + + fn ctw_search_ctx() -> InfotheoryCtx { + InfotheoryCtx::from_specs( + RateBackend::Ctw { depth: 10 }, + CompressionBackend::Rate { + rate_backend: RateBackend::Ctw { depth: 10 }, + coder: crate::coders::CoderType::AC, + framing: crate::compression::FramingMode::Raw, + }, + ) + .expect("ctw search context should compile") + } + + #[test] + fn resolve_query_bytes_prefers_file_contents() { + let path = temp_path("query"); + fs::write(&path, b"query-from-file").expect("write query file"); + let got = resolve_query_bytes(path.to_string_lossy().as_ref()); + assert_eq!(got, b"query-from-file"); + let _ = fs::remove_file(path); + } + + #[test] + fn file_to_candidates_skips_binary_extensions() { + let path = temp_path("binary").with_extension("png"); + fs::write(&path, b"not-actually-image").expect("write pseudo-binary"); + let out = file_to_candidates(&path, SearchGranularity::File); + assert!(out.is_empty(), "binary extension should be skipped"); + let _ = fs::remove_file(path); + } + + #[test] + fn file_to_candidates_generates_snippets() { + let path = temp_path("snippet").with_extension("txt"); + let mut text = String::new(); + for i in 0..120 { + text.push_str(&format!("line-{i:03}\n")); + } + fs::write(&path, text.as_bytes()).expect("write snippet file"); + let out = file_to_candidates(&path, SearchGranularity::Snippet); + assert!(!out.is_empty(), "expected snippet candidates"); + assert!(out.iter().all(|s| s.end_line >= s.start_line)); + let _ = fs::remove_file(path); + } + + #[test] + fn collect_candidates_skips_hidden_directories() { + let root = temp_path("tree"); + let hidden = root.join(".hidden"); + let visible = root.join("visible"); + fs::create_dir_all(&hidden).expect("create hidden dir"); + fs::create_dir_all(&visible).expect("create visible dir"); + fs::write(hidden.join("secret.txt"), b"hidden").expect("write hidden file"); + fs::write(visible.join("public.txt"), b"visible\ntext\n").expect("write visible file"); + + let out = collect_candidates(root.to_string_lossy().as_ref(), SearchGranularity::File); + assert_eq!(out.len(), 1, "only visible file should be collected"); + assert!( + out[0].path.to_string_lossy().contains("public.txt"), + "unexpected collected file path: {}", + out[0].path.display() + ); + + let _ = fs::remove_dir_all(root); + } + + #[test] + fn stage0_prefilter_respects_topk_floor() { + let mut candidates = Vec::new(); + for i in 0..10 { + candidates.push(Snippet { + path: PathBuf::from(format!("f{i}.txt")), + start_line: 1, + end_line: 1, + content: format!("candidate-{i}").into_bytes(), + score: 0.0, + }); + } + let opts = SearchOptions { + top_k: 4, + stage0_keep_frac: 0.1, + ..SearchOptions::try_default().expect("search defaults") + }; + let kept = stage0_prefilter(b"candidate", candidates, &opts, false) + .expect("stage0 prefilter should succeed"); + assert!( + kept.len() >= 4, + "stage0 must keep at least top_k candidates, got {}", + kept.len() + ); + } + + #[test] + fn stage0_prefilter_full_fraction_is_noop() { + let candidates = vec![ + Snippet { + path: PathBuf::from("a.txt"), + start_line: 1, + end_line: 1, + content: b"alpha beta".to_vec(), + score: 0.0, + }, + Snippet { + path: PathBuf::from("b.txt"), + start_line: 2, + end_line: 3, + content: b"gamma delta".to_vec(), + score: 0.0, + }, + ]; + let opts = SearchOptions { + stage0_keep_frac: 1.0, + top_k: 1, + ..SearchOptions::try_default().expect("search defaults") + }; + let kept = stage0_prefilter(b"alpha", candidates.clone(), &opts, false) + .expect("prefilter should succeed"); + assert_eq!(kept.len(), candidates.len()); + assert_eq!(kept[0].path, candidates[0].path); + assert_eq!(kept[1].path, candidates[1].path); + } + + #[test] + fn search_with_options_returns_error_for_empty_query() { + let path = temp_path("search-empty").with_extension("txt"); + fs::write(&path, b"content").expect("write search target"); + let opts = SearchOptions::try_default().expect("search defaults"); + let err = search_with_options("", path.to_string_lossy().as_ref(), &opts) + .expect_err("empty query should return an error"); + assert!(err.to_string().contains("query is empty")); + let _ = fs::remove_file(path); + } + + #[test] + fn search_with_options_returns_error_for_missing_target() { + let opts = SearchOptions::try_default().expect("search defaults"); + let err = search_with_options( + "needle", + "/definitely/missing/infotheory-search-target", + &opts, + ) + .expect_err("missing target should return an error"); + assert!(err.to_string().contains("no accessible files found")); + } + + #[test] + fn stage1_filter_no_prior_prefers_exact_match_candidate() { + let opts = SearchOptions { + granularity: SearchGranularity::File, + top_k: 2, + stage0_keep_frac: 1.0, + ctx: ctw_search_ctx(), + ..SearchOptions::try_default().expect("search defaults") + }; + let candidates = vec![ + Snippet { + path: PathBuf::from("noise.txt"), + start_line: 1, + end_line: 1, + content: b"background entropy without the query phrase".to_vec(), + score: 0.0, + }, + Snippet { + path: PathBuf::from("match.txt"), + start_line: 1, + end_line: 1, + content: b"needle exact stage one phrase repeated needle exact stage one phrase" + .to_vec(), + score: 0.0, + }, + ]; + let scored = stage1_filter_no_prior(b"needle exact stage one phrase", candidates, &opts) + .expect("stage1 without prior should succeed"); + assert_eq!(scored.len(), 2); + assert!( + scored[1].score > scored[0].score, + "exact match candidate should score above unrelated content" + ); + } + + #[test] + fn stage1_filter_with_universal_prior_prefers_prior_consistent_candidate() { + let prior_root = temp_path("stage1-prior"); + fs::create_dir_all(&prior_root).expect("create prior dir"); + write_text( + &prior_root.join("prior.txt"), + "predictive coding exact phrase context\npredictive coding exact phrase context\n", + ); + + let opts = SearchOptions { + granularity: SearchGranularity::File, + universal_prior: Some(prior_root.to_string_lossy().to_string()), + stage2_prior_mode: Stage2PriorMode::Use, + top_k: 2, + stage0_keep_frac: 1.0, + ctx: ctw_search_ctx(), + }; + let candidates = vec![ + Snippet { + path: PathBuf::from("noise.txt"), + start_line: 1, + end_line: 1, + content: b"background corpus without predictive coding context".to_vec(), + score: 0.0, + }, + Snippet { + path: PathBuf::from("match.txt"), + start_line: 1, + end_line: 1, + content: b"predictive coding exact phrase continuation".to_vec(), + score: 0.0, + }, + ]; + + let scored = stage1_filter_with_universal_prior( + b"predictive coding exact phrase", + prior_root.to_string_lossy().as_ref(), + candidates, + &opts, + ) + .expect("stage1 with prior should succeed"); + assert_eq!(scored.len(), 2); + assert!( + scored[1].score > scored[0].score, + "prior-consistent candidate should outrank unrelated content" + ); + + let _ = fs::remove_dir_all(prior_root); + } + + #[test] + fn search_with_options_snippet_granularity_prefers_matching_window() { + let path = temp_path("snippet-search").with_extension("txt"); + let mut text = String::new(); + for i in 0..80 { + if i == 41 { + text.push_str("needle exact snippet phrase lives here\n"); + } else { + text.push_str(&format!("background line {i}\n")); + } + } + fs::write(&path, text.as_bytes()).expect("write snippet corpus"); + + let opts = SearchOptions { + granularity: SearchGranularity::Snippet, + top_k: 1, + stage0_keep_frac: 1.0, + ctx: ctw_search_ctx(), + ..SearchOptions::try_default().expect("search defaults") + }; + let results = search_with_options( + "needle exact snippet phrase", + path.to_string_lossy().as_ref(), + &opts, + ) + .expect("snippet search should succeed"); + assert_eq!(results.len(), 1); + assert_eq!(results[0].path, path); + assert!(results[0].start_line <= 42 && results[0].end_line >= 42); + + let _ = fs::remove_file(path); + } + + #[test] + fn search_with_options_file_granularity_truncates_and_sorts_results() { + let root = temp_path("file-search"); + fs::create_dir_all(&root).expect("create target dir"); + let best = root.join("best.txt"); + let second = root.join("second.txt"); + let noise = root.join("noise.txt"); + write_text( + &best, + "needle exact file phrase\nneedle exact file phrase\nneedle exact file phrase\n", + ); + write_text(&second, "needle exact file\npartial overlap only\n"); + write_text(&noise, "completely unrelated material\n"); + + let opts = SearchOptions { + granularity: SearchGranularity::File, + top_k: 2, + stage0_keep_frac: 1.0, + ctx: ctw_search_ctx(), + ..SearchOptions::try_default().expect("search defaults") + }; + let results = search_with_options( + "needle exact file phrase", + root.to_string_lossy().as_ref(), + &opts, + ) + .expect("file search should succeed"); + assert_eq!(results.len(), 2); + assert_eq!(results[0].path, best); + assert!(results[0].score >= results[1].score); + assert_ne!( + results[1].path, noise, + "noise candidate should be truncated away" + ); + + let _ = fs::remove_dir_all(root); + } + + #[cfg(feature = "backend-zpaq")] + #[test] + fn search_with_options_surfaces_runtime_backend_errors() { + let path = temp_path("search-runtime").with_extension("txt"); + fs::write(&path, b"haystack").expect("write search target"); + + let ctx = InfotheoryCtx::from_specs( + RateBackend::RosaPlus { max_order: -1 }, + CompressionBackend::zpaq("definitely-invalid-zpaq-method"), + ) + .expect("context should compile"); + + let opts = SearchOptions { + top_k: 1, + ctx, + ..SearchOptions::try_default().expect("search defaults") + }; + let err = search_with_options("needle", path.to_string_lossy().as_ref(), &opts) + .expect_err("invalid compression method should surface as a search error"); + assert!(matches!( + err, + InfotheoryError::Runtime(_) + | InfotheoryError::Unsupported(_) + | InfotheoryError::InvalidBackendConfig(_) + )); + + let _ = fs::remove_file(path); + } + + #[test] + fn search_defaults_match_feature_slice_defaults() { + let opts = SearchOptions::try_default().expect("search defaults"); + match opts.ctx.compression_backend.canonical_spec() { + #[cfg(feature = "backend-zpaq")] + crate::api::CompressionBackend::Zpaq { method, .. } => assert_eq!(method.value(), "5"), + #[cfg(feature = "backend-zpaq")] + crate::api::CompressionBackend::Rate { .. } => { + panic!("zpaq-enabled default search context should use zpaq compression"); + } + #[cfg(not(feature = "backend-zpaq"))] + crate::api::CompressionBackend::Rate { + rate_backend, + coder, + framing, + } => { + assert!(matches!( + rate_backend, + &crate::api::RateBackend::RosaPlus { .. } + )); + assert_eq!(*coder, crate::coders::CoderType::AC); + assert_eq!(*framing, crate::compression::FramingMode::Raw); + } + other => panic!( + "unexpected default search compression backend: {:?}", + other.kind() + ), + } + } + + #[test] + fn search_with_universal_prior_modes_keeps_exact_match_first_for_generic_rate_backend() { + let target_root = temp_path("prior-modes-target"); + let prior_root = temp_path("prior-modes-prior"); + fs::create_dir_all(&target_root).expect("create target dir"); + fs::create_dir_all(&prior_root).expect("create prior dir"); + + let relevant_path = target_root.join("relevant.txt"); + let distractor_path = target_root.join("distractor.txt"); + write_text( + &relevant_path, + "needle signal exact match\nneedle signal exact match\nneedle signal exact match\n", + ); + write_text( + &distractor_path, + "unrelated noise\nentropy without the exact query phrase\n", + ); + write_text( + &prior_root.join("prior.txt"), + "needle signal context\nbackground corpus bytes\n", + ); + + for mode in [ + Stage2PriorMode::Disable, + Stage2PriorMode::Use, + Stage2PriorMode::Summarize, + ] { + let opts = SearchOptions { + granularity: SearchGranularity::File, + universal_prior: Some(prior_root.to_string_lossy().to_string()), + stage2_prior_mode: mode, + top_k: 2, + stage0_keep_frac: 1.0, + ctx: ctw_search_ctx(), + }; + let results = search_with_options( + "needle signal exact match", + target_root.to_string_lossy().as_ref(), + &opts, + ) + .expect("search with prior should succeed"); + assert_eq!(results.len(), 2); + assert_eq!(results[0].path, relevant_path); + assert!( + results[0].score >= results[1].score, + "results must remain score-sorted after reranking for mode {mode:?}" + ); + } + + let _ = fs::remove_dir_all(target_root); + let _ = fs::remove_dir_all(prior_root); + } + + #[cfg(feature = "backend-rosa")] + #[test] + fn search_with_rosa_prior_model_training_ranks_relevant_file_first() { + let target_root = temp_path("rosa-prior-target"); + let prior_root = temp_path("rosa-prior-corpus"); + fs::create_dir_all(&target_root).expect("create target dir"); + fs::create_dir_all(&prior_root).expect("create prior dir"); + + let relevant_path = target_root.join("relevant.txt"); + write_text( + &relevant_path, + "predictive coding with exact entropy reduction signal\n\ + predictive coding with exact entropy reduction signal\n\ + predictive coding with exact entropy reduction signal\n", + ); + write_text( + &target_root.join("noise.txt"), + "generic unrelated text that should not outrank the exact match\n", + ); + write_text( + &prior_root.join("prior_a.txt"), + "predictive coding prior corpus\nentropy reduction prior corpus\n", + ); + write_text( + &prior_root.join("prior_b.txt"), + "additional prior conditioning bytes for the rosa branch\n", + ); + + let mut opts = SearchOptions::try_default().expect("search defaults"); + opts.granularity = SearchGranularity::File; + opts.universal_prior = Some(prior_root.to_string_lossy().to_string()); + opts.stage2_prior_mode = Stage2PriorMode::Use; + opts.top_k = 2; + opts.stage0_keep_frac = 1.0; + + let first = search_with_options( + "predictive coding exact entropy reduction signal", + target_root.to_string_lossy().as_ref(), + &opts, + ) + .expect("rosa prior search should succeed"); + let second = search_with_options( + "predictive coding exact entropy reduction signal", + target_root.to_string_lossy().as_ref(), + &opts, + ) + .expect("repeated rosa prior search should stay valid"); + + assert_eq!(first[0].path, relevant_path); + assert_eq!(second[0].path, relevant_path); + assert!( + first[0].score.is_finite() && second[0].score.is_finite(), + "rosa prior branch must produce finite scores" + ); + + let _ = fs::remove_dir_all(target_root); + let _ = fs::remove_dir_all(prior_root); + } + + #[test] + fn corpus_bytes_and_prior_cache_helpers_are_stable() { + let root = temp_path("corpus-root"); + fs::create_dir_all(&root).expect("create corpus dir"); + write_text(&root.join("a.txt"), "alpha"); + write_text(&root.join("b.txt"), "beta"); + + let bytes = corpus_bytes(root.to_string_lossy().as_ref(), SearchGranularity::File); + assert!(bytes.windows(5).any(|window| window == b"alpha")); + assert!(bytes.windows(4).any(|window| window == b"beta")); + assert!(bytes.windows(2).any(|window| window == b"\n\n")); + + let cache_a = prior_cache_path(root.to_string_lossy().as_ref(), 7).expect("cache path"); + let cache_b = prior_cache_path(root.to_string_lossy().as_ref(), 7).expect("cache path"); + let cache_c = prior_cache_path(root.to_string_lossy().as_ref(), 9).expect("cache path"); + assert_eq!(cache_a, cache_b); + assert_ne!(cache_a, cache_c); + + let _ = fs::remove_dir_all(root); + } + + #[test] + fn summarize_prior_for_query_returns_empty_when_prior_corpus_is_empty() { + let root = temp_path("empty-prior"); + fs::create_dir_all(&root).expect("create empty prior dir"); + let opts = SearchOptions { + granularity: SearchGranularity::File, + top_k: 1, + stage0_keep_frac: 1.0, + ctx: ctw_search_ctx(), + ..SearchOptions::try_default().expect("search defaults") + }; + let summary = summarize_prior_for_query(b"query", root.to_string_lossy().as_ref(), &opts) + .expect("summarization should succeed"); + assert!(summary.is_empty()); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn summarize_prior_for_query_and_stage2_rerank_prefer_relevant_content() { + let prior_root = temp_path("prior-summary"); + fs::create_dir_all(&prior_root).expect("create prior dir"); + write_text( + &prior_root.join("relevant.txt"), + "needle exact search phrase appears here\nneedle exact search phrase appears here\n", + ); + write_text( + &prior_root.join("noise.txt"), + "background corpus bytes with unrelated content\n", + ); + + let opts = SearchOptions { + granularity: SearchGranularity::File, + universal_prior: Some(prior_root.to_string_lossy().to_string()), + stage2_prior_mode: Stage2PriorMode::Summarize, + top_k: 2, + stage0_keep_frac: 1.0, + ctx: ctw_search_ctx(), + }; + let query = b"needle exact search phrase"; + let summary = + summarize_prior_for_query(query, prior_root.to_string_lossy().as_ref(), &opts) + .expect("prior summary"); + assert!( + String::from_utf8_lossy(&summary).contains("needle exact search phrase"), + "summary should select the relevant prior candidate" + ); + + let mut snippets = vec![ + Snippet { + path: prior_root.join("noise.txt"), + start_line: 1, + end_line: 1, + content: b"completely unrelated background".to_vec(), + score: 0.0, + }, + Snippet { + path: prior_root.join("relevant.txt"), + start_line: 1, + end_line: 1, + content: b"needle exact search phrase repeated".to_vec(), + score: 0.0, + }, + ]; + stage2_rerank_kmi(query, &mut snippets, &opts).expect("stage2 rerank"); + snippets.sort_by(|lhs, rhs| rhs.score.total_cmp(&lhs.score)); + assert_eq!(snippets[0].path, prior_root.join("relevant.txt")); + + let _ = fs::remove_dir_all(prior_root); + } + + #[cfg(feature = "backend-rosa")] + #[test] + fn load_or_train_prior_model_creates_and_reuses_cache() { + let prior_root = temp_path("rosa-cache-corpus"); + let cache_root = temp_path("rosa-cache-home"); + fs::create_dir_all(&prior_root).expect("create prior dir"); + fs::create_dir_all(&cache_root).expect("create cache dir"); + write_text( + &prior_root.join("prior.txt"), + "predictive coding prior text\npredictive coding prior text\n", + ); + + let old_cache = std::env::var("XDG_CACHE_HOME").ok(); + // Test-only process environment override. This test does not spawn + // threads or retain references into the environment across mutation. + unsafe { + std::env::set_var("XDG_CACHE_HOME", &cache_root); + } + + let opts = SearchOptions { + granularity: SearchGranularity::File, + universal_prior: Some(prior_root.to_string_lossy().to_string()), + stage2_prior_mode: Stage2PriorMode::Use, + top_k: 1, + stage0_keep_frac: 1.0, + ctx: InfotheoryCtx::from_specs( + RateBackend::RosaPlus { max_order: 7 }, + CompressionBackend::Rate { + rate_backend: RateBackend::RosaPlus { max_order: 7 }, + coder: crate::coders::CoderType::AC, + framing: crate::compression::FramingMode::Raw, + }, + ) + .expect("rosa search context"), + }; + + let prior_path = prior_root.to_string_lossy().to_string(); + let cache_path = prior_cache_path(&prior_path, 7).expect("cache path"); + assert!(!cache_path.exists()); + + let first = load_or_train_prior_model(&prior_path, &opts); + assert!(cache_path.exists(), "training should populate cache"); + let second = load_or_train_prior_model(&prior_path, &opts); + assert_eq!(first.lm_alpha_n(), 256); + assert_eq!(second.lm_alpha_n(), 256); + + match old_cache { + Some(value) => unsafe { + // Restore the original process environment after the isolated + // cache-path test finishes. + std::env::set_var("XDG_CACHE_HOME", value); + }, + None => unsafe { + // Restore the pre-test absence of `XDG_CACHE_HOME`. + std::env::remove_var("XDG_CACHE_HOME"); + }, + } + + let _ = fs::remove_dir_all(prior_root); + let _ = fs::remove_dir_all(cache_root); + } +} diff --git a/crates/infotheory/src/simd_math.rs b/crates/infotheory/src/simd_math.rs new file mode 100644 index 00000000..08834957 --- /dev/null +++ b/crates/infotheory/src/simd_math.rs @@ -0,0 +1,213 @@ +use wide::f64x4; + +#[allow(dead_code)] +#[inline] +pub(crate) fn dot_wide(lhs: &[f64], rhs: &[f64]) -> f64 { + let n = lhs.len().min(rhs.len()); + let mut acc = f64x4::ZERO; + let mut i = 0usize; + while i + 4 <= n { + let a = f64x4::new([lhs[i], lhs[i + 1], lhs[i + 2], lhs[i + 3]]); + let b = f64x4::new([rhs[i], rhs[i + 1], rhs[i + 2], rhs[i + 3]]); + acc += a * b; + i += 4; + } + let lanes = acc.to_array(); + let mut out = lanes[0] + lanes[1] + lanes[2] + lanes[3]; + while i < n { + out += lhs[i] * rhs[i]; + i += 1; + } + out +} + +#[allow(dead_code)] +#[inline] +pub(crate) fn max_wide(xs: &[f64]) -> f64 { + if xs.is_empty() { + return f64::NEG_INFINITY; + } + let mut i = 0usize; + let mut max4 = f64x4::splat(f64::NEG_INFINITY); + while i + 4 <= xs.len() { + let v = f64x4::new([xs[i], xs[i + 1], xs[i + 2], xs[i + 3]]); + max4 = max4.max(v); + i += 4; + } + let lanes = max4.to_array(); + let mut max_v = lanes[0].max(lanes[1]).max(lanes[2]).max(lanes[3]); + while i < xs.len() { + if xs[i] > max_v { + max_v = xs[i]; + } + i += 1; + } + max_v +} + +#[allow(dead_code)] +#[inline] +pub(crate) fn logsumexp_wide(xs: &[f64]) -> f64 { + let max_v = max_wide(xs); + if !max_v.is_finite() { + return max_v; + } + let mut sum = 0.0; + for &v in xs { + sum += (v - max_v).exp(); + } + max_v + sum.ln() +} + +#[allow(dead_code)] +#[inline] +pub(crate) fn axpy_wide(dst: &mut [f64], alpha: f64, src: &[f64]) { + let n = dst.len().min(src.len()); + let mut i = 0usize; + let a4 = f64x4::splat(alpha); + while i + 4 <= n { + let d = f64x4::new([dst[i], dst[i + 1], dst[i + 2], dst[i + 3]]); + let s = f64x4::new([src[i], src[i + 1], src[i + 2], src[i + 3]]); + let r = d + a4 * s; + let lanes = r.to_array(); + dst[i] = lanes[0]; + dst[i + 1] = lanes[1]; + dst[i + 2] = lanes[2]; + dst[i + 3] = lanes[3]; + i += 4; + } + while i < n { + dst[i] += alpha * src[i]; + i += 1; + } +} + +#[allow(dead_code)] +#[inline] +pub(crate) fn affine3_wide( + dst: &mut [f64], + bias: &[f64], + weights: [f64; 3], + src0: &[f64], + src1: &[f64], + src2: &[f64], +) { + let n = dst.len(); + assert!(bias.len() >= n, "bias shorter than dst"); + assert!(src0.len() >= n, "src0 shorter than dst"); + assert!(src1.len() >= n, "src1 shorter than dst"); + assert!(src2.len() >= n, "src2 shorter than dst"); + let mut i = 0usize; + let w0 = f64x4::splat(weights[0]); + let w1 = f64x4::splat(weights[1]); + let w2 = f64x4::splat(weights[2]); + while i + 4 <= n { + let b = f64x4::new([bias[i], bias[i + 1], bias[i + 2], bias[i + 3]]); + let x0 = f64x4::new([src0[i], src0[i + 1], src0[i + 2], src0[i + 3]]); + let x1 = f64x4::new([src1[i], src1[i + 1], src1[i + 2], src1[i + 3]]); + let x2 = f64x4::new([src2[i], src2[i + 1], src2[i + 2], src2[i + 3]]); + let r = b + w0 * x0 + w1 * x1 + w2 * x2; + let lanes = r.to_array(); + dst[i] = lanes[0]; + dst[i + 1] = lanes[1]; + dst[i + 2] = lanes[2]; + dst[i + 3] = lanes[3]; + i += 4; + } + while i < n { + dst[i] = bias[i] + weights[0] * src0[i] + weights[1] * src1[i] + weights[2] * src2[i]; + i += 1; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn assert_close(lhs: f64, rhs: f64, tol: f64) { + let delta: f64 = (lhs - rhs).abs(); + assert!( + delta <= tol, + "lhs={lhs}, rhs={rhs}, delta={delta}, tol={tol}" + ); + } + + #[test] + fn dot_and_axpy_cover_vector_and_scalar_tails() { + let lhs: Vec = vec![1.0, -2.0, 3.0, 4.0, 5.0, 9.0]; + let rhs: Vec = vec![0.5, 2.0, -1.0, 0.25, -3.0]; + let dot: f64 = dot_wide(&lhs, &rhs); + let expected_dot: f64 = lhs.iter().zip(rhs.iter()).map(|(a, b)| a * b).sum::(); + assert_close(dot, expected_dot, 1e-12); + + let mut dst: Vec = vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0]; + let src: Vec = vec![10.0, -1.0, 2.0, 0.5, -4.0]; + axpy_wide(&mut dst, 0.25, &src); + let expected_dst: Vec = vec![ + 1.0 + 0.25 * 10.0, + 2.0 - 0.25, + 3.0 + 0.25 * 2.0, + 4.0 + 0.25 * 0.5, + 5.0 + 0.25 * -4.0, + 6.0, + ]; + assert_eq!(dst, expected_dst); + } + + #[test] + fn max_and_logsumexp_cover_empty_and_finite_inputs() { + assert_eq!(max_wide(&[]), f64::NEG_INFINITY); + assert_eq!(logsumexp_wide(&[]), f64::NEG_INFINITY); + + let xs: Vec = vec![-3.0, -1.0, -2.0, -4.0, -0.5]; + assert_close(max_wide(&xs), -0.5, 1e-12); + + let max_v: f64 = xs.iter().copied().fold(f64::NEG_INFINITY, f64::max); + let expected_lse: f64 = max_v + xs.iter().map(|v| (v - max_v).exp()).sum::().ln(); + assert_close(logsumexp_wide(&xs), expected_lse, 1e-12); + } + + #[test] + fn affine3_wide_matches_scalar_reference() { + let bias: Vec = vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0]; + let src0: Vec = vec![0.0, 1.0, 0.5, -1.0, 2.0, 1.5]; + let src1: Vec = vec![2.0, 0.0, -1.0, 1.0, 0.5, -0.5]; + let src2: Vec = vec![1.0, -2.0, 3.0, 0.5, -1.5, 2.5]; + let weights: [f64; 3] = [0.25, -0.5, 1.5]; + let mut dst: Vec = vec![0.0; bias.len()]; + + affine3_wide(&mut dst, &bias, weights, &src0, &src1, &src2); + + let expected: Vec = (0..bias.len()) + .map(|i| bias[i] + weights[0] * src0[i] + weights[1] * src1[i] + weights[2] * src2[i]) + .collect(); + assert_eq!(dst, expected); + } + + #[test] + fn affine3_wide_panics_on_short_inputs() { + let mut dst: Vec = vec![0.0; 4]; + let bias: Vec = vec![0.0; 4]; + let src: Vec = vec![0.0; 4]; + + let short_bias = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + affine3_wide(&mut dst, &bias[..3], [1.0, 1.0, 1.0], &src, &src, &src); + })); + assert!(short_bias.is_err()); + + let short_src0 = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + affine3_wide(&mut dst, &bias, [1.0, 1.0, 1.0], &src[..3], &src, &src); + })); + assert!(short_src0.is_err()); + + let short_src1 = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + affine3_wide(&mut dst, &bias, [1.0, 1.0, 1.0], &src, &src[..3], &src); + })); + assert!(short_src1.is_err()); + + let short_src2 = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + affine3_wide(&mut dst, &bias, [1.0, 1.0, 1.0], &src, &src, &src[..3]); + })); + assert!(short_src2.is_err()); + } +} diff --git a/crates/infotheory/src/spec.rs b/crates/infotheory/src/spec.rs new file mode 100644 index 00000000..ab425806 --- /dev/null +++ b/crates/infotheory/src/spec.rs @@ -0,0 +1,3690 @@ +//! Canonical backend/spec parsing shared by Rust, CLI, and Python surfaces. + +pub mod core; +mod document; + +pub use self::core::{ + AssetRef, CanonicalBytes, CompiledCompressionBackend, CompiledRateBackend, + CompressionBackendCapabilities, MethodBackendFamily, RateBackendCapabilities, + RateBackendTraceStrategy, SpecEnvironment, ValidatedCompressionBackend, ValidatedRateBackend, +}; +#[cfg(feature = "tuner")] +pub(crate) use self::document::TuneInvalidReason; +#[cfg(feature = "aixi")] +pub use self::document::WarmStartExactJhControllerSpec; +pub use self::document::{ + AiqiDiscountedControllerSpec, AssetBinding, AssetId, BuiltinEnvironmentSpec, + CompiledPlannerController, CompiledPlannerRunSpec, CompiledSpecDocument, ControllerSpec, + EnvironmentSpec, McAixiControllerSpec, ParsedSpecDocument, PlannerInterfaceSpec, + PlannerRunSpec, PlannerRuntimeSpec, ResolvedAssetBinding, SharedMemoryPolicySpec, SpecDocument, + ValidatedPlannerRunSpec, ValidatedSpecDocument, VmActionFilterSpec, VmEnvironmentSpec, + VmFuzzMutatorSpec, VmObservationPolicySpec, VmObservationStreamModeSpec, VmPayloadEncodingSpec, + VmRewardPolicySpec, VmRewardShapingSpec, VmRuntimeActionSourceSpec, VmTraceSpec, + load_spec_document, +}; +#[cfg(feature = "tuner")] +pub use self::document::{ + AiqiDiscountedTuneControllerSpec, AnnealedHillClimbingTuneControllerSpec, + CompiledTuneController, CompiledTuneSpec, McAixiFacCtwTuneControllerSpec, TuneBoundsSpec, + TuneControllerKind, TuneControllerSpec, TuneParameterRangeSpec, TunePlannerInterfaceSpec, + TuneSpec, ValidatedTuneSpec, WarmStartExactJhTuneControllerSpec, +}; + +use crate::api::{ + CalibratedSpec, CalibrationContextKind, CompressionBackend, MAX_MIXTURE_NESTING, + MixtureExpertSpec, MixtureKind, MixtureScheduleMode, MixtureSpec, ParticleSpec, RateBackend, + parse_mixture_kind_name, parse_mixture_schedule_name, +}; +use crate::validate_zpaq_rate_method; +use std::error::Error; +use std::fmt; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +/// Result type used by the shared spec/parsing layer. +pub type SpecResult = Result; + +/// Serialize JSON hash payloads with recursive lexicographic object-key order. +/// +/// This is the crate-local byte contract for CRC/SHA commitments over ad-hoc +/// JSON payloads. It deliberately avoids relying on `serde_json::Map`'s backing +/// type or feature-unified insertion-order behavior. +#[cfg(feature = "aixi")] +pub(crate) fn canonical_json_bytes( + value: &serde_json::Value, +) -> Result, serde_json::Error> { + let mut bytes = Vec::::new(); + write_canonical_json_value(value, &mut bytes)?; + Ok(bytes) +} + +#[cfg(feature = "aixi")] +fn write_canonical_json_value( + value: &serde_json::Value, + out: &mut Vec, +) -> Result<(), serde_json::Error> { + match value { + serde_json::Value::Array(items) => { + out.push(b'['); + for (index, item) in items.iter().enumerate() { + if index > 0 { + out.push(b','); + } + write_canonical_json_value(item, out)?; + } + out.push(b']'); + Ok(()) + } + serde_json::Value::Object(object) => { + let mut entries = object.iter().collect::>(); + entries.sort_unstable_by(|(left, _), (right, _)| left.cmp(right)); + out.push(b'{'); + for (index, (key, item)) in entries.into_iter().enumerate() { + if index > 0 { + out.push(b','); + } + serde_json::to_writer(&mut *out, key)?; + out.push(b':'); + write_canonical_json_value(item, out)?; + } + out.push(b'}'); + Ok(()) + } + scalar => serde_json::to_writer(out, scalar), + } +} + +/// Trait for deterministic canonical JSON serialization. +pub trait CanonicalJson { + /// Serialize this value into canonical JSON value form. + fn to_canonical_json_value(&self) -> SpecResult; + + /// Serialize this value into deterministic canonical JSON text. + fn to_canonical_json(&self) -> SpecResult { + serde_json::to_string_pretty(&self.to_canonical_json_value()?).map_err(SpecError::from) + } +} + +/// Lightweight error type for spec/config parsing and loading. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct SpecError { + message: String, +} + +impl SpecError { + /// Create a new spec error from a message. + pub fn new(message: impl Into) -> Self { + Self { + message: message.into(), + } + } +} + +impl fmt::Display for SpecError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.message) + } +} + +impl Error for SpecError {} + +impl From<&str> for SpecError { + fn from(value: &str) -> Self { + Self::new(value) + } +} + +impl From for SpecError { + fn from(value: String) -> Self { + Self::new(value) + } +} + +impl From for SpecError { + fn from(value: std::io::Error) -> Self { + Self::new(value.to_string()) + } +} + +impl From for SpecError { + fn from(value: serde_json::Error) -> Self { + Self::new(value.to_string()) + } +} + +/// Defaults for shorthand backend parsing such as CLI `--rate-backend ... --method ...`. +#[derive(Clone, Debug)] +#[non_exhaustive] +pub struct RateBackendShorthandOptions { + /// Base directory used to resolve relative spec/model paths. + pub base_dir: PathBuf, + /// Default CTW depth when no numeric method is supplied. + pub ctw_depth: usize, + /// Default FAC-CTW base depth when no numeric method is supplied. + pub fac_ctw_base_depth: usize, + /// Default FAC-CTW percept width. + pub fac_ctw_num_percept_bits: usize, + /// Default FAC-CTW symbol encoding width. + pub fac_ctw_encoding_bits: usize, + /// Optional FAC-CTW MSB-first override for shorthand CLI parsing. + /// + /// `None` defers to compile-time default (`encoding_bits == 8` → MSB-first). + pub fac_ctw_msb_first: Option, + /// Default PPMD order. + pub ppmd_order: usize, + /// Default PPMD memory budget in MiB. + pub ppmd_memory_mb: usize, + /// Default Sequitur context window. + pub sequitur_context_bytes: usize, + /// Default ZPAQ rate method. + pub zpaq_method: String, + /// Optional default Mamba model path when no method is supplied. + pub default_mamba_model_path: Option, + /// Optional default RWKV7 model path when no method is supplied. + pub default_rwkv_model_path: Option, + /// Whether `particle` without a method should build `ParticleSpec::default()`. + pub particle_default_if_missing_method: bool, +} + +impl Default for RateBackendShorthandOptions { + fn default() -> Self { + Self { + base_dir: PathBuf::from("."), + ctw_depth: crate::rate_defaults::SHORTHAND_DEFAULT_CTW_DEPTH, + fac_ctw_base_depth: crate::rate_defaults::SHORTHAND_DEFAULT_FAC_CTW_BASE_DEPTH, + fac_ctw_num_percept_bits: + crate::rate_defaults::SHORTHAND_DEFAULT_FAC_CTW_NUM_PERCEPT_BITS, + fac_ctw_encoding_bits: crate::rate_defaults::SHORTHAND_DEFAULT_FAC_CTW_ENCODING_BITS, + fac_ctw_msb_first: None, + ppmd_order: crate::rate_defaults::SHORTHAND_DEFAULT_PPMD_ORDER, + ppmd_memory_mb: crate::rate_defaults::SHORTHAND_DEFAULT_PPMD_MEMORY_MB, + sequitur_context_bytes: crate::rate_defaults::SHORTHAND_DEFAULT_SEQUITUR_CONTEXT_BYTES, + zpaq_method: crate::rate_defaults::SHORTHAND_DEFAULT_ZPAQ_RATE_METHOD.to_string(), + default_mamba_model_path: None, + default_rwkv_model_path: None, + particle_default_if_missing_method: true, + } + } +} + +/// Defaults for shorthand compression-backend parsing such as +/// CLI/Python `--compression-backend ... --method ...`. +#[derive(Clone)] +#[non_exhaustive] +pub struct CompressionBackendShorthandOptions { + /// Base directory used to resolve relative model/spec paths. + pub base_dir: PathBuf, + /// Default ZPAQ compression method. + pub zpaq_method: String, + /// Default rate backend for `rate-ac`/`rate-rans` shorthands. + pub default_rate_backend: Option, + /// Default framing mode for generic rate-coded compression backends. + pub default_framing: crate::compression::FramingMode, + /// Optional default RWKV7 model path when no method is supplied. + pub default_rwkv_model_path: Option, +} + +impl Default for CompressionBackendShorthandOptions { + fn default() -> Self { + Self { + base_dir: PathBuf::from("."), + zpaq_method: "5".to_string(), + default_rate_backend: None, + default_framing: crate::compression::FramingMode::Framed, + default_rwkv_model_path: None, + } + } +} + +/// Resolve a spec path against a base directory. +/// +/// Absolute paths are returned unchanged; relative paths are joined to `base_dir`. +pub fn resolve_spec_path(base_dir: &Path, path: impl AsRef) -> PathBuf { + let path = path.as_ref(); + if path.is_absolute() { + path.to_path_buf() + } else { + base_dir.join(path) + } +} + +/// Resolve a rate backend alias and require that it is enabled in the current build. +fn resolve_enabled_rate_backend_kind(input: &str) -> SpecResult { + match crate::runtime::find_backend_descriptor_in_registry( + crate::runtime::RATE_BACKEND_REGISTRY, + input, + ) { + Some(descriptor) if descriptor.enabled => Ok(descriptor.kind), + Some(descriptor) => Err(SpecError::new(format!( + "backend '{}' requires infotheory feature '{}'", + descriptor.canonical, + descriptor + .feature + .unwrap_or("__internal-registry-mismatch__") + ))), + None => Err(SpecError::new(format!("unknown backend '{input}'"))), + } +} + +/// Resolve a rate backend alias and require that it is enabled in the current build. +pub fn resolve_enabled_rate_backend_name(input: &str) -> SpecResult<&'static str> { + let kind = resolve_enabled_rate_backend_kind(input)?; + Ok(crate::runtime::describe_rate_backend_kind(kind) + .map_err(SpecError::new)? + .canonical) +} + +/// Resolve a compression backend alias and require that it is enabled in the current build. +fn resolve_enabled_compression_backend_kind( + input: &str, +) -> SpecResult { + match crate::runtime::find_backend_descriptor_in_registry( + crate::runtime::COMPRESSION_BACKEND_REGISTRY, + input, + ) { + Some(descriptor) if descriptor.enabled => Ok(descriptor.kind), + Some(descriptor) => Err(SpecError::new(format!( + "compression backend '{}' requires infotheory feature '{}'", + descriptor.canonical, + descriptor + .feature + .unwrap_or("__internal-registry-mismatch__") + ))), + None => Err(SpecError::new(format!( + "unknown compression backend '{input}'" + ))), + } +} + +/// Resolve a compression backend alias and require that it is enabled in the current build. +pub fn resolve_enabled_compression_backend_name(input: &str) -> SpecResult<&'static str> { + let kind = resolve_enabled_compression_backend_kind(input)?; + Ok(crate::runtime::describe_compression_backend_kind(kind) + .map_err(SpecError::new)? + .canonical) +} + +fn resolve_default_rate_backend_spec( + default_rate_backend: Option, +) -> SpecResult { + default_rate_backend.map(Ok).unwrap_or_else(|| { + RateBackend::try_default().map_err(|err| SpecError::new(err.to_string())) + }) +} + +/// Load a plain JSON value from disk, resolving relative paths against `base_dir`. +pub fn load_json_value_from_path( + base_dir: &Path, + path: &str, + label: &str, +) -> SpecResult<(serde_json::Value, PathBuf)> { + let full = resolve_spec_path(base_dir, path); + let raw = std::fs::read(&full) + .map_err(|e| SpecError::new(format!("failed to read {label} '{}': {e}", full.display())))?; + let value = serde_json::from_slice(&raw) + .map_err(|e| SpecError::new(format!("invalid {label} JSON '{}': {e}", full.display())))?; + Ok((value, full)) +} + +fn parse_calibration_context_kind(value: Option<&str>) -> SpecResult { + match value.unwrap_or("text").trim().to_ascii_lowercase().as_str() { + "global" => Ok(CalibrationContextKind::Global), + "byteclass" => Ok(CalibrationContextKind::ByteClass), + "text" => Ok(CalibrationContextKind::Text), + "repeat" => Ok(CalibrationContextKind::Repeat), + "textrepeat" => Ok(CalibrationContextKind::TextRepeat), + other => Err(SpecError::new(format!( + "unknown calibration context '{other}'" + ))), + } +} + +fn parse_mixture_kind(kind: &str) -> SpecResult { + parse_mixture_kind_name(kind).map_err(SpecError::from) +} + +fn parse_mixture_schedule(schedule: &str) -> SpecResult { + parse_mixture_schedule_name(schedule).map_err(SpecError::from) +} + +fn parse_framing_mode(value: Option<&str>) -> SpecResult { + match value + .unwrap_or("framed") + .trim() + .to_ascii_lowercase() + .as_str() + { + "framed" => Ok(crate::compression::FramingMode::Framed), + "raw" => Ok(crate::compression::FramingMode::Raw), + other => Err(SpecError::new(format!("unknown framing mode '{other}'"))), + } +} + +fn mixture_kind_name(kind: MixtureKind) -> &'static str { + match kind { + MixtureKind::Bayes => "bayes", + MixtureKind::FadingBayes => "fading-bayes", + MixtureKind::Switching => "switching", + MixtureKind::Convex => "convex", + MixtureKind::Mdl => "mdl", + MixtureKind::Neural => "neural", + } +} + +fn mixture_schedule_name(schedule: MixtureScheduleMode) -> &'static str { + match schedule { + MixtureScheduleMode::Default => "default", + MixtureScheduleMode::Theorem => "theorem", + } +} + +fn calibration_context_kind_name(kind: CalibrationContextKind) -> &'static str { + match kind { + CalibrationContextKind::Global => "global", + CalibrationContextKind::ByteClass => "byteclass", + CalibrationContextKind::Text => "text", + CalibrationContextKind::Repeat => "repeat", + CalibrationContextKind::TextRepeat => "textrepeat", + } +} + +fn framing_mode_name(mode: crate::compression::FramingMode) -> &'static str { + match mode { + crate::compression::FramingMode::Raw => "raw", + crate::compression::FramingMode::Framed => "framed", + } +} + +#[cfg(any(feature = "backend-rwkv", feature = "backend-mamba"))] +fn canonicalize_explicit_file_method( + base_dir: &Path, + method: &str, + backend_label: &str, +) -> SpecResult> { + let (base, policy) = crate::backends::llm_policy::split_method_policy_segments(method) + .map_err(|err| SpecError::new(err.to_string()))?; + let Some(path) = base.strip_prefix("file:") else { + return Ok(None); + }; + let path = crate::backends::llm_policy::parse_method_file_path(path.trim()); + if path.as_os_str().is_empty() { + return Err(SpecError::new(format!( + "empty file path in {backend_label} method" + ))); + } + let full = resolve_spec_path(base_dir, &path); + let mut canonical = format!( + "file:{}", + crate::backends::llm_policy::render_method_file_path(&full) + ); + if let Some(policy) = policy { + canonical.push_str(";policy:"); + canonical.push_str(policy.trim()); + } + Ok(Some(canonical)) +} + +#[cfg(feature = "backend-rwkv")] +fn validate_rwkv_method_eager(method: &str) -> SpecResult<()> { + crate::rwkvzip::Compressor::new_from_method(method) + .map(|_| ()) + .map_err(|err| SpecError::new(err.to_string())) +} + +#[cfg(feature = "backend-mamba")] +fn validate_mamba_method_eager(method: &str) -> SpecResult<()> { + crate::mambazip::Compressor::new_from_method(method) + .map(|_| ()) + .map_err(|err| SpecError::new(err.to_string())) +} + +#[cfg(feature = "backend-rwkv")] +fn normalize_rwkv_method_spec_for_base_dir( + base_dir: &Path, + method: &crate::rwkvzip::MethodSpec, +) -> SpecResult { + let normalized = match method { + crate::rwkvzip::MethodSpec::File { path, policy } => crate::rwkvzip::MethodSpec::File { + path: resolve_spec_path(base_dir, path), + policy: policy.clone(), + }, + crate::rwkvzip::MethodSpec::Online { cfg, policy } => crate::rwkvzip::MethodSpec::Online { + cfg: cfg.clone(), + policy: policy.clone(), + }, + }; + let canonical = crate::rwkvzip::canonical_method_string(&normalized) + .map_err(|err| SpecError::new(err.to_string()))?; + validate_rwkv_method_eager(&canonical)?; + Ok(normalized) +} + +#[cfg(feature = "backend-rwkv")] +fn normalize_rwkv_path_method( + base_dir: &Path, + model_path: &str, +) -> SpecResult { + normalize_rwkv_method_spec_for_base_dir( + base_dir, + &crate::rwkvzip::MethodSpec::File { + path: PathBuf::from(model_path), + policy: None, + }, + ) +} + +#[cfg(feature = "backend-mamba")] +fn normalize_mamba_method_spec_for_base_dir( + base_dir: &Path, + method: &crate::mambazip::MethodSpec, +) -> SpecResult { + let normalized = match method { + crate::mambazip::MethodSpec::File { path, policy } => crate::mambazip::MethodSpec::File { + path: resolve_spec_path(base_dir, path), + policy: policy.clone(), + }, + crate::mambazip::MethodSpec::Online { cfg, policy } => { + crate::mambazip::MethodSpec::Online { + cfg: cfg.clone(), + policy: policy.clone(), + } + } + }; + let canonical = crate::mambazip::canonical_method_string(&normalized) + .map_err(|err| SpecError::new(err.to_string()))?; + validate_mamba_method_eager(&canonical)?; + Ok(normalized) +} + +#[cfg(feature = "backend-mamba")] +fn normalize_mamba_path_method( + base_dir: &Path, + model_path: &str, +) -> SpecResult { + normalize_mamba_method_spec_for_base_dir( + base_dir, + &crate::mambazip::MethodSpec::File { + path: PathBuf::from(model_path), + policy: None, + }, + ) +} + +#[cfg(feature = "backend-rwkv")] +fn normalize_rwkv_method_for_base_dir( + base_dir: &Path, + method: &str, +) -> SpecResult { + if let Some(canonical) = canonicalize_explicit_file_method(base_dir, method, "rwkv")? { + let parsed = crate::rwkvzip::parse_method_spec(&canonical) + .map_err(|err| SpecError::new(err.to_string()))?; + normalize_rwkv_method_spec_for_base_dir(base_dir, &parsed) + } else { + let parsed = crate::rwkvzip::parse_method_spec(method) + .map_err(|err| SpecError::new(err.to_string()))?; + normalize_rwkv_method_spec_for_base_dir(base_dir, &parsed) + } +} + +#[cfg(feature = "backend-mamba")] +fn normalize_mamba_method_for_base_dir( + base_dir: &Path, + method: &str, +) -> SpecResult { + if let Some(canonical) = canonicalize_explicit_file_method(base_dir, method, "mamba")? { + let parsed = crate::mambazip::parse_method_spec(&canonical) + .map_err(|err| SpecError::new(err.to_string()))?; + normalize_mamba_method_spec_for_base_dir(base_dir, &parsed) + } else { + let parsed = crate::mambazip::parse_method_spec(method) + .map_err(|err| SpecError::new(err.to_string()))?; + normalize_mamba_method_spec_for_base_dir(base_dir, &parsed) + } +} + +#[cfg(feature = "backend-rwkv")] +const RWKV_POLICY_SCOPES: &[&str] = &[ + "embed", + "pre_norm", + "attn_norm", + "ffn_norm", + "attn", + "ffn", + "head", + "bias", + "all", + "none", +]; + +#[cfg(feature = "backend-mamba")] +const MAMBA_POLICY_SCOPES: &[&str] = &[ + "embed", + "layer_norm", + "mixer_conv", + "mixer_ssm", + "mixer_proj", + "head", + "bias", + "all", + "none", +]; + +fn zpaq_method_to_json_value(method: &crate::api::ZpaqMethodSpec) -> serde_json::Value { + match method { + crate::api::ZpaqMethodSpec::Literal { value } => serde_json::json!({ + "kind": "literal", + "value": value, + }), + } +} + +fn parse_zpaq_method_json_value( + value: &serde_json::Value, + default: &str, +) -> SpecResult { + if value.is_null() { + return Ok(crate::api::ZpaqMethodSpec::literal(default)); + } + if value.is_string() { + return Err(SpecError::new( + "zpaq method must use object form {'kind':'literal','value':'...'}", + )); + } + + let kind = value["kind"] + .as_str() + .ok_or_else(|| SpecError::new("zpaq method.kind is required for object form"))?; + if kind != "literal" { + return Err(SpecError::new(format!("unknown zpaq method kind '{kind}'"))); + } + + let method = value["value"] + .as_str() + .ok_or_else(|| SpecError::new("zpaq method.value must be a string"))?; + Ok(crate::api::ZpaqMethodSpec::literal(method)) +} + +#[cfg(feature = "backend-rwkv")] +fn rwkv_online_config_to_json_value(cfg: &crate::rwkvzip::OnlineConfig) -> serde_json::Value { + serde_json::json!({ + "hidden": cfg.hidden, + "layers": cfg.layers, + "intermediate": cfg.intermediate, + "decay_rank": cfg.decay_rank, + "a_rank": cfg.a_rank, + "v_rank": cfg.v_rank, + "g_rank": cfg.g_rank, + "seed": cfg.seed, + "train_mode": match cfg.train_mode { + crate::rwkvzip::OnlineTrainMode::None => "none", + crate::rwkvzip::OnlineTrainMode::Sgd => "sgd", + crate::rwkvzip::OnlineTrainMode::Adam => "adam", + }, + "lr": cfg.lr, + "stride": cfg.stride, + }) +} + +#[cfg(feature = "backend-rwkv")] +fn rwkv_online_config_from_json_value( + value: &serde_json::Value, +) -> SpecResult { + let defaults = crate::rwkvzip::OnlineConfig::default(); + let train_mode = match value["train_mode"].as_str().unwrap_or("none") { + "none" => crate::rwkvzip::OnlineTrainMode::None, + "sgd" => crate::rwkvzip::OnlineTrainMode::Sgd, + "adam" => crate::rwkvzip::OnlineTrainMode::Adam, + other => { + return Err(SpecError::new(format!("unknown rwkv train_mode '{other}'"))); + } + }; + Ok(crate::rwkvzip::OnlineConfig { + hidden: value["hidden"].as_u64().unwrap_or(defaults.hidden as u64) as usize, + layers: value["layers"].as_u64().unwrap_or(defaults.layers as u64) as usize, + intermediate: value["intermediate"] + .as_u64() + .unwrap_or(defaults.intermediate as u64) as usize, + decay_rank: value["decay_rank"] + .as_u64() + .unwrap_or(defaults.decay_rank as u64) as usize, + a_rank: value["a_rank"].as_u64().unwrap_or(defaults.a_rank as u64) as usize, + v_rank: value["v_rank"].as_u64().unwrap_or(defaults.v_rank as u64) as usize, + g_rank: value["g_rank"].as_u64().unwrap_or(defaults.g_rank as u64) as usize, + seed: value["seed"].as_u64().unwrap_or(defaults.seed), + train_mode, + lr: value["lr"].as_f64().unwrap_or(defaults.lr as f64) as f32, + stride: value["stride"].as_u64().unwrap_or(defaults.stride as u64) as usize, + }) +} + +#[cfg(feature = "backend-rwkv")] +fn rwkv_method_to_json_value(method: &crate::rwkvzip::MethodSpec) -> SpecResult { + Ok(match method { + crate::rwkvzip::MethodSpec::File { path, policy } => serde_json::json!({ + "kind": "file", + "path": path.to_string_lossy(), + "policy": policy.as_ref().map(crate::backends::llm_policy::LlmPolicy::canonical), + }), + crate::rwkvzip::MethodSpec::Online { cfg, policy } => serde_json::json!({ + "kind": "online", + "cfg": rwkv_online_config_to_json_value(cfg), + "policy": policy.as_ref().map(crate::backends::llm_policy::LlmPolicy::canonical), + }), + }) +} + +#[cfg(feature = "backend-rwkv")] +fn parse_rwkv_method_json_value( + value: &serde_json::Value, + base_dir: &Path, +) -> SpecResult { + if let Some(method) = value.as_str() { + return normalize_rwkv_method_for_base_dir(base_dir, method); + } + match value["kind"].as_str().unwrap_or("file") { + "file" => { + let path = value["path"] + .as_str() + .ok_or_else(|| SpecError::new("rwkv method.path is required"))?; + let policy = value["policy"] + .as_str() + .map(|raw| { + crate::backends::llm_policy::parse_policy_segment(raw, RWKV_POLICY_SCOPES) + }) + .transpose() + .map_err(|err| SpecError::new(err.to_string()))?; + normalize_rwkv_method_spec_for_base_dir( + base_dir, + &crate::rwkvzip::MethodSpec::File { + path: resolve_spec_path(base_dir, path), + policy, + }, + ) + } + "online" => { + let cfg = rwkv_online_config_from_json_value(&value["cfg"])?; + let policy = value["policy"] + .as_str() + .map(|raw| { + crate::backends::llm_policy::parse_policy_segment(raw, RWKV_POLICY_SCOPES) + }) + .transpose() + .map_err(|err| SpecError::new(err.to_string()))?; + normalize_rwkv_method_spec_for_base_dir( + base_dir, + &crate::rwkvzip::MethodSpec::Online { cfg, policy }, + ) + } + other => Err(SpecError::new(format!( + "unknown rwkv method kind '{other}'" + ))), + } +} + +#[cfg(feature = "backend-mamba")] +fn mamba_online_config_to_json_value(cfg: &crate::mambazip::OnlineConfig) -> serde_json::Value { + serde_json::json!({ + "hidden": cfg.hidden, + "layers": cfg.layers, + "intermediate": cfg.intermediate, + "state": cfg.state, + "conv": cfg.conv, + "dt_rank": cfg.dt_rank, + "seed": cfg.seed, + "train_mode": match cfg.train_mode { + crate::mambazip::OnlineTrainMode::None => "none", + crate::mambazip::OnlineTrainMode::Sgd => "sgd", + crate::mambazip::OnlineTrainMode::Adam => "adam", + }, + "lr": cfg.lr, + "stride": cfg.stride, + }) +} + +#[cfg(feature = "backend-mamba")] +fn mamba_online_config_from_json_value( + value: &serde_json::Value, +) -> SpecResult { + let defaults = crate::mambazip::OnlineConfig::default(); + let train_mode = match value["train_mode"].as_str().unwrap_or("none") { + "none" => crate::mambazip::OnlineTrainMode::None, + "sgd" => crate::mambazip::OnlineTrainMode::Sgd, + "adam" => crate::mambazip::OnlineTrainMode::Adam, + other => { + return Err(SpecError::new(format!( + "unknown mamba train_mode '{other}'" + ))); + } + }; + Ok(crate::mambazip::OnlineConfig { + hidden: value["hidden"].as_u64().unwrap_or(defaults.hidden as u64) as usize, + layers: value["layers"].as_u64().unwrap_or(defaults.layers as u64) as usize, + intermediate: value["intermediate"] + .as_u64() + .unwrap_or(defaults.intermediate as u64) as usize, + state: value["state"].as_u64().unwrap_or(defaults.state as u64) as usize, + conv: value["conv"].as_u64().unwrap_or(defaults.conv as u64) as usize, + dt_rank: value["dt_rank"].as_u64().unwrap_or(defaults.dt_rank as u64) as usize, + seed: value["seed"].as_u64().unwrap_or(defaults.seed), + train_mode, + lr: value["lr"].as_f64().unwrap_or(defaults.lr as f64) as f32, + stride: value["stride"].as_u64().unwrap_or(defaults.stride as u64) as usize, + }) +} + +#[cfg(feature = "backend-mamba")] +fn mamba_method_to_json_value( + method: &crate::mambazip::MethodSpec, +) -> SpecResult { + Ok(match method { + crate::mambazip::MethodSpec::File { path, policy } => serde_json::json!({ + "kind": "file", + "path": path.to_string_lossy(), + "policy": policy.as_ref().map(crate::backends::llm_policy::LlmPolicy::canonical), + }), + crate::mambazip::MethodSpec::Online { cfg, policy } => serde_json::json!({ + "kind": "online", + "cfg": mamba_online_config_to_json_value(cfg), + "policy": policy.as_ref().map(crate::backends::llm_policy::LlmPolicy::canonical), + }), + }) +} + +#[cfg(feature = "backend-mamba")] +fn parse_mamba_method_json_value( + value: &serde_json::Value, + base_dir: &Path, +) -> SpecResult { + if let Some(method) = value.as_str() { + return normalize_mamba_method_for_base_dir(base_dir, method); + } + match value["kind"].as_str().unwrap_or("file") { + "file" => { + let path = value["path"] + .as_str() + .ok_or_else(|| SpecError::new("mamba method.path is required"))?; + let policy = value["policy"] + .as_str() + .map(|raw| { + crate::backends::llm_policy::parse_policy_segment(raw, MAMBA_POLICY_SCOPES) + }) + .transpose() + .map_err(|err| SpecError::new(err.to_string()))?; + normalize_mamba_method_spec_for_base_dir( + base_dir, + &crate::mambazip::MethodSpec::File { + path: resolve_spec_path(base_dir, path), + policy, + }, + ) + } + "online" => { + let cfg = mamba_online_config_from_json_value(&value["cfg"])?; + let policy = value["policy"] + .as_str() + .map(|raw| { + crate::backends::llm_policy::parse_policy_segment(raw, MAMBA_POLICY_SCOPES) + }) + .transpose() + .map_err(|err| SpecError::new(err.to_string()))?; + normalize_mamba_method_spec_for_base_dir( + base_dir, + &crate::mambazip::MethodSpec::Online { cfg, policy }, + ) + } + other => Err(SpecError::new(format!( + "unknown mamba method kind '{other}'" + ))), + } +} + +/// Parse an RWKV7 compression backend from a method string or configured model path, +/// preserving the shared direct-vs-rate-coded lowering semantics across CLI, JSON, +/// and binding surfaces. +#[cfg(feature = "backend-rwkv")] +fn lower_rwkv7_compression_backend_method( + method: crate::rwkvzip::MethodSpec, + coder: crate::coders::CoderType, + framing: crate::compression::FramingMode, +) -> CompressionBackend { + match method { + crate::rwkvzip::MethodSpec::File { policy: None, .. } => { + CompressionBackend::Rwkv7 { method, coder } + } + crate::rwkvzip::MethodSpec::File { + policy: Some(_), .. + } + | crate::rwkvzip::MethodSpec::Online { .. } => CompressionBackend::Rate { + rate_backend: RateBackend::Rwkv7Method { method }, + coder, + framing, + }, + } +} + +/// Parse RWKV7 compression backend shorthand into a canonical backend configuration. +/// +/// When `method` is empty or absent, this falls back to +/// [`CompressionBackendShorthandOptions::default_rwkv_model_path`]. +/// If the `backend-rwkv` feature is disabled, this returns a spec error. +pub fn parse_rwkv7_compression_backend_method( + method: Option<&str>, + coder: crate::coders::CoderType, + options: &CompressionBackendShorthandOptions, +) -> SpecResult { + #[cfg(feature = "backend-rwkv")] + { + let method = if let Some(method) = method.filter(|value| !value.is_empty()) { + normalize_rwkv_method_for_base_dir(&options.base_dir, method)? + } else { + let model_path = options.default_rwkv_model_path.as_deref().ok_or_else(|| { + SpecError::new( + "rwkv7 compression backend requires a method string or a configured model path", + ) + })?; + normalize_rwkv_path_method(&options.base_dir, model_path)? + }; + Ok(lower_rwkv7_compression_backend_method( + method, + coder, + options.default_framing, + )) + } + #[cfg(not(feature = "backend-rwkv"))] + { + let _ = method; + let _ = coder; + let _ = options; + Err(SpecError::new( + "rwkv7 compression backend disabled at compile time", + )) + } +} + +/// Serialize a `ParticleSpec` into the canonical JSON representation. +pub fn particle_spec_to_json_value(spec: &ParticleSpec) -> serde_json::Value { + serde_json::json!({ + "num_particles": spec.num_particles, + "context_window": spec.context_window, + "unroll_steps": spec.unroll_steps, + "num_cells": spec.num_cells, + "cell_dim": spec.cell_dim, + "num_rules": spec.num_rules, + "selector_hidden": spec.selector_hidden, + "rule_hidden": spec.rule_hidden, + "noise_dim": spec.noise_dim, + "deterministic": spec.deterministic, + "enable_noise": spec.enable_noise, + "noise_scale": spec.noise_scale, + "noise_anneal_steps": spec.noise_anneal_steps, + "learning_rate_readout": spec.learning_rate_readout, + "learning_rate_selector": spec.learning_rate_selector, + "learning_rate_rule": spec.learning_rate_rule, + "bptt_depth": spec.bptt_depth, + "optimizer_momentum": spec.optimizer_momentum, + "grad_clip": spec.grad_clip, + "state_clip": spec.state_clip, + "forget_lambda": spec.forget_lambda, + "resample_threshold": spec.resample_threshold, + "mutate_fraction": spec.mutate_fraction, + "mutate_scale": spec.mutate_scale, + "mutate_model_params": spec.mutate_model_params, + "diagnostics_interval": spec.diagnostics_interval, + "min_prob": spec.min_prob, + "seed": spec.seed, + }) +} + +/// Serialize a `MixtureExpertSpec` into the canonical JSON representation. +pub fn mixture_expert_spec_to_json_value( + spec: &MixtureExpertSpec, +) -> SpecResult { + let mut value = rate_backend_to_json_value(&spec.backend)?; + let object = value + .as_object_mut() + .expect("rate backend serialization must produce a JSON object"); + if let Some(name) = &spec.name { + object.insert("name".to_string(), serde_json::Value::String(name.clone())); + } + object.insert("log_prior".to_string(), serde_json::json!(spec.log_prior)); + Ok(value) +} + +/// Serialize a `MixtureSpec` into the canonical JSON representation. +pub fn mixture_spec_to_json_value(spec: &MixtureSpec) -> SpecResult { + let mut experts = Vec::with_capacity(spec.experts.len()); + for expert in &spec.experts { + experts.push(mixture_expert_spec_to_json_value(expert)?); + } + Ok(serde_json::json!({ + "kind": mixture_kind_name(spec.kind), + "schedule": mixture_schedule_name(spec.schedule), + "alpha": spec.alpha, + "decay": spec.decay, + "experts": experts, + })) +} + +/// Serialize a `CalibratedSpec` into the canonical JSON representation. +pub fn calibrated_spec_to_json_value(spec: &CalibratedSpec) -> SpecResult { + Ok(serde_json::json!({ + "base": rate_backend_to_json_value(&spec.base)?, + "context": calibration_context_kind_name(spec.context), + "bins": spec.bins, + "learning_rate": spec.learning_rate, + "bias_clip": spec.bias_clip, + })) +} + +fn rate_backend_to_json_leaf_value( + canonical: &str, + backend: &RateBackend, +) -> Option> { + match backend { + RateBackend::RosaPlus { max_order } => Some(Ok(serde_json::json!({ + "kind": canonical, + "max_order": max_order, + }))), + RateBackend::Match { + hash_bits, + min_len, + max_len, + base_mix, + confidence_scale, + } => Some(Ok(serde_json::json!({ + "kind": canonical, + "hash_bits": hash_bits, + "min_len": min_len, + "max_len": max_len, + "base_mix": base_mix, + "confidence_scale": confidence_scale, + }))), + RateBackend::SparseMatch { + hash_bits, + min_len, + max_len, + gap_min, + gap_max, + base_mix, + confidence_scale, + } => Some(Ok(serde_json::json!({ + "kind": canonical, + "hash_bits": hash_bits, + "min_len": min_len, + "max_len": max_len, + "gap_min": gap_min, + "gap_max": gap_max, + "base_mix": base_mix, + "confidence_scale": confidence_scale, + }))), + RateBackend::Ppmd { order, memory_mb } => Some(Ok(serde_json::json!({ + "kind": canonical, + "order": order, + "memory_mb": memory_mb, + }))), + RateBackend::Sequitur { context_bytes } => Some(Ok(serde_json::json!({ + "kind": canonical, + "context_bytes": context_bytes, + }))), + RateBackend::Zpaq { method } => Some(Ok(serde_json::json!({ + "kind": canonical, + "method": zpaq_method_to_json_value(method), + }))), + RateBackend::Ctw { depth } => Some(Ok(serde_json::json!({ + "kind": canonical, + "depth": depth, + }))), + RateBackend::FacCtw { + base_depth, + num_percept_bits, + encoding_bits, + msb_first, + } => { + let mut value = serde_json::json!({ + "kind": canonical, + "base_depth": base_depth, + "num_percept_bits": num_percept_bits, + "encoding_bits": encoding_bits, + }); + if let Some(msb_first) = msb_first { + value["msb_first"] = serde_json::Value::Bool(*msb_first); + } + Some(Ok(value)) + } + _ => None, + } +} + +/// Serialize a `RateBackend` into the canonical JSON representation. +pub fn rate_backend_to_json_value(backend: &RateBackend) -> SpecResult { + let canonical = backend.descriptor().map_err(SpecError::new)?.canonical; + if let Some(value) = rate_backend_to_json_leaf_value(canonical, backend) { + return value; + } + match backend { + #[cfg(feature = "backend-mamba")] + RateBackend::MambaMethod { method } => Ok(serde_json::json!({ + "kind": canonical, + "method": mamba_method_to_json_value(method)?, + })), + #[cfg(feature = "backend-rwkv")] + RateBackend::Rwkv7Method { method } => Ok(serde_json::json!({ + "kind": canonical, + "method": rwkv_method_to_json_value(method)?, + })), + RateBackend::Mixture { spec } => Ok(serde_json::json!({ + "kind": canonical, + "spec": mixture_spec_to_json_value(spec.as_ref())?, + })), + RateBackend::Particle { spec } => Ok(serde_json::json!({ + "kind": canonical, + "spec": particle_spec_to_json_value(spec.as_ref()), + })), + RateBackend::Calibrated { spec } => Ok(serde_json::json!({ + "kind": canonical, + "spec": calibrated_spec_to_json_value(spec.as_ref())?, + })), + _ => Err(SpecError::new( + "internal backend serialization mismatch for current feature set", + )), + } +} + +/// Serialize a `CompressionBackend` into the canonical JSON representation. +pub fn compression_backend_to_json_value( + backend: &CompressionBackend, +) -> SpecResult { + let canonical = backend.descriptor().map_err(SpecError::new)?.canonical; + match backend { + CompressionBackend::Zpaq { method, threads } => Ok(serde_json::json!({ + "kind": canonical, + "method": zpaq_method_to_json_value(method), + "threads": threads.get(), + })), + #[cfg(feature = "backend-rwkv")] + CompressionBackend::Rwkv7 { method, coder } => Ok(serde_json::json!({ + "kind": canonical, + "method": rwkv_method_to_json_value(method)?, + "coder": match coder { + crate::coders::CoderType::AC => "ac", + crate::coders::CoderType::RANS => "rans", + }, + })), + CompressionBackend::Rate { + rate_backend, + coder: _, + framing, + } => Ok(serde_json::json!({ + "kind": canonical, + "rate_backend": rate_backend_to_json_value(rate_backend)?, + "framing": framing_mode_name(*framing), + })), + } +} + +/// Parse a `ParticleSpec` from JSON. +pub fn parse_particle_spec_value(v: &serde_json::Value) -> SpecResult { + if v.get("experts").is_some() { + return Err(SpecError::new( + "looks like a mixture spec (found 'experts'); expected ParticleSpec JSON", + )); + } + if let Some(kind) = v.get("kind").and_then(|k| k.as_str()) { + let k = kind.to_ascii_lowercase(); + if matches!( + k.as_str(), + "bayes" + | "fading" + | "fading-bayes" + | "switch" + | "switching" + | "mdl" + | "neural" + | "mixture" + ) { + return Err(SpecError::new(format!( + "looks like a mixture spec (kind='{kind}'); expected ParticleSpec JSON" + ))); + } + } + let d = ParticleSpec::default(); + Ok(ParticleSpec { + num_particles: v["num_particles"] + .as_u64() + .unwrap_or(d.num_particles as u64) as usize, + context_window: v["context_window"] + .as_u64() + .unwrap_or(d.context_window as u64) as usize, + unroll_steps: v["unroll_steps"].as_u64().unwrap_or(d.unroll_steps as u64) as usize, + num_cells: v["num_cells"].as_u64().unwrap_or(d.num_cells as u64) as usize, + cell_dim: v["cell_dim"].as_u64().unwrap_or(d.cell_dim as u64) as usize, + num_rules: v["num_rules"].as_u64().unwrap_or(d.num_rules as u64) as usize, + selector_hidden: v["selector_hidden"] + .as_u64() + .unwrap_or(d.selector_hidden as u64) as usize, + rule_hidden: v["rule_hidden"].as_u64().unwrap_or(d.rule_hidden as u64) as usize, + noise_dim: v["noise_dim"].as_u64().unwrap_or(d.noise_dim as u64) as usize, + deterministic: v["deterministic"].as_bool().unwrap_or(d.deterministic), + enable_noise: v["enable_noise"].as_bool().unwrap_or(d.enable_noise), + noise_scale: v["noise_scale"].as_f64().unwrap_or(d.noise_scale), + noise_anneal_steps: v["noise_anneal_steps"] + .as_u64() + .unwrap_or(d.noise_anneal_steps as u64) as usize, + learning_rate_readout: v["learning_rate_readout"] + .as_f64() + .unwrap_or(d.learning_rate_readout), + learning_rate_selector: v["learning_rate_selector"] + .as_f64() + .unwrap_or(d.learning_rate_selector), + learning_rate_rule: v["learning_rate_rule"] + .as_f64() + .unwrap_or(d.learning_rate_rule), + bptt_depth: v["bptt_depth"].as_u64().unwrap_or(d.bptt_depth as u64) as usize, + optimizer_momentum: v["optimizer_momentum"] + .as_f64() + .unwrap_or(d.optimizer_momentum), + grad_clip: v["grad_clip"].as_f64().unwrap_or(d.grad_clip), + state_clip: v["state_clip"].as_f64().unwrap_or(d.state_clip), + forget_lambda: v["forget_lambda"].as_f64().unwrap_or(d.forget_lambda), + resample_threshold: v["resample_threshold"] + .as_f64() + .unwrap_or(d.resample_threshold), + mutate_fraction: v["mutate_fraction"].as_f64().unwrap_or(d.mutate_fraction), + mutate_scale: v["mutate_scale"].as_f64().unwrap_or(d.mutate_scale), + mutate_model_params: v["mutate_model_params"] + .as_bool() + .unwrap_or(d.mutate_model_params), + diagnostics_interval: v["diagnostics_interval"] + .as_u64() + .unwrap_or(d.diagnostics_interval as u64) as usize, + min_prob: v["min_prob"].as_f64().unwrap_or(d.min_prob), + seed: v["seed"].as_u64().unwrap_or(d.seed), + }) +} + +/// Parse a canonical rate-backend JSON object. +pub fn parse_rate_backend_json( + v: &serde_json::Value, + base_dir: &Path, + depth: usize, +) -> SpecResult { + if depth == 0 { + return Err(SpecError::new("backend spec nesting too deep")); + } + + let raw_kind = v["kind"] + .as_str() + .ok_or_else(|| SpecError::new("backend spec missing 'kind'"))?; + let kind = resolve_enabled_rate_backend_kind(raw_kind)?; + if let Some(backend) = parse_rate_backend_json_leaf(kind, v)? { + return Ok(backend); + } + + match kind { + crate::runtime::RateBackendKind::Mamba => { + #[cfg(feature = "backend-mamba")] + { + if !v["method"].is_null() { + Ok(RateBackend::MambaMethod { + method: parse_mamba_method_json_value(&v["method"], base_dir)?, + }) + } else { + let model_path = v["model_path"].as_str().ok_or_else(|| { + SpecError::new("mamba backend requires 'method' or 'model_path'") + })?; + Ok(RateBackend::MambaMethod { + method: normalize_mamba_path_method(base_dir, model_path)?, + }) + } + } + #[cfg(not(feature = "backend-mamba"))] + { + Err(SpecError::new("mamba backend disabled at compile time")) + } + } + crate::runtime::RateBackendKind::Rwkv7 => { + #[cfg(feature = "backend-rwkv")] + { + if !v["method"].is_null() { + Ok(RateBackend::Rwkv7Method { + method: parse_rwkv_method_json_value(&v["method"], base_dir)?, + }) + } else { + let model_path = v["model_path"].as_str().ok_or_else(|| { + SpecError::new("rwkv7 backend requires 'method' or 'model_path'") + })?; + Ok(RateBackend::Rwkv7Method { + method: normalize_rwkv_path_method(base_dir, model_path)?, + }) + } + } + #[cfg(not(feature = "backend-rwkv"))] + { + Err(SpecError::new("rwkv backend disabled at compile time")) + } + } + crate::runtime::RateBackendKind::Mixture => { + let spec = if let Some(spec_v) = v.get("spec").filter(|value| value.is_object()) { + parse_mixture_spec_value(spec_v, base_dir, depth - 1)? + } else if let Some(path) = v["spec_path"].as_str() { + let full = resolve_spec_path(base_dir, path); + load_mixture_spec_with_depth(full.to_string_lossy().as_ref(), depth - 1)? + } else { + parse_mixture_spec_value(v, base_dir, depth - 1)? + }; + Ok(RateBackend::Mixture { + spec: Arc::new(spec), + }) + } + crate::runtime::RateBackendKind::Particle => { + let spec = if let Some(spec_v) = v.get("spec").filter(|value| value.is_object()) { + parse_particle_spec_value(spec_v)? + } else if let Some(path) = v["spec_path"].as_str() { + let full = resolve_spec_path(base_dir, path); + load_particle_spec(full.to_string_lossy().as_ref())? + } else { + parse_particle_spec_value(v)? + }; + spec.validate() + .map_err(|err| SpecError::new(err.to_string()))?; + Ok(RateBackend::Particle { + spec: Arc::new(spec), + }) + } + crate::runtime::RateBackendKind::Calibrated => { + let spec = if let Some(spec_v) = v.get("spec").filter(|value| value.is_object()) { + parse_calibrated_spec_value(spec_v, base_dir, depth - 1)? + } else if let Some(path) = v["spec_path"].as_str() { + let full = resolve_spec_path(base_dir, path); + load_calibrated_spec(full.to_string_lossy().as_ref())? + } else { + parse_calibrated_spec_value(v, base_dir, depth - 1)? + }; + Ok(RateBackend::Calibrated { + spec: Arc::new(spec), + }) + } + _ => Err(SpecError::new( + "internal backend parse mismatch for current feature set", + )), + } +} + +fn parse_rate_backend_json_leaf( + kind: crate::runtime::RateBackendKind, + v: &serde_json::Value, +) -> SpecResult> { + let backend = match kind { + crate::runtime::RateBackendKind::RosaPlus => RateBackend::RosaPlus { + max_order: v["max_order"] + .as_i64() + .or_else(|| v["order"].as_i64()) + .unwrap_or(-1), + }, + crate::runtime::RateBackendKind::Ctw => RateBackend::Ctw { + depth: v["depth"] + .as_u64() + .unwrap_or(crate::rate_defaults::JSON_DEFAULT_CTW_DEPTH as u64) + as usize, + }, + crate::runtime::RateBackendKind::FacCtw => { + let base_depth = v["base_depth"] + .as_u64() + .unwrap_or(crate::rate_defaults::JSON_DEFAULT_FAC_CTW_BASE_DEPTH as u64) + as usize; + let encoding_bits = v["encoding_bits"] + .as_u64() + .unwrap_or(crate::rate_defaults::JSON_DEFAULT_FAC_CTW_ENCODING_BITS as u64) + as usize; + let num_percept_bits = v["num_percept_bits"] + .as_u64() + .unwrap_or(encoding_bits as u64) as usize; + let msb_first = v.get("msb_first").and_then(serde_json::Value::as_bool); + RateBackend::FacCtw { + base_depth, + num_percept_bits, + encoding_bits, + msb_first, + } + } + crate::runtime::RateBackendKind::Match => RateBackend::Match { + hash_bits: v["hash_bits"] + .as_u64() + .unwrap_or(crate::rate_defaults::JSON_DEFAULT_MATCH_HASH_BITS as u64) + as usize, + min_len: v["min_len"] + .as_u64() + .unwrap_or(crate::rate_defaults::JSON_DEFAULT_MATCH_MIN_LEN as u64) + as usize, + max_len: v["max_len"] + .as_u64() + .unwrap_or(crate::rate_defaults::JSON_DEFAULT_MATCH_MAX_LEN as u64) + as usize, + base_mix: v["base_mix"] + .as_f64() + .unwrap_or(crate::rate_defaults::JSON_DEFAULT_MATCH_BASE_MIX), + confidence_scale: v["confidence_scale"] + .as_f64() + .unwrap_or(crate::rate_defaults::JSON_DEFAULT_MATCH_CONFIDENCE_SCALE), + }, + crate::runtime::RateBackendKind::SparseMatch => RateBackend::SparseMatch { + hash_bits: v["hash_bits"] + .as_u64() + .unwrap_or(crate::rate_defaults::JSON_DEFAULT_SPARSE_MATCH_HASH_BITS as u64) + as usize, + min_len: v["min_len"] + .as_u64() + .unwrap_or(crate::rate_defaults::JSON_DEFAULT_SPARSE_MATCH_MIN_LEN as u64) + as usize, + max_len: v["max_len"] + .as_u64() + .unwrap_or(crate::rate_defaults::JSON_DEFAULT_SPARSE_MATCH_MAX_LEN as u64) + as usize, + gap_min: v["gap_min"] + .as_u64() + .unwrap_or(crate::rate_defaults::JSON_DEFAULT_SPARSE_MATCH_GAP_MIN as u64) + as usize, + gap_max: v["gap_max"] + .as_u64() + .unwrap_or(crate::rate_defaults::JSON_DEFAULT_SPARSE_MATCH_GAP_MAX as u64) + as usize, + base_mix: v["base_mix"] + .as_f64() + .unwrap_or(crate::rate_defaults::JSON_DEFAULT_SPARSE_MATCH_BASE_MIX), + confidence_scale: v["confidence_scale"] + .as_f64() + .unwrap_or(crate::rate_defaults::JSON_DEFAULT_SPARSE_MATCH_CONFIDENCE_SCALE), + }, + crate::runtime::RateBackendKind::Ppmd => RateBackend::Ppmd { + order: v["order"] + .as_u64() + .unwrap_or(crate::rate_defaults::JSON_DEFAULT_PPMD_ORDER as u64) + as usize, + memory_mb: v["memory_mb"] + .as_u64() + .unwrap_or(crate::rate_defaults::JSON_DEFAULT_PPMD_MEMORY_MB as u64) + as usize, + }, + crate::runtime::RateBackendKind::Sequitur => RateBackend::Sequitur { + context_bytes: v["context_bytes"] + .as_u64() + .unwrap_or(crate::rate_defaults::JSON_DEFAULT_SEQUITUR_CONTEXT_BYTES as u64) + as usize, + }, + crate::runtime::RateBackendKind::Zpaq => { + let method = parse_zpaq_method_json_value( + &v["method"], + crate::rate_defaults::JSON_DEFAULT_ZPAQ_RATE_METHOD, + )?; + validate_zpaq_rate_method(method.value()) + .map_err(|err| SpecError::new(err.to_string()))?; + RateBackend::Zpaq { method } + } + _ => return Ok(None), + }; + Ok(Some(backend)) +} + +/// Parse a canonical compression-backend JSON object. +pub fn parse_compression_backend_json( + v: &serde_json::Value, + base_dir: &Path, + default_rate_backend: Option, + default_framing: crate::compression::FramingMode, +) -> SpecResult { + let raw_kind = v["kind"] + .as_str() + .ok_or_else(|| SpecError::new("compression backend spec missing 'kind'"))?; + let kind = resolve_enabled_compression_backend_kind(raw_kind)?; + let framing = v["framing"] + .as_str() + .map(|value| parse_framing_mode(Some(value))) + .transpose()? + .unwrap_or(default_framing); + + match kind { + crate::runtime::CompressionBackendKind::Zpaq => { + let method = parse_zpaq_method_json_value(&v["method"], "5")?; + let threads = if let Some(raw_value) = v.get("threads").filter(|value| !value.is_null()) + { + let raw_u64 = raw_value + .as_u64() + .ok_or_else(|| SpecError::new("zpaq threads must be an integer >= 1"))?; + let raw = usize::try_from(raw_u64) + .map_err(|_| SpecError::new("zpaq threads exceeds usize::MAX"))?; + std::num::NonZeroUsize::new(raw) + .ok_or_else(|| SpecError::new("zpaq threads must be >= 1"))? + } else { + std::num::NonZeroUsize::MIN + }; + crate::zpaq_compress_to_vec(&[], method.value()).map_err(|err| { + SpecError::new(format!( + "invalid zpaq compression method '{}': {err}", + method.value() + )) + })?; + Ok(CompressionBackend::Zpaq { method, threads }) + } + crate::runtime::CompressionBackendKind::RateAc + | crate::runtime::CompressionBackendKind::RateRans => { + let rate_backend = if let Some(rate_backend_v) = v.get("rate_backend") { + parse_rate_backend_json(rate_backend_v, base_dir, MAX_MIXTURE_NESTING)? + } else if let Some(backend_v) = v.get("backend_spec") { + parse_rate_backend_json(backend_v, base_dir, MAX_MIXTURE_NESTING)? + } else { + resolve_default_rate_backend_spec(default_rate_backend.clone())? + }; + let coder = if kind == crate::runtime::CompressionBackendKind::RateAc { + crate::coders::CoderType::AC + } else { + crate::coders::CoderType::RANS + }; + Ok(CompressionBackend::Rate { + rate_backend, + coder, + framing, + }) + } + crate::runtime::CompressionBackendKind::Rwkv7 => { + #[cfg(feature = "backend-rwkv")] + { + let coder = if let Some(value) = v["coder"].as_str() { + crate::backends::parse_rwkv7_coder(value).ok_or_else(|| { + SpecError::new("rwkv7 compression coder must be 'ac' or 'rans'") + })? + } else { + crate::coders::CoderType::AC + }; + let parsed_method = if !v["method"].is_null() { + Some(parse_rwkv_method_json_value(&v["method"], base_dir)?) + } else { + None + }; + let model_path = v["model_path"].as_str(); + if parsed_method.is_none() && model_path.is_none() { + Err(SpecError::new( + "rwkv7 compression backend requires 'method' or 'model_path'", + )) + } else if let Some(method) = parsed_method { + Ok(lower_rwkv7_compression_backend_method( + method, coder, framing, + )) + } else { + let opts = CompressionBackendShorthandOptions { + base_dir: base_dir.to_path_buf(), + default_framing: framing, + default_rwkv_model_path: model_path.map(ToOwned::to_owned), + ..Default::default() + }; + parse_rwkv7_compression_backend_method(None, coder, &opts) + } + } + #[cfg(not(feature = "backend-rwkv"))] + { + Err(SpecError::new( + "rwkv7 compression backend disabled at compile time", + )) + } + } + } +} + +/// Parse a calibrated backend specification. +pub fn parse_calibrated_spec_value( + v: &serde_json::Value, + base_dir: &Path, + depth: usize, +) -> SpecResult { + if depth == 0 { + return Err(SpecError::new("calibrated spec nesting too deep")); + } + + let base_backend = if let Some(base_v) = v.get("base") { + parse_rate_backend_json(base_v, base_dir, depth - 1)? + } else if let Some(path) = v["base_path"].as_str() { + let (value, full) = load_json_value_from_path(base_dir, path, "calibrated base backend")?; + parse_rate_backend_json(&value, full.parent().unwrap_or(base_dir), depth - 1)? + } else { + return Err(SpecError::new( + "calibrated backend requires 'base' or 'base_path'", + )); + }; + + Ok(CalibratedSpec { + base: base_backend, + context: parse_calibration_context_kind(v["context"].as_str())?, + bins: v["bins"].as_u64().unwrap_or(33) as usize, + learning_rate: v["learning_rate"].as_f64().unwrap_or(0.02), + bias_clip: v["bias_clip"].as_f64().unwrap_or(4.0), + }) +} + +/// Parse one mixture expert entry. +pub fn parse_mixture_expert_value( + v: &serde_json::Value, + base_dir: &Path, + depth: usize, +) -> SpecResult { + if depth == 0 { + return Err(SpecError::new("mixture spec nesting too deep")); + } + + let backend = parse_rate_backend_json(v, base_dir, depth - 1)?; + + Ok(MixtureExpertSpec { + name: v["name"].as_str().map(|s| s.to_string()), + log_prior: v["log_prior"] + .as_f64() + .or_else(|| v["prior"].as_f64()) + .unwrap_or(0.0), + backend, + }) +} + +/// Parse a `MixtureSpec` from JSON. +pub fn parse_mixture_spec_value( + v: &serde_json::Value, + base_dir: &Path, + depth: usize, +) -> SpecResult { + if depth == 0 { + return Err(SpecError::new("mixture spec nesting too deep")); + } + + let kind_str = v["kind"] + .as_str() + .or_else(|| v["mixture_kind"].as_str()) + .or_else(|| v["mix_kind"].as_str()) + .unwrap_or("bayes"); + let kind = parse_mixture_kind(kind_str)?; + let schedule = v["schedule"] + .as_str() + .or_else(|| v["schedule_mode"].as_str()) + .or_else(|| v["mixture_schedule"].as_str()) + .map(parse_mixture_schedule) + .transpose()? + .unwrap_or(MixtureScheduleMode::Default); + + let experts_v = v["experts"] + .as_array() + .ok_or_else(|| SpecError::new("mixture spec missing 'experts' array"))?; + if experts_v.is_empty() { + return Err(SpecError::new( + "mixture spec must include at least one expert", + )); + } + + let mut experts = Vec::with_capacity(experts_v.len()); + for expert in experts_v { + experts.push(parse_mixture_expert_value(expert, base_dir, depth - 1)?); + } + + let mut spec = MixtureSpec::new(kind, experts) + .with_schedule(schedule) + .with_alpha(v["alpha"].as_f64().unwrap_or(0.01)); + if let Some(decay) = v["decay"].as_f64() { + spec = spec.with_decay(decay); + } + spec.validate() + .map_err(|err| SpecError::new(err.to_string()))?; + Ok(spec) +} + +fn read_mixture_value_with_zpaq_fallback(path: &Path) -> SpecResult { + let raw = std::fs::read(path).map_err(|e| { + SpecError::new(format!( + "failed to read mixture spec '{}': {e}", + path.display() + )) + })?; + match serde_json::from_slice(&raw) { + Ok(value) => Ok(value), + Err(json_err) => { + #[cfg(feature = "backend-zpaq")] + { + let decompressed = zpaq_rs::decompress_to_vec(&raw).map_err(|_| { + SpecError::new(format!( + "failed to parse mixture JSON '{}': {json_err}", + path.display() + )) + })?; + serde_json::from_slice(&decompressed).map_err(|e| { + SpecError::new(format!("invalid mixture JSON '{}': {e}", path.display())) + }) + } + #[cfg(not(feature = "backend-zpaq"))] + { + Err(SpecError::new(format!( + "failed to parse mixture JSON '{}', and zpaq support is disabled at compile time: {json_err}", + path.display() + ))) + } + } + } +} + +/// Load a mixture spec from disk. +pub fn load_mixture_spec(path: &str) -> SpecResult { + load_mixture_spec_with_depth(path, MAX_MIXTURE_NESTING) +} + +/// Load a mixture spec from disk with an explicit remaining nesting budget. +pub fn load_mixture_spec_with_depth(path: &str, depth: usize) -> SpecResult { + let full = Path::new(path); + let value = read_mixture_value_with_zpaq_fallback(full)?; + let base_dir = full.parent().unwrap_or_else(|| Path::new(".")); + parse_mixture_spec_value(&value, base_dir, depth) +} + +/// Load a particle spec from disk. +pub fn load_particle_spec(path: &str) -> SpecResult { + let full = Path::new(path); + let raw = std::fs::read(full).map_err(|e| { + SpecError::new(format!( + "failed to read particle spec '{}': {e}", + full.display() + )) + })?; + let value: serde_json::Value = serde_json::from_slice(&raw).map_err(|e| { + SpecError::new(format!( + "invalid particle spec JSON '{}': {e}", + full.display() + )) + })?; + let spec = parse_particle_spec_value(&value)?; + spec.validate() + .map_err(|err| SpecError::new(err.to_string()))?; + Ok(spec) +} + +/// Load a calibrated spec from disk. +pub fn load_calibrated_spec(path: &str) -> SpecResult { + let full = Path::new(path); + let raw = std::fs::read(full).map_err(|e| { + SpecError::new(format!( + "failed to read calibrated spec '{}': {e}", + full.display() + )) + })?; + let value: serde_json::Value = serde_json::from_slice(&raw).map_err(|e| { + SpecError::new(format!( + "invalid calibrated spec JSON '{}': {e}", + full.display() + )) + })?; + let base_dir = full.parent().unwrap_or_else(|| Path::new(".")); + parse_calibrated_spec_value(&value, base_dir, 4) +} + +/// Load a single expert spec from disk. +pub fn load_expert_spec(path: &str) -> SpecResult { + let full = Path::new(path); + let raw = std::fs::read(full).map_err(|e| { + SpecError::new(format!( + "failed to read expert spec '{}': {e}", + full.display() + )) + })?; + let value: serde_json::Value = serde_json::from_slice(&raw).map_err(|e| { + SpecError::new(format!( + "invalid expert spec JSON '{}': {e}", + full.display() + )) + })?; + let base_dir = full.parent().unwrap_or_else(|| Path::new(".")); + parse_mixture_expert_value(&value, base_dir, MAX_MIXTURE_NESTING) +} + +/// Build a backend from shorthand CLI/Python-style `name` + optional `method` inputs. +pub fn parse_rate_backend_name_method( + name: &str, + method: Option<&str>, + options: &RateBackendShorthandOptions, +) -> SpecResult { + let kind = resolve_enabled_rate_backend_kind(name)?; + let method = method.filter(|value| !value.is_empty()); + if let Some(backend) = parse_rate_backend_name_method_leaf(kind, method, options)? { + return Ok(backend); + } + + match kind { + crate::runtime::RateBackendKind::Mamba => { + #[cfg(feature = "backend-mamba")] + { + if let Some(method) = method { + Ok(RateBackend::MambaMethod { + method: normalize_mamba_method_for_base_dir(&options.base_dir, method)?, + }) + } else if let Some(path) = options.default_mamba_model_path.as_deref() { + Ok(RateBackend::MambaMethod { + method: normalize_mamba_path_method(&options.base_dir, path)?, + }) + } else { + Err(SpecError::new( + "mamba backend requires method string (cfg:...;policy:... or file:...) or a configured model path", + )) + } + } + #[cfg(not(feature = "backend-mamba"))] + { + Err(SpecError::new("mamba backend disabled at compile time")) + } + } + crate::runtime::RateBackendKind::Rwkv7 => { + #[cfg(feature = "backend-rwkv")] + { + if let Some(method) = method { + Ok(RateBackend::Rwkv7Method { + method: normalize_rwkv_method_for_base_dir(&options.base_dir, method)?, + }) + } else if let Some(path) = options.default_rwkv_model_path.as_deref() { + Ok(RateBackend::Rwkv7Method { + method: normalize_rwkv_path_method(&options.base_dir, path)?, + }) + } else { + Err(SpecError::new( + "rwkv backend requires method string (cfg:...;policy:... or file:...) or a configured model path", + )) + } + } + #[cfg(not(feature = "backend-rwkv"))] + { + Err(SpecError::new("rwkv backend disabled at compile time")) + } + } + crate::runtime::RateBackendKind::Mixture => { + let path = method.ok_or_else(|| { + SpecError::new("mixture backend requires a path to a MixtureSpec JSON file") + })?; + let full = resolve_spec_path(&options.base_dir, path); + Ok(RateBackend::Mixture { + spec: Arc::new(load_mixture_spec(full.to_string_lossy().as_ref())?), + }) + } + crate::runtime::RateBackendKind::Particle => { + let spec = if let Some(path) = method { + let full = resolve_spec_path(&options.base_dir, path); + load_particle_spec(full.to_string_lossy().as_ref())? + } else if options.particle_default_if_missing_method { + ParticleSpec::default() + } else { + return Err(SpecError::new( + "particle backend requires a path to a ParticleSpec JSON file", + )); + }; + spec.validate() + .map_err(|err| SpecError::new(err.to_string()))?; + Ok(RateBackend::Particle { + spec: Arc::new(spec), + }) + } + crate::runtime::RateBackendKind::Calibrated => { + let path = method.ok_or_else(|| { + SpecError::new("calibrated backend requires a path to a CalibratedSpec JSON file") + })?; + let full = resolve_spec_path(&options.base_dir, path); + Ok(RateBackend::Calibrated { + spec: Arc::new(load_calibrated_spec(full.to_string_lossy().as_ref())?), + }) + } + _ => Err(SpecError::new( + "internal backend shorthand parse mismatch for current feature set", + )), + } +} + +fn parse_rate_backend_name_method_leaf( + kind: crate::runtime::RateBackendKind, + method: Option<&str>, + options: &RateBackendShorthandOptions, +) -> SpecResult> { + let backend = match kind { + crate::runtime::RateBackendKind::RosaPlus => RateBackend::RosaPlus { + max_order: method + .and_then(|value| value.parse::().ok()) + .unwrap_or(-1), + }, + crate::runtime::RateBackendKind::Match => RateBackend::Match { + hash_bits: crate::rate_defaults::JSON_DEFAULT_MATCH_HASH_BITS, + min_len: crate::rate_defaults::JSON_DEFAULT_MATCH_MIN_LEN, + max_len: crate::rate_defaults::JSON_DEFAULT_MATCH_MAX_LEN, + base_mix: crate::rate_defaults::JSON_DEFAULT_MATCH_BASE_MIX, + confidence_scale: crate::rate_defaults::JSON_DEFAULT_MATCH_CONFIDENCE_SCALE, + }, + crate::runtime::RateBackendKind::SparseMatch => RateBackend::SparseMatch { + hash_bits: crate::rate_defaults::JSON_DEFAULT_SPARSE_MATCH_HASH_BITS, + min_len: crate::rate_defaults::JSON_DEFAULT_SPARSE_MATCH_MIN_LEN, + max_len: crate::rate_defaults::JSON_DEFAULT_SPARSE_MATCH_MAX_LEN, + gap_min: crate::rate_defaults::JSON_DEFAULT_SPARSE_MATCH_GAP_MIN, + gap_max: crate::rate_defaults::JSON_DEFAULT_SPARSE_MATCH_GAP_MAX, + base_mix: crate::rate_defaults::JSON_DEFAULT_SPARSE_MATCH_BASE_MIX, + confidence_scale: crate::rate_defaults::JSON_DEFAULT_SPARSE_MATCH_CONFIDENCE_SCALE, + }, + crate::runtime::RateBackendKind::Ppmd => RateBackend::Ppmd { + order: method + .and_then(|value| value.parse::().ok()) + .unwrap_or(options.ppmd_order), + memory_mb: options.ppmd_memory_mb, + }, + crate::runtime::RateBackendKind::Sequitur => RateBackend::Sequitur { + context_bytes: method + .and_then(|value| value.parse::().ok()) + .unwrap_or(options.sequitur_context_bytes), + }, + crate::runtime::RateBackendKind::Ctw => RateBackend::Ctw { + depth: method + .and_then(|value| value.parse::().ok()) + .unwrap_or(options.ctw_depth), + }, + crate::runtime::RateBackendKind::FacCtw => { + let base_depth = method + .and_then(|value| value.parse::().ok()) + .unwrap_or(options.fac_ctw_base_depth); + RateBackend::FacCtw { + base_depth, + num_percept_bits: options.fac_ctw_num_percept_bits, + encoding_bits: options.fac_ctw_encoding_bits, + msb_first: options.fac_ctw_msb_first, + } + } + crate::runtime::RateBackendKind::Zpaq => { + let method = method + .map(ToOwned::to_owned) + .unwrap_or_else(|| options.zpaq_method.clone()); + validate_zpaq_rate_method(&method).map_err(|err| SpecError::new(err.to_string()))?; + RateBackend::Zpaq { + method: crate::api::ZpaqMethodSpec::literal(method), + } + } + _ => return Ok(None), + }; + Ok(Some(backend)) +} + +/// Parse and compile a shorthand CLI/Python-style rate backend. +pub fn compile_rate_backend_name_method( + name: &str, + method: Option<&str>, + options: &RateBackendShorthandOptions, +) -> SpecResult { + let env = SpecEnvironment::new(options.base_dir.clone()); + parse_rate_backend_name_method(name, method, options)?.compile_in(&env) +} + +/// Build a compression backend from shorthand CLI/Python-style +/// `name` + optional `method` inputs. +pub fn parse_compression_backend_name_method( + name: &str, + method: Option<&str>, + rate_backend: Option, + options: &CompressionBackendShorthandOptions, +) -> SpecResult { + let kind = resolve_enabled_compression_backend_kind(name)?; + let method = method.filter(|value| !value.is_empty()); + + match kind { + crate::runtime::CompressionBackendKind::Zpaq => { + let method = method + .map(ToOwned::to_owned) + .unwrap_or_else(|| options.zpaq_method.clone()); + crate::zpaq_compress_to_vec(&[], &method).map_err(|err| { + SpecError::new(format!("invalid zpaq compression method '{method}': {err}")) + })?; + Ok(CompressionBackend::zpaq(method)) + } + crate::runtime::CompressionBackendKind::RateAc => Ok(CompressionBackend::Rate { + rate_backend: rate_backend + .or_else(|| options.default_rate_backend.clone()) + .map(Ok) + .unwrap_or_else(|| resolve_default_rate_backend_spec(None))?, + coder: crate::coders::CoderType::AC, + framing: options.default_framing, + }), + crate::runtime::CompressionBackendKind::RateRans => Ok(CompressionBackend::Rate { + rate_backend: rate_backend + .or_else(|| options.default_rate_backend.clone()) + .map(Ok) + .unwrap_or_else(|| resolve_default_rate_backend_spec(None))?, + coder: crate::coders::CoderType::RANS, + framing: options.default_framing, + }), + crate::runtime::CompressionBackendKind::Rwkv7 => { + #[cfg(feature = "backend-rwkv")] + { + match method { + Some(m) if crate::backends::parse_rwkv7_coder(m).is_some() => { + let model_path = options.default_rwkv_model_path.as_deref().ok_or_else(|| { + SpecError::new( + "rwkv7 compression backend requires a configured model path when only a coder alias is provided", + ) + })?; + Ok(CompressionBackend::Rwkv7 { + method: normalize_rwkv_path_method(&options.base_dir, model_path)?, + coder: crate::backends::parse_rwkv7_coder(m) + .expect("coder alias already validated"), + }) + } + Some(m) => parse_rwkv7_compression_backend_method( + Some(m), + crate::coders::CoderType::AC, + options, + ), + None => parse_rwkv7_compression_backend_method( + None, + crate::coders::CoderType::AC, + options, + ), + } + } + #[cfg(not(feature = "backend-rwkv"))] + { + Err(SpecError::new( + "rwkv7 compression backend disabled at compile time", + )) + } + } + } +} + +/// Parse and compile a shorthand CLI/Python-style compression backend. +pub fn compile_compression_backend_name_method( + name: &str, + method: Option<&str>, + rate_backend: Option, + options: &CompressionBackendShorthandOptions, +) -> SpecResult { + let env = SpecEnvironment::new(options.base_dir.clone()); + parse_compression_backend_name_method(name, method, rate_backend, options)?.compile_in(&env) +} + +#[cfg(test)] +mod tests { + #[allow(unused_imports)] + use super::*; + use crate::api::{ + CalibratedSpec, CalibrationContextKind, CompressionBackend, MAX_MIXTURE_NESTING, + MixtureExpertSpec, MixtureKind, MixtureSpec, RateBackend, + }; + #[cfg(any( + feature = "all-backends", + feature = "backend-rwkv", + feature = "backend-mamba" + ))] + use crate::coders::CoderType; + use std::fs; + use std::path::Path; + use std::sync::Arc; + use std::time::{SystemTime, UNIX_EPOCH}; + + fn unique_temp_dir(label: &str) -> std::path::PathBuf { + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("system clock must be after unix epoch") + .as_nanos(); + std::env::temp_dir().join(format!( + "infotheory-spec-tests-{label}-{}-{nonce}", + std::process::id() + )) + } + + fn sample_self_contained_rate_backend( + kind: crate::runtime::RateBackendKind, + ) -> Option { + crate::runtime::default_rate_backend_spec(kind) + } + + #[cfg(feature = "aixi")] + #[test] + fn canonical_json_bytes_sort_object_keys_recursively_and_preserve_array_order() { + let value = serde_json::json!({ + "b": 1, + "a": { + "z": [2, 1], + "a": false + }, + "c": [ + { + "b": 2, + "a": 1 + }, + null + ] + }); + let bytes = canonical_json_bytes(&value).expect("canonical JSON bytes"); + assert_eq!( + std::str::from_utf8(&bytes).expect("canonical JSON is UTF-8"), + r#"{"a":{"a":false,"z":[2,1]},"b":1,"c":[{"a":1,"b":2},null]}"# + ); + } + + #[cfg(feature = "backend-rosa")] + #[test] + fn shorthand_rate_aliases_compile_to_identical_canonical_bytes() { + let opts = RateBackendShorthandOptions::default(); + let rosa = + compile_rate_backend_name_method("rosa", None, &opts).expect("compile rosa alias"); + let rosaplus = compile_rate_backend_name_method("rosaplus", None, &opts) + .expect("compile rosaplus canonical"); + assert_eq!( + rosa.canonical_bytes().as_slice(), + rosaplus.canonical_bytes().as_slice() + ); + assert_eq!( + rosa.canonical_spec().to_canonical_json().unwrap(), + rosaplus.canonical_spec().to_canonical_json().unwrap() + ); + } + + #[test] + fn shorthand_compression_requires_canonical_names() { + let Some(default_rate_backend) = sample_enabled_leaf_rate_backend() else { + return; + }; + let opts = CompressionBackendShorthandOptions { + default_rate_backend: Some(default_rate_backend), + ..CompressionBackendShorthandOptions::default() + }; + let canonical = compile_compression_backend_name_method("rate-ac", None, None, &opts) + .expect("compile rate-ac canonical"); + let canonical_kind = canonical.canonical_spec().kind(); + assert_eq!( + canonical_kind, + crate::runtime::CompressionBackendKind::RateAc + ); + + let err = match compile_compression_backend_name_method("rate_ac", None, None, &opts) { + Ok(_) => panic!("legacy alias must be rejected"), + Err(err) => err, + }; + let msg = err.to_string(); + assert!( + msg.contains("unknown compression backend") || msg.contains("not available"), + "unexpected error: {msg}" + ); + } + + fn sample_enabled_leaf_rate_backend() -> Option { + crate::runtime::RATE_BACKEND_REGISTRY + .iter() + .filter(|descriptor| descriptor.enabled) + .find_map(|descriptor| sample_self_contained_rate_backend(descriptor.kind)) + } + + fn sample_roundtrip_rate_backends() -> Vec { + let mut backends = Vec::new(); + let leaf = sample_enabled_leaf_rate_backend(); + + for descriptor in crate::runtime::RATE_BACKEND_REGISTRY { + if !descriptor.enabled { + continue; + } + + match descriptor.kind { + crate::runtime::RateBackendKind::Mixture => { + if let Some(base) = leaf.clone() { + backends.push(RateBackend::Mixture { + spec: Arc::new(MixtureSpec::new( + MixtureKind::Bayes, + vec![MixtureExpertSpec { + name: Some("leaf".to_string()), + log_prior: 0.0, + backend: base, + }], + )), + }); + } + } + crate::runtime::RateBackendKind::Calibrated => { + if let Some(base) = leaf.clone() { + backends.push(RateBackend::Calibrated { + spec: Arc::new(CalibratedSpec { + base, + context: CalibrationContextKind::Text, + bins: 17, + learning_rate: 0.05, + bias_clip: 3.0, + }), + }); + } + } + _ => { + if let Some(backend) = sample_self_contained_rate_backend(descriptor.kind) { + backends.push(backend); + } + } + } + } + + backends + } + + fn sample_roundtrip_compression_backends() -> Vec { + let mut backends = Vec::new(); + let leaf = sample_enabled_leaf_rate_backend(); + + if cfg!(feature = "backend-zpaq") { + backends.push(CompressionBackend::zpaq("5")); + } + + if let Some(rate_backend) = leaf.clone() { + backends.push(CompressionBackend::Rate { + rate_backend: rate_backend.clone(), + coder: crate::coders::CoderType::AC, + framing: crate::compression::FramingMode::Raw, + }); + backends.push(CompressionBackend::Rate { + rate_backend, + coder: crate::coders::CoderType::RANS, + framing: crate::compression::FramingMode::Framed, + }); + } + + #[cfg(feature = "backend-rwkv")] + { + let opts = CompressionBackendShorthandOptions { + default_framing: crate::compression::FramingMode::Raw, + ..Default::default() + }; + let backend = parse_compression_backend_name_method( + "rwkv7", + Some( + "cfg:hidden=64,intermediate=64,layers=1,train=sgd,lr=0.01;policy:schedule=0..100:infer", + ), + None, + &opts, + ) + .expect("rwkv shorthand should parse"); + backends.push(backend); + } + + backends + } + + #[test] + fn rate_backend_aliases_share_registry_resolution_and_feature_errors() { + for descriptor in crate::runtime::RATE_BACKEND_REGISTRY { + for alias in descriptor.aliases { + if descriptor.enabled { + assert_eq!( + resolve_enabled_rate_backend_name(alias) + .expect("enabled alias should resolve"), + descriptor.canonical + ); + } else { + let err = resolve_enabled_rate_backend_name(alias) + .expect_err("disabled alias should fail"); + let message = err.to_string(); + assert!(message.contains(descriptor.canonical)); + assert!( + message.contains(descriptor.feature.expect("disabled feature metadata")) + ); + } + } + } + } + + #[test] + fn compression_backend_aliases_share_registry_resolution_and_feature_errors() { + for descriptor in crate::runtime::COMPRESSION_BACKEND_REGISTRY { + for alias in descriptor.aliases { + if descriptor.enabled { + assert_eq!( + resolve_enabled_compression_backend_name(alias) + .expect("enabled alias should resolve"), + descriptor.canonical + ); + } else { + let err = resolve_enabled_compression_backend_name(alias) + .expect_err("disabled alias should fail"); + let message = err.to_string(); + assert!(message.contains(descriptor.canonical)); + assert!( + message.contains(descriptor.feature.expect("disabled feature metadata")) + ); + } + } + } + } + + #[test] + fn enabled_self_contained_rate_backends_validate_and_roundtrip() { + for backend in sample_roundtrip_rate_backends() { + crate::api::validate_rate_backend(&backend).expect("sample backend should validate"); + let json = rate_backend_to_json_value(&backend).expect("serialize backend"); + let reparsed = parse_rate_backend_json(&json, Path::new("."), MAX_MIXTURE_NESTING) + .expect("parse backend"); + let roundtrip = rate_backend_to_json_value(&reparsed).expect("re-serialize backend"); + assert_eq!(json, roundtrip); + } + } + + #[test] + fn enabled_compression_parse_paths_validate_and_roundtrip() { + for backend in sample_roundtrip_compression_backends() { + crate::api::validate_compression_backend(&backend) + .expect("sample compression backend should validate"); + let json = + compression_backend_to_json_value(&backend).expect("serialize compression backend"); + let reparsed = parse_compression_backend_json( + &json, + Path::new("."), + None, + crate::compression::FramingMode::Framed, + ) + .expect("parse compression backend"); + let roundtrip = compression_backend_to_json_value(&reparsed) + .expect("re-serialize compression backend"); + assert_eq!(json, roundtrip); + } + } + + #[cfg(feature = "all-backends")] + #[test] + fn rate_backend_json_roundtrip_handles_nested_specs() { + let backend = RateBackend::Calibrated { + spec: Arc::new(CalibratedSpec { + base: RateBackend::Mixture { + spec: Arc::new( + MixtureSpec::new( + MixtureKind::Switching, + vec![ + MixtureExpertSpec { + name: Some("rosa".to_string()), + log_prior: -0.2, + backend: RateBackend::RosaPlus { max_order: 8 }, + }, + MixtureExpertSpec { + name: Some("ctw".to_string()), + log_prior: -1.4, + backend: RateBackend::Ctw { depth: 12 }, + }, + MixtureExpertSpec { + name: Some("particle".to_string()), + log_prior: -2.0, + backend: RateBackend::Particle { + spec: Arc::new(ParticleSpec { + num_particles: 4, + ..ParticleSpec::default() + }), + }, + }, + ], + ) + .with_schedule(MixtureScheduleMode::Theorem) + .with_alpha(0.25), + ), + }, + context: CalibrationContextKind::TextRepeat, + bins: 31, + learning_rate: 0.05, + bias_clip: 3.0, + }), + }; + + let json = rate_backend_to_json_value(&backend).expect("serialize backend"); + let reparsed = parse_rate_backend_json(&json, Path::new("."), MAX_MIXTURE_NESTING) + .expect("parse backend"); + let roundtrip = rate_backend_to_json_value(&reparsed).expect("re-serialize backend"); + assert_eq!(json, roundtrip); + } + + #[cfg(feature = "all-backends")] + #[test] + fn compression_backend_json_roundtrip_handles_rate_wrappers() { + let backend = CompressionBackend::Rate { + rate_backend: RateBackend::Match { + hash_bits: 18, + min_len: 5, + max_len: 128, + base_mix: 0.03, + confidence_scale: 0.8, + }, + coder: CoderType::RANS, + framing: crate::compression::FramingMode::Framed, + }; + + let json = + compression_backend_to_json_value(&backend).expect("serialize compression backend"); + let reparsed = parse_compression_backend_json( + &json, + Path::new("."), + None, + crate::compression::FramingMode::Framed, + ) + .expect("parse compression backend"); + let roundtrip = + compression_backend_to_json_value(&reparsed).expect("re-serialize compression backend"); + assert_eq!(json, roundtrip); + } + + #[cfg(feature = "all-backends")] + #[test] + fn parse_compression_backend_name_method_uses_shared_shorthand_defaults() { + let rate_backend = RateBackend::Ppmd { + order: 7, + memory_mb: 32, + }; + let opts = CompressionBackendShorthandOptions { + default_rate_backend: Some(rate_backend.clone()), + default_framing: crate::compression::FramingMode::Framed, + ..Default::default() + }; + + let ac = parse_compression_backend_name_method("rate-ac", None, None, &opts) + .expect("parse rate-ac"); + let rans = parse_compression_backend_name_method("rate-rans", None, None, &opts) + .expect("parse rate-rans"); + let zpaq = + parse_compression_backend_name_method("zpaq", None, None, &opts).expect("parse zpaq"); + + match ac { + CompressionBackend::Rate { + rate_backend: RateBackend::Ppmd { order, memory_mb }, + coder, + framing, + } => { + assert_eq!(order, 7); + assert_eq!(memory_mb, 32); + assert_eq!(coder, CoderType::AC); + assert_eq!(framing, crate::compression::FramingMode::Framed); + } + _ => panic!("unexpected rate-ac backend"), + } + + match rans { + CompressionBackend::Rate { coder, .. } => { + assert_eq!(coder, CoderType::RANS); + } + _ => panic!("unexpected rate-rans backend"), + } + + match zpaq { + CompressionBackend::Zpaq { method, .. } => assert_eq!(method.value(), "5"), + _ => panic!("unexpected zpaq backend"), + } + } + + #[cfg(feature = "backend-rwkv")] + #[test] + fn parse_compression_backend_name_method_wraps_rwkv_cfg_methods_as_rate_backend() { + let opts = CompressionBackendShorthandOptions { + default_framing: crate::compression::FramingMode::Raw, + ..Default::default() + }; + + let backend = parse_compression_backend_name_method( + "rwkv7", + Some( + "cfg:hidden=64,intermediate=64,layers=1,train=sgd,lr=0.01;policy:schedule=0..100:infer", + ), + None, + &opts, + ) + .expect("parse rwkv cfg compression backend"); + + match backend { + CompressionBackend::Rate { + rate_backend: RateBackend::Rwkv7Method { .. }, + coder, + framing, + } => { + assert_eq!(coder, CoderType::AC); + assert_eq!(framing, crate::compression::FramingMode::Raw); + } + _ => panic!("expected rate-coded RWKV backend"), + } + } + + #[cfg(feature = "backend-rwkv")] + #[test] + fn parse_compression_backend_json_wraps_rwkv_cfg_methods_as_rate_backend() { + let json = serde_json::json!({ + "kind": "rwkv7", + "method": "cfg:hidden=64,intermediate=64,layers=1,train=sgd,lr=0.01;policy:schedule=0..100:infer", + "coder": "rans", + "framing": "raw" + }); + + let backend = parse_compression_backend_json( + &json, + Path::new("."), + None, + crate::compression::FramingMode::Framed, + ) + .expect("parse rwkv cfg compression backend json"); + + match backend { + CompressionBackend::Rate { + rate_backend: RateBackend::Rwkv7Method { .. }, + coder, + framing, + } => { + assert_eq!(coder, CoderType::RANS); + assert_eq!(framing, crate::compression::FramingMode::Raw); + } + _ => panic!("expected rate-coded RWKV backend"), + } + } + + #[cfg(feature = "backend-rwkv")] + #[test] + fn parse_compression_backend_json_wraps_typed_rwkv_methods_as_rate_backend() { + let json = serde_json::json!({ + "kind": "rwkv7", + "method": { + "kind": "online", + "cfg": { + "hidden": 64, + "intermediate": 64, + "layers": 1, + "train_mode": "sgd", + "lr": 0.01 + }, + "policy": "schedule=0..100:infer" + }, + "coder": "ac", + "framing": "framed" + }); + + let backend = parse_compression_backend_json( + &json, + Path::new("."), + None, + crate::compression::FramingMode::Raw, + ) + .expect("parse typed rwkv compression backend json"); + + match backend { + CompressionBackend::Rate { + rate_backend: RateBackend::Rwkv7Method { .. }, + coder, + framing, + } => { + assert_eq!(coder, CoderType::AC); + assert_eq!(framing, crate::compression::FramingMode::Framed); + } + _ => panic!("expected rate-coded RWKV backend"), + } + } + + #[cfg(feature = "backend-rwkv")] + #[test] + fn normalize_rwkv_method_for_base_dir_decodes_reserved_file_escapes_once() { + let base_dir = Path::new("/tmp/spec-base"); + let method = "file:weights/model%3Bv1%25done.safetensors"; + let normalized = canonicalize_explicit_file_method(base_dir, method, "rwkv") + .expect("canonicalize rwkv method") + .expect("file method"); + assert_eq!( + normalized, + "file:/tmp/spec-base/weights/model%3Bv1%25done.safetensors" + ); + } + + #[cfg(all(feature = "backend-rwkv", windows))] + #[test] + fn normalize_rwkv_method_for_base_dir_renders_forward_slashes_on_windows() { + let base_dir = Path::new(r"C:\tmp\spec-base"); + let method = "file:weights/model%3Bv1%25done.safetensors"; + let normalized = canonicalize_explicit_file_method(base_dir, method, "rwkv") + .expect("canonicalize rwkv method") + .expect("file method"); + assert_eq!( + normalized, + "file:C:/tmp/spec-base/weights/model%3Bv1%25done.safetensors" + ); + } + + #[cfg(feature = "backend-rwkv")] + #[test] + fn normalize_rwkv_method_for_base_dir_rejects_ambiguous_file_suffixes() { + let err = normalize_rwkv_method_for_base_dir( + Path::new("/tmp/spec-base"), + "file:weights/model;polciy:infer", + ) + .unwrap_err(); + assert!( + err.message + .contains("ambiguous file method segment ';polciy:'") + ); + } + + #[cfg(feature = "backend-mamba")] + #[test] + fn parse_compression_backend_json_wraps_mamba_cfg_rate_backend() { + let json = serde_json::json!({ + "kind": "rate-ac", + "framing": "raw", + "rate_backend": { + "kind": "mamba", + "method": "cfg:hidden=64,layers=1,intermediate=96,state=16,conv=4,dt_rank=16,seed=26,train=none,lr=0.0,stride=1;policy:schedule=0..100:infer" + } + }); + + let backend = parse_compression_backend_json( + &json, + Path::new("."), + None, + crate::compression::FramingMode::Framed, + ) + .expect("parse mamba rate-coded compression backend json"); + + match backend { + CompressionBackend::Rate { + rate_backend: RateBackend::MambaMethod { .. }, + coder, + framing, + } => { + assert_eq!(coder, CoderType::AC); + assert_eq!(framing, crate::compression::FramingMode::Raw); + } + _ => panic!("expected rate-coded mamba backend"), + } + } + + #[cfg(feature = "backend-zpaq")] + #[test] + fn canonical_json_emits_typed_zpaq_method_objects() { + let rate = serde_json::from_str::( + &RateBackend::Zpaq { + method: crate::api::ZpaqMethodSpec::literal("5"), + } + .to_canonical_json() + .expect("rate json"), + ) + .expect("valid rate json"); + assert_eq!(rate["kind"], "zpaq"); + assert_eq!(rate["method"]["kind"], "literal"); + assert_eq!(rate["method"]["value"], "5"); + + let compression = serde_json::from_str::( + &CompressionBackend::zpaq("5") + .to_canonical_json() + .expect("compression json"), + ) + .expect("valid compression json"); + assert_eq!(compression["kind"], "zpaq"); + assert_eq!(compression["method"]["kind"], "literal"); + assert_eq!(compression["method"]["value"], "5"); + } + + #[cfg(feature = "backend-zpaq")] + #[test] + fn parse_typed_zpaq_method_objects_are_strictly_validated() { + let err = parse_rate_backend_json( + &serde_json::json!({ + "kind": "zpaq", + "method": {"kind": "literal"} + }), + Path::new("."), + MAX_MIXTURE_NESTING, + ) + .err() + .expect("missing typed zpaq value must fail"); + assert!(err.message.contains("method.value"), "{err}"); + + let err = parse_rate_backend_json( + &serde_json::json!({ + "kind": "zpaq", + "method": {"value": "2"} + }), + Path::new("."), + MAX_MIXTURE_NESTING, + ) + .err() + .expect("missing typed zpaq kind must fail"); + assert!(err.message.contains("method.kind"), "{err}"); + + let err = parse_compression_backend_json( + &serde_json::json!({ + "kind": "zpaq", + "method": {"kind": "nonliteral", "value": "5"} + }), + Path::new("."), + None, + crate::compression::FramingMode::Framed, + ) + .err() + .expect("unknown typed zpaq kind must fail"); + assert!(err.message.contains("unknown zpaq method kind"), "{err}"); + } + + #[cfg(feature = "backend-zpaq")] + #[test] + fn parse_zpaq_method_requires_typed_object_form() { + let err = parse_rate_backend_json( + &serde_json::json!({"kind": "zpaq", "method": "2"}), + Path::new("."), + MAX_MIXTURE_NESTING, + ) + .err() + .expect("legacy zpaq method string must be rejected"); + assert!( + err.message + .contains("zpaq method must use object form {'kind':'literal','value':'...'}"), + "{err}" + ); + + let compression = parse_compression_backend_json( + &serde_json::json!({"kind": "zpaq"}), + Path::new("."), + None, + crate::compression::FramingMode::Framed, + ) + .expect("missing method should use default compression method"); + assert!( + matches!(compression, CompressionBackend::Zpaq { method, .. } if method.value() == "5") + ); + } + + #[cfg(feature = "backend-rwkv")] + #[test] + fn canonical_json_emits_typed_rwkv_method_objects() { + let value = serde_json::from_str::( + &RateBackend::Rwkv7Method { + method: crate::rwkvzip::parse_method_spec("cfg:hidden=64,intermediate=64,layers=1") + .expect("rwkv method spec"), + } + .to_canonical_json() + .expect("rate json"), + ) + .expect("valid rate json"); + + assert_eq!(value["kind"], "rwkv7"); + assert_eq!(value["method"]["kind"], "online"); + assert_eq!(value["method"]["cfg"]["hidden"], 64); + assert_eq!(value["method"]["cfg"]["layers"], 1); + assert_eq!(value["method"]["cfg"]["intermediate"], 64); + } + + #[cfg(feature = "backend-mamba")] + #[test] + fn canonical_json_emits_typed_mamba_method_objects() { + let value = serde_json::from_str::( + &RateBackend::MambaMethod { + method: crate::mambazip::parse_method_spec( + "cfg:hidden=64,layers=1,intermediate=96,state=16,conv=4,dt_rank=16", + ) + .expect("mamba method spec"), + } + .to_canonical_json() + .expect("rate json"), + ) + .expect("valid rate json"); + + assert_eq!(value["kind"], "mamba"); + assert_eq!(value["method"]["kind"], "online"); + assert_eq!(value["method"]["cfg"]["hidden"], 64); + assert_eq!(value["method"]["cfg"]["layers"], 1); + assert_eq!(value["method"]["cfg"]["intermediate"], 96); + } + + #[test] + fn helper_parsers_and_name_renderers_use_canonical_forms() { + assert_eq!( + resolve_spec_path(Path::new("/tmp/base"), "child/spec.json"), + Path::new("/tmp/base").join("child/spec.json") + ); + assert_eq!( + resolve_spec_path(Path::new("/tmp/base"), Path::new("/tmp/absolute.json")), + Path::new("/tmp/absolute.json") + ); + + assert_eq!( + parse_calibration_context_kind(None).expect("default calibration context"), + CalibrationContextKind::Text + ); + assert_eq!( + parse_calibration_context_kind(Some("repeat")).expect("repeat context"), + CalibrationContextKind::Repeat + ); + assert!(parse_calibration_context_kind(Some("legacy")).is_err()); + + assert_eq!( + parse_mixture_kind("switching").expect("switching"), + MixtureKind::Switching + ); + assert_eq!( + parse_mixture_schedule("theorem").expect("theorem schedule"), + MixtureScheduleMode::Theorem + ); + assert_eq!(mixture_kind_name(MixtureKind::Neural), "neural"); + assert_eq!( + mixture_schedule_name(MixtureScheduleMode::Default), + "default" + ); + assert_eq!( + calibration_context_kind_name(CalibrationContextKind::Text), + "text" + ); + + assert_eq!( + parse_framing_mode(None).expect("default framing"), + crate::compression::FramingMode::Framed + ); + assert_eq!( + parse_framing_mode(Some("raw")).expect("raw framing"), + crate::compression::FramingMode::Raw + ); + assert!(parse_framing_mode(Some("legacy")).is_err()); + assert_eq!( + framing_mode_name(crate::compression::FramingMode::Framed), + "framed" + ); + + let zpaq_json = zpaq_method_to_json_value(&crate::api::ZpaqMethodSpec::literal("3")); + assert_eq!(zpaq_json["kind"], "literal"); + assert_eq!( + parse_zpaq_method_json_value(&zpaq_json, "5") + .expect("typed zpaq method") + .value(), + "3" + ); + } + + #[test] + fn load_json_value_from_path_reports_read_and_parse_context() { + let dir = unique_temp_dir("load-json"); + fs::create_dir_all(&dir).expect("create temp dir"); + let valid = dir.join("valid.json"); + let invalid = dir.join("invalid.json"); + fs::write(&valid, br#"{ "alpha": 1 }"#).expect("write valid json"); + fs::write(&invalid, b"{ invalid").expect("write invalid json"); + + let (value, full) = + load_json_value_from_path(&dir, "valid.json", "spec fixture").expect("load valid"); + assert_eq!(value["alpha"], 1); + assert_eq!(full, valid); + + let err = load_json_value_from_path(&dir, "missing.json", "spec fixture") + .expect_err("missing json must fail"); + assert!(err.to_string().contains("failed to read spec fixture")); + assert!(err.to_string().contains("missing.json")); + + let err = load_json_value_from_path(&dir, "invalid.json", "spec fixture") + .expect_err("invalid json must fail"); + assert!(err.to_string().contains("invalid spec fixture JSON")); + assert!(err.to_string().contains("invalid.json")); + + let _ = fs::remove_file(valid); + let _ = fs::remove_file(invalid); + let _ = fs::remove_dir(dir); + } + + #[cfg(feature = "backend-ctw")] + #[test] + fn shorthand_rate_backend_parsers_load_file_backed_specs_and_respect_particle_default_policy() { + let dir = unique_temp_dir("backend-files"); + fs::create_dir_all(&dir).expect("create temp dir"); + + let mixture_path = dir.join("mixture.json"); + let calibrated_path = dir.join("calibrated.json"); + let particle_path = dir.join("particle.json"); + + let mixture = MixtureSpec::new( + MixtureKind::Bayes, + vec![MixtureExpertSpec { + name: Some("ctw".to_string()), + log_prior: 0.0, + backend: RateBackend::Ctw { depth: 4 }, + }], + ); + let calibrated = CalibratedSpec { + base: RateBackend::Ctw { depth: 5 }, + context: CalibrationContextKind::Text, + bins: 17, + learning_rate: 0.05, + bias_clip: 3.0, + }; + let particle = ParticleSpec::default(); + + fs::write( + &mixture_path, + mixture.to_canonical_json().expect("mixture canonical json"), + ) + .expect("write mixture spec"); + fs::write( + &calibrated_path, + calibrated + .to_canonical_json() + .expect("calibrated canonical json"), + ) + .expect("write calibrated spec"); + fs::write( + &particle_path, + particle + .to_canonical_json() + .expect("particle canonical json"), + ) + .expect("write particle spec"); + + let options = RateBackendShorthandOptions { + base_dir: dir.clone(), + particle_default_if_missing_method: false, + ..RateBackendShorthandOptions::default() + }; + + let mixture_result = + parse_rate_backend_name_method("mixture", Some("mixture.json"), &options); + #[cfg(feature = "backend-mixture")] + { + let mixture_backend = mixture_result.expect("mixture shorthand should load JSON file"); + assert!(matches!(mixture_backend, RateBackend::Mixture { .. })); + } + #[cfg(not(feature = "backend-mixture"))] + { + let err = match mixture_result { + Ok(_) => { + panic!("mixture shorthand should fail when feature is disabled") + } + Err(err) => err, + }; + assert!( + err.to_string() + .contains("backend 'mixture' requires infotheory feature 'backend-mixture'"), + "unexpected mixture error: {err}" + ); + } + + let particle_result = + parse_rate_backend_name_method("particle", Some("particle.json"), &options); + #[cfg(feature = "backend-particle")] + { + let particle_backend = + particle_result.expect("particle shorthand should load JSON file"); + assert!(matches!(particle_backend, RateBackend::Particle { .. })); + } + #[cfg(not(feature = "backend-particle"))] + { + let err = match particle_result { + Ok(_) => { + panic!("particle shorthand should fail when feature is disabled") + } + Err(err) => err, + }; + assert!( + err.to_string() + .contains("backend 'particle' requires infotheory feature 'backend-particle'"), + "unexpected particle error: {err}" + ); + } + + let calibrated_result = + parse_rate_backend_name_method("calibrated", Some("calibrated.json"), &options); + #[cfg(feature = "backend-calibrated")] + { + let calibrated_backend = + calibrated_result.expect("calibrated shorthand should load JSON file"); + assert!(matches!(calibrated_backend, RateBackend::Calibrated { .. })); + } + #[cfg(not(feature = "backend-calibrated"))] + { + let err = match calibrated_result { + Ok(_) => { + panic!("calibrated shorthand should fail when feature is disabled") + } + Err(err) => err, + }; + assert!( + err.to_string().contains( + "backend 'calibrated' requires infotheory feature 'backend-calibrated'" + ), + "unexpected calibrated error: {err}" + ); + } + + let particle_missing_method_result = + parse_rate_backend_name_method("particle", None, &options); + #[cfg(feature = "backend-particle")] + { + let err = match particle_missing_method_result { + Ok(_) => { + panic!("particle shorthand should require path when disabled") + } + Err(err) => err, + }; + assert!( + err.to_string() + .contains("particle backend requires a path to a ParticleSpec JSON file"), + "unexpected particle missing-method error: {err}" + ); + } + #[cfg(not(feature = "backend-particle"))] + { + let err = match particle_missing_method_result { + Ok(_) => { + panic!("particle shorthand without feature should report capability error") + } + Err(err) => err, + }; + assert!( + err.to_string() + .contains("backend 'particle' requires infotheory feature 'backend-particle'"), + "unexpected particle feature error: {err}" + ); + } + + let _ = fs::remove_file(mixture_path); + let _ = fs::remove_file(calibrated_path); + let _ = fs::remove_file(particle_path); + let _ = fs::remove_dir(dir); + } + + #[test] + fn parse_particle_spec_value_preserves_defaults_and_rejects_mixture_shapes() { + let defaults = ParticleSpec::default(); + let parsed = parse_particle_spec_value(&serde_json::json!({ + "num_particles": defaults.num_particles + 7, + "deterministic": !defaults.deterministic, + })) + .expect("particle subset should parse with defaults"); + assert_eq!(parsed.num_particles, defaults.num_particles + 7); + assert_eq!(parsed.context_window, defaults.context_window); + assert_eq!(parsed.seed, defaults.seed); + assert_eq!(parsed.deterministic, !defaults.deterministic); + + let err = parse_particle_spec_value(&serde_json::json!({ + "kind": "mixture", + "num_particles": 8, + })) + .expect_err("mixture-looking kind must be rejected"); + assert!( + err.to_string() + .contains("looks like a mixture spec (kind='mixture')"), + "unexpected error: {err}" + ); + + let err = parse_particle_spec_value(&serde_json::json!({ + "experts": [], + })) + .expect_err("mixture-shaped object must be rejected"); + assert!( + err.to_string() + .contains("looks like a mixture spec (found 'experts')"), + "unexpected error: {err}" + ); + } + + #[test] + fn fac_ctw_default_projections_remain_explicit_and_consistent() { + let shorthand = RateBackendShorthandOptions::default(); + assert_eq!( + shorthand.fac_ctw_num_percept_bits, + crate::rate_defaults::FAC_CTW_DEFAULT_NUM_PERCEPT_BITS + ); + assert_eq!( + shorthand.fac_ctw_encoding_bits, + crate::rate_defaults::JSON_DEFAULT_FAC_CTW_ENCODING_BITS + ); + + #[cfg(feature = "backend-ctw")] + { + let parsed = parse_rate_backend_json( + &serde_json::json!({"kind":"fac-ctw"}), + Path::new("."), + MAX_MIXTURE_NESTING, + ) + .expect("fac-ctw json parse"); + let parsed_for_compile = parsed.clone(); + match parsed { + RateBackend::FacCtw { + base_depth, + num_percept_bits, + encoding_bits, + msb_first, + } => { + assert_eq!( + base_depth, + crate::rate_defaults::JSON_DEFAULT_FAC_CTW_BASE_DEPTH + ); + assert_eq!( + encoding_bits, + crate::rate_defaults::JSON_DEFAULT_FAC_CTW_ENCODING_BITS + ); + assert_eq!(num_percept_bits, encoding_bits); + assert_eq!(msb_first, None); + } + _ => panic!("expected fac-ctw backend"), + } + + let explicit_lsb = parse_rate_backend_json( + &serde_json::json!({ + "kind": "fac-ctw", + "base_depth": 9, + "encoding_bits": 8, + "num_percept_bits": 8, + "msb_first": false, + }), + Path::new("."), + MAX_MIXTURE_NESTING, + ) + .expect("fac-ctw explicit LSB json parse"); + let compiled_lsb = explicit_lsb + .compile() + .expect("fac-ctw explicit LSB compiles"); + match compiled_lsb.plan() { + crate::spec::core::RateBackendPlan::FacCtw { msb_first, .. } => { + assert!(!*msb_first, "explicit msb_first=false must survive compile"); + } + _ => panic!("expected fac-ctw compiled plan"), + } + + let compiled_default = parsed_for_compile + .compile() + .expect("fac-ctw default compiles"); + match compiled_default.plan() { + crate::spec::core::RateBackendPlan::FacCtw { + encoding_bits, + msb_first, + .. + } => { + assert_eq!(*encoding_bits, 8); + assert!( + *msb_first, + "omitted msb_first defaults to MSB-first for byte-width FacCtw" + ); + } + _ => panic!("expected fac-ctw compiled plan"), + } + + let invalid_width = parse_rate_backend_json( + &serde_json::json!({ + "kind": "fac-ctw", + "base_depth": 9, + "encoding_bits": 9, + "num_percept_bits": 9, + }), + Path::new("."), + MAX_MIXTURE_NESTING, + ) + .expect("fac-ctw invalid-width json parses before semantic compile validation"); + let err = match invalid_width.compile() { + Ok(_) => panic!("fac-ctw encoding_bits outside 1..=8 must be rejected"), + Err(err) => err, + }; + assert!( + err.to_string() + .contains("fac-ctw encoding_bits must be in 1..=8, got 9"), + "unexpected fac-ctw encoding_bits error: {err}" + ); + } + + #[cfg(not(feature = "backend-ctw"))] + { + let err = match parse_rate_backend_json( + &serde_json::json!({"kind":"fac-ctw"}), + Path::new("."), + MAX_MIXTURE_NESTING, + ) { + Ok(_) => panic!("disabled fac-ctw backend must report a feature error"), + Err(err) => err, + }; + assert!( + err.to_string() + .contains("backend 'fac-ctw' requires infotheory feature 'backend-ctw'"), + "unexpected fac-ctw feature error: {err}" + ); + } + + let runtime_default = + crate::runtime::default_rate_backend_spec(crate::runtime::RateBackendKind::FacCtw) + .expect("runtime fac-ctw default"); + match runtime_default { + RateBackend::FacCtw { + base_depth, + num_percept_bits, + encoding_bits, + msb_first, + } => { + assert_eq!(base_depth, 8); + assert_eq!( + encoding_bits, + crate::rate_defaults::JSON_DEFAULT_FAC_CTW_ENCODING_BITS + ); + assert_eq!( + num_percept_bits, + crate::rate_defaults::FAC_CTW_DEFAULT_NUM_PERCEPT_BITS + ); + assert_eq!(msb_first, None); + } + _ => panic!("expected runtime fac-ctw default backend"), + } + let fac_ctw_default_json = rate_backend_to_json_value(&runtime_default) + .expect("serialize runtime fac-ctw default backend"); + assert!( + fac_ctw_default_json.get("msb_first").is_none(), + "canonical fac-ctw json must omit msb_first when unset" + ); + + #[cfg(feature = "backend-ctw")] + { + let msb_shorthand = RateBackendShorthandOptions { + fac_ctw_msb_first: Some(true), + ..RateBackendShorthandOptions::default() + }; + let parsed_msb = parse_rate_backend_name_method("fac-ctw", Some("9"), &msb_shorthand) + .expect("fac-ctw shorthand with msb_first"); + match parsed_msb { + RateBackend::FacCtw { msb_first, .. } => { + assert_eq!(msb_first, Some(true)); + } + _ => panic!("expected fac-ctw backend"), + } + let compiled_msb = parsed_msb.compile().expect("fac-ctw msb compiles"); + match compiled_msb.plan() { + crate::spec::core::RateBackendPlan::FacCtw { msb_first, .. } => { + assert!(*msb_first, "shorthand msb_first=true must survive compile"); + } + _ => panic!("expected fac-ctw compiled plan"), + } + + let lsb_shorthand = RateBackendShorthandOptions { + fac_ctw_msb_first: Some(false), + ..RateBackendShorthandOptions::default() + }; + let parsed_lsb = parse_rate_backend_name_method("fac-ctw", Some("9"), &lsb_shorthand) + .expect("fac-ctw shorthand with lsb_first"); + match parsed_lsb.compile().expect("fac-ctw lsb compiles").plan() { + crate::spec::core::RateBackendPlan::FacCtw { msb_first, .. } => { + assert!( + !*msb_first, + "shorthand msb_first=false must survive compile" + ); + } + _ => panic!("expected fac-ctw compiled plan"), + } + + let factory_json = crate::rate_defaults::fac_ctw_spec_json(9, 8, 8, Some(false)); + assert_eq!(factory_json["msb_first"], serde_json::json!(false)); + let factory_default = crate::rate_defaults::fac_ctw_spec_json(9, 8, 8, None); + assert!( + factory_default.get("msb_first").is_none(), + "factory omits msb_first when None" + ); + } + } + + #[cfg(feature = "all-backends")] + #[test] + fn load_sidecar_specs_report_read_and_json_error_context() { + let dir = unique_temp_dir("load-sidecar-errors"); + fs::create_dir_all(&dir).expect("create temp dir"); + + let particle_invalid = dir.join("particle-invalid.json"); + let calibrated_invalid = dir.join("calibrated-invalid.json"); + let expert_invalid = dir.join("expert-invalid.json"); + fs::write(&particle_invalid, b"{ invalid").expect("write invalid particle json"); + fs::write(&calibrated_invalid, b"{ invalid").expect("write invalid calibrated json"); + fs::write(&expert_invalid, b"{ invalid").expect("write invalid expert json"); + + let missing_particle = dir.join("particle-missing.json"); + let err = match load_particle_spec(missing_particle.to_str().expect("utf8 path")) { + Ok(_) => panic!("missing particle spec should fail"), + Err(err) => err, + }; + assert!( + err.to_string().contains("failed to read particle spec"), + "{err}" + ); + + let err = match load_particle_spec(particle_invalid.to_str().expect("utf8 path")) { + Ok(_) => panic!("invalid particle spec JSON should fail"), + Err(err) => err, + }; + assert!( + err.to_string().contains("invalid particle spec JSON"), + "{err}" + ); + + let missing_calibrated = dir.join("calibrated-missing.json"); + let err = match load_calibrated_spec(missing_calibrated.to_str().expect("utf8 path")) { + Ok(_) => panic!("missing calibrated spec should fail"), + Err(err) => err, + }; + assert!( + err.to_string().contains("failed to read calibrated spec"), + "{err}" + ); + + let err = match load_calibrated_spec(calibrated_invalid.to_str().expect("utf8 path")) { + Ok(_) => panic!("invalid calibrated spec JSON should fail"), + Err(err) => err, + }; + assert!( + err.to_string().contains("invalid calibrated spec JSON"), + "{err}" + ); + + let missing_expert = dir.join("expert-missing.json"); + let err = match load_expert_spec(missing_expert.to_str().expect("utf8 path")) { + Ok(_) => panic!("missing expert spec should fail"), + Err(err) => err, + }; + assert!( + err.to_string().contains("failed to read expert spec"), + "{err}" + ); + + let err = match load_expert_spec(expert_invalid.to_str().expect("utf8 path")) { + Ok(_) => panic!("invalid expert spec JSON should fail"), + Err(err) => err, + }; + assert!( + err.to_string().contains("invalid expert spec JSON"), + "{err}" + ); + + let _ = fs::remove_file(particle_invalid); + let _ = fs::remove_file(calibrated_invalid); + let _ = fs::remove_file(expert_invalid); + let _ = fs::remove_dir(dir); + } + + #[cfg(feature = "all-backends")] + #[test] + fn shorthand_rate_backend_parser_covers_leaf_defaults_and_model_path_contracts() { + let options = RateBackendShorthandOptions::default(); + + let sparse_match = parse_rate_backend_name_method("sparse-match", None, &options) + .expect("sparse-match shorthand should parse"); + match sparse_match { + RateBackend::SparseMatch { + hash_bits, + min_len, + max_len, + gap_min, + gap_max, + base_mix, + confidence_scale, + } => { + assert_eq!(hash_bits, 19); + assert_eq!(min_len, 3); + assert_eq!(max_len, 64); + assert_eq!(gap_min, 1); + assert_eq!(gap_max, 2); + assert!((base_mix - 0.05).abs() < f64::EPSILON); + assert!((confidence_scale - 1.0).abs() < f64::EPSILON); + } + _ => panic!("expected sparse-match backend"), + } + + let fac_ctw = parse_rate_backend_name_method("fac-ctw", Some("11"), &options) + .expect("fac-ctw shorthand should parse"); + match fac_ctw { + RateBackend::FacCtw { + base_depth, + num_percept_bits, + encoding_bits, + msb_first, + } => { + assert_eq!(base_depth, 11); + assert_eq!(num_percept_bits, options.fac_ctw_num_percept_bits); + assert_eq!(encoding_bits, options.fac_ctw_encoding_bits); + assert_eq!(msb_first, None); + } + _ => panic!("expected fac-ctw backend"), + } + + let zpaq = parse_rate_backend_name_method("zpaq", None, &options) + .expect("zpaq shorthand should parse"); + assert!( + matches!(zpaq, RateBackend::Zpaq { method } if method.value() == options.zpaq_method) + ); + + let particle = parse_rate_backend_name_method("particle", None, &options) + .expect("particle shorthand should use default spec"); + assert!(matches!(particle, RateBackend::Particle { .. })); + + let mixture_err = match parse_rate_backend_name_method("mixture", None, &options) { + Ok(_) => panic!("mixture shorthand without path should fail"), + Err(err) => err, + }; + assert!( + mixture_err + .to_string() + .contains("mixture backend requires a path to a MixtureSpec JSON file"), + "{mixture_err}" + ); + + let calibrated_err = match parse_rate_backend_name_method("calibrated", None, &options) { + Ok(_) => panic!("calibrated shorthand without path should fail"), + Err(err) => err, + }; + assert!( + calibrated_err + .to_string() + .contains("calibrated backend requires a path to a CalibratedSpec JSON file"), + "{calibrated_err}" + ); + + let mamba_err = match parse_rate_backend_name_method("mamba", None, &options) { + Ok(_) => panic!("mamba shorthand without method/model path should fail"), + Err(err) => err, + }; + assert!( + mamba_err + .to_string() + .contains("mamba backend requires method string"), + "{mamba_err}" + ); + + let rwkv_err = match parse_rate_backend_name_method("rwkv7", None, &options) { + Ok(_) => panic!("rwkv shorthand without method/model path should fail"), + Err(err) => err, + }; + assert!( + rwkv_err + .to_string() + .contains("rwkv backend requires method string"), + "{rwkv_err}" + ); + } + + #[cfg(feature = "backend-ctw")] + #[test] + fn parse_rate_backend_json_loads_nested_spec_paths_relative_to_base_dir() { + let dir = unique_temp_dir("nested-backend-specs"); + fs::create_dir_all(&dir).expect("create temp dir"); + + let nested_dir = dir.join("nested"); + fs::create_dir_all(&nested_dir).expect("create nested dir"); + + let particle_path = nested_dir.join("particle.json"); + let calibrated_path = nested_dir.join("calibrated.json"); + let mixture_path = nested_dir.join("mixture.json"); + + let particle = ParticleSpec { + num_particles: 11, + context_window: 19, + ..ParticleSpec::default() + }; + let calibrated = CalibratedSpec { + base: RateBackend::Ctw { depth: 9 }, + context: CalibrationContextKind::Repeat, + bins: 21, + learning_rate: 0.03, + bias_clip: 2.5, + }; + let mixture = MixtureSpec::new( + MixtureKind::Bayes, + vec![MixtureExpertSpec { + name: Some("ctw-nine".to_string()), + log_prior: -0.5, + backend: RateBackend::Ctw { depth: 9 }, + }], + ); + + fs::write( + &particle_path, + particle.to_canonical_json().expect("particle json"), + ) + .expect("write particle spec"); + fs::write( + &calibrated_path, + calibrated.to_canonical_json().expect("calibrated json"), + ) + .expect("write calibrated spec"); + fs::write( + &mixture_path, + mixture.to_canonical_json().expect("mixture json"), + ) + .expect("write mixture spec"); + + let particle_result = parse_rate_backend_json( + &serde_json::json!({ + "kind": "particle", + "spec_path": "nested/particle.json", + }), + &dir, + MAX_MIXTURE_NESTING, + ); + #[cfg(feature = "backend-particle")] + { + let particle_backend = + particle_result.expect("particle spec_path should resolve relative to base dir"); + match particle_backend { + RateBackend::Particle { spec } => { + assert_eq!(spec.num_particles, 11); + assert_eq!(spec.context_window, 19); + } + _ => panic!("expected particle backend"), + } + } + #[cfg(not(feature = "backend-particle"))] + { + let err = match particle_result { + Ok(_) => panic!("particle backend should report feature gate in this slice"), + Err(err) => err, + }; + assert!( + err.to_string() + .contains("backend 'particle' requires infotheory feature 'backend-particle'"), + "unexpected particle error: {err}" + ); + } + + let calibrated_result = parse_rate_backend_json( + &serde_json::json!({ + "kind": "calibrated", + "spec_path": "nested/calibrated.json", + }), + &dir, + MAX_MIXTURE_NESTING, + ); + #[cfg(feature = "backend-calibrated")] + { + let calibrated_backend = calibrated_result + .expect("calibrated spec_path should resolve relative to base dir"); + match calibrated_backend { + RateBackend::Calibrated { spec } => { + assert_eq!(spec.bins, 21); + assert_eq!(spec.context, CalibrationContextKind::Repeat); + match spec.base { + RateBackend::Ctw { depth } => assert_eq!(depth, 9), + _ => panic!("expected ctw base backend"), + } + } + _ => panic!("expected calibrated backend"), + } + } + #[cfg(not(feature = "backend-calibrated"))] + { + let err = match calibrated_result { + Ok(_) => panic!("calibrated backend should report feature gate in this slice"), + Err(err) => err, + }; + assert!( + err.to_string().contains( + "backend 'calibrated' requires infotheory feature 'backend-calibrated'" + ), + "unexpected calibrated error: {err}" + ); + } + + let mixture_result = parse_rate_backend_json( + &serde_json::json!({ + "kind": "mixture", + "spec_path": "nested/mixture.json", + }), + &dir, + MAX_MIXTURE_NESTING, + ); + #[cfg(feature = "backend-mixture")] + { + let mixture_backend = + mixture_result.expect("mixture spec_path should resolve relative to base dir"); + match mixture_backend { + RateBackend::Mixture { spec } => { + assert_eq!(spec.kind, MixtureKind::Bayes); + assert_eq!(spec.experts.len(), 1); + assert_eq!(spec.experts[0].name.as_deref(), Some("ctw-nine")); + } + _ => panic!("expected mixture backend"), + } + } + #[cfg(not(feature = "backend-mixture"))] + { + let err = match mixture_result { + Ok(_) => panic!("mixture backend should report feature gate in this slice"), + Err(err) => err, + }; + assert!( + err.to_string() + .contains("backend 'mixture' requires infotheory feature 'backend-mixture'"), + "unexpected mixture error: {err}" + ); + } + + let _ = fs::remove_file(particle_path); + let _ = fs::remove_file(calibrated_path); + let _ = fs::remove_file(mixture_path); + let _ = fs::remove_dir(nested_dir); + let _ = fs::remove_dir(dir); + } + + #[cfg(feature = "backend-rwkv")] + #[test] + fn parse_rwkv7_compression_backend_json_requires_method_or_model_path() { + let err = match parse_compression_backend_json( + &serde_json::json!({ + "kind": "rwkv7", + }), + Path::new("."), + None, + crate::compression::FramingMode::Framed, + ) { + Ok(_) => panic!("rwkv7 compression json without method/model_path must fail"), + Err(err) => err, + }; + assert!( + err.to_string() + .contains("rwkv7 compression backend requires 'method' or 'model_path'"), + "unexpected error: {err}" + ); + } + + #[cfg(feature = "backend-rwkv")] + #[test] + fn parse_rwkv7_compression_backend_json_lowers_typed_method_and_coder() { + let backend = parse_compression_backend_json( + &serde_json::json!({ + "kind": "rwkv7", + "coder": "rans", + "framing": "raw", + "method": { + "kind": "online", + "cfg": { + "hidden": 64, + "layers": 1, + "intermediate": 64, + "decay_rank": 8, + "a_rank": 8, + "v_rank": 8, + "g_rank": 8, + "seed": 5, + "train": "none", + "lr": 0.01, + "stride": 2 + }, + "policy": "schedule=0..100:infer" + } + }), + Path::new("."), + None, + crate::compression::FramingMode::Framed, + ) + .expect("typed rwkv7 compression backend should parse"); + + match backend { + CompressionBackend::Rate { + rate_backend, + coder, + framing, + } => { + assert_eq!(coder, crate::coders::CoderType::RANS); + assert_eq!(framing, crate::compression::FramingMode::Raw); + match rate_backend { + RateBackend::Rwkv7Method { method } => match method { + crate::rwkvzip::MethodSpec::Online { cfg, policy } => { + assert_eq!(cfg.hidden, 64); + assert_eq!(cfg.layers, 1); + assert_eq!(cfg.stride, 2); + assert!(policy.is_some(), "policy should be preserved"); + } + _ => panic!("expected online rwkv method"), + }, + _ => panic!("expected rwkv7 rate backend"), + } + } + CompressionBackend::Rwkv7 { method, coder } => { + assert_eq!(coder, crate::coders::CoderType::RANS); + match method { + crate::rwkvzip::MethodSpec::Online { cfg, policy } => { + assert_eq!(cfg.hidden, 64); + assert_eq!(cfg.layers, 1); + assert_eq!(cfg.stride, 2); + assert!(policy.is_some(), "policy should be preserved"); + } + _ => panic!("expected online rwkv method"), + } + } + _ => panic!("expected rwkv7-derived compression backend"), + } + } + + #[cfg(feature = "backend-rwkv")] + #[test] + fn parse_rwkv7_compression_shorthand_coder_alias_requires_or_uses_model_path() { + let base_dir = unique_temp_dir("rwkv-coder-alias"); + fs::create_dir_all(&base_dir).expect("create temp dir"); + + let missing_path_options = CompressionBackendShorthandOptions { + base_dir: base_dir.clone(), + ..CompressionBackendShorthandOptions::default() + }; + let err = match parse_compression_backend_name_method( + "rwkv7", + Some("ac"), + None, + &missing_path_options, + ) { + Ok(_) => panic!("coder alias without default model path should fail"), + Err(err) => err, + }; + assert!( + err.to_string().contains( + "rwkv7 compression backend requires a configured model path when only a coder alias is provided" + ), + "{err}" + ); + + let with_path_options = CompressionBackendShorthandOptions { + base_dir: base_dir.clone(), + default_rwkv_model_path: Some("weights/model.safetensors".to_string()), + ..CompressionBackendShorthandOptions::default() + }; + let with_path_err = match parse_compression_backend_name_method( + "rwkv7", + Some("rans"), + None, + &with_path_options, + ) { + Ok(_) => panic!("missing RWKV model weights should fail deterministically"), + Err(err) => err, + }; + assert!( + with_path_err + .to_string() + .contains("Failed to load model weights"), + "{with_path_err}" + ); + assert!( + with_path_err + .to_string() + .contains("weights/model.safetensors"), + "{with_path_err}" + ); + + let _ = fs::remove_dir(base_dir); + } +} diff --git a/crates/infotheory/src/spec/core.rs b/crates/infotheory/src/spec/core.rs new file mode 100644 index 00000000..8b719f0e --- /dev/null +++ b/crates/infotheory/src/spec/core.rs @@ -0,0 +1,1058 @@ +//! Canonical validated and compiled backend plans. +//! +//! This module is the internal/public bridge between wrapper AST specs +//! (`RateBackend`, `CompressionBackend`) and the compiled runtime plans used by +//! the generic Rust API. + +use crate::api::{ + CalibratedSpec, CalibrationContextKind, CompressionBackend, MAX_MIXTURE_NESTING, + MixtureExpertSpec, MixtureKind, MixtureScheduleMode, MixtureSpec, ParticleSpec, RateBackend, +}; +use crate::coders::CoderType; +use crate::compression::FramingMode; +use crate::spec::{SpecError, SpecResult}; +use std::fmt; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +mod adapters; +mod compile; +pub(crate) use adapters::*; +pub(crate) use compile::*; + +/// Compilation environment for backend/spec validation and canonicalization. +#[derive(Clone, Debug, Default, Eq, PartialEq)] +#[non_exhaustive] +pub struct SpecEnvironment { + base_dir: PathBuf, +} + +impl SpecEnvironment { + /// Create an environment rooted at `base_dir`. + pub fn new(base_dir: impl Into) -> Self { + Self { + base_dir: base_dir.into(), + } + } + + /// Base directory used to resolve relative asset references. + pub fn base_dir(&self) -> &Path { + &self.base_dir + } +} + +/// Typed external asset reference captured by a validated/compiled spec. +#[derive(Clone, Debug, Eq, PartialEq, Hash)] +#[non_exhaustive] +pub enum AssetRef { + /// Filesystem-backed asset resolved relative to a [`SpecEnvironment`]. + Filesystem(PathBuf), +} + +/// Deterministic binary canonical code for a validated spec. +#[derive(Clone, Eq, PartialEq, Hash)] +pub struct CanonicalBytes(Arc<[u8]>); + +impl CanonicalBytes { + /// View the canonical bytes. + pub fn as_slice(&self) -> &[u8] { + &self.0 + } + + /// Length of the canonical code in bytes. + pub fn len(&self) -> usize { + self.0.len() + } + + /// Whether the canonical code is empty. + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } +} + +impl fmt::Debug for CanonicalBytes { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("CanonicalBytes") + .field(&self.0.len()) + .finish() + } +} + +impl From> for CanonicalBytes { + fn from(value: Vec) -> Self { + Self(Arc::<[u8]>::from(value)) + } +} + +/// Shared execution family for method-backed neural backends. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] +#[non_exhaustive] +pub enum MethodBackendFamily { + /// Mamba family. + Mamba, + /// RWKV-7 family. + Rwkv7, +} + +/// Trace-model execution strategy used by VM/AIXI adapters. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] +#[non_exhaustive] +pub enum RateBackendTraceStrategy { + /// Direct ROSA-specific strategy. + Rosa, + /// Direct CTW strategy. + Ctw, + /// Direct FAC-CTW strategy. + FacCtw, + /// Generic predictor-backed strategy. + PredictorBacked, + /// ZPAQ-specific rate model strategy. + Zpaq, + /// Mamba compressor-backed strategy. + Mamba, + /// RWKV compressor-backed strategy. + Rwkv7, +} + +/// Shared capability metadata for rate backends. +#[derive(Clone, Debug)] +#[non_exhaustive] +pub struct RateBackendCapabilities { + /// Canonical backend family name. + pub canonical_name: &'static str, + /// Human-readable backend label. + pub display_label: Arc, + /// VM/AIXI trace strategy for this backend family. + pub trace_strategy: RateBackendTraceStrategy, + /// Whether biased/plugin entropy is supported. + pub supports_biased_entropy: bool, + /// Whether generic frozen-conditioning is supported. + pub supports_frozen_conditioning: bool, + /// Whether generic rate-coded compression wrapping is supported. + pub supports_rate_coded_compression: bool, + /// Whether this backend has a direct native bit predictor rather than a + /// byte-symbol adaptation over `{0,1}`. + pub supports_native_bit_prediction: bool, + /// Whether this backend can expose its byte PDF as a lazy binary prefix mass. + pub supports_byte_prefix_mass: bool, + /// Whether repeated byte-packed bit-session prefix queries remain practical + /// without a replay-heavy fallback. + pub supports_efficient_byte_packed_bit_sessions: bool, + /// Whether bit observations can be undone exactly after update. + pub supports_reversible_bit_updates: bool, + /// Whether the backend graph contains any ZPAQ component. + pub contains_zpaq: bool, + /// Whether this is a method-backed neural family. + pub method_family: Option, +} + +/// Shared capability metadata for compression backends. +#[derive(Clone, Debug)] +#[non_exhaustive] +pub struct CompressionBackendCapabilities { + /// Canonical backend family name. + pub canonical_name: &'static str, + /// Human-readable backend label. + pub display_label: Arc, + /// Whether this backend wraps a predictive rate backend. + pub uses_rate_backend: bool, + /// Whether this backend supports decompression. + pub supports_decompression: bool, +} + +#[derive(Clone, Debug)] +pub(crate) struct RateBackendPlanExpert { + pub name: Option, + pub log_prior: f64, + pub backend: Arc, +} + +#[derive(Clone, Debug)] +pub(crate) enum RateBackendPlan { + RosaPlus { + max_order: i64, + }, + Match { + hash_bits: usize, + min_len: usize, + max_len: usize, + base_mix: f64, + confidence_scale: f64, + }, + SparseMatch { + hash_bits: usize, + min_len: usize, + max_len: usize, + gap_min: usize, + gap_max: usize, + base_mix: f64, + confidence_scale: f64, + }, + Ppmd { + order: usize, + memory_mb: usize, + }, + Sequitur { + context_bytes: usize, + }, + Ctw { + depth: usize, + }, + FacCtw { + base_depth: usize, + num_percept_bits: usize, + encoding_bits: usize, + msb_first: bool, + }, + Zpaq { + method: String, + }, + #[cfg(feature = "backend-mamba")] + Mamba { + method: String, + parsed_method: crate::mambazip::MethodSpec, + asset: Option, + }, + #[cfg(feature = "backend-rwkv")] + Rwkv7 { + method: String, + parsed_method: crate::rwkvzip::MethodSpec, + asset: Option, + }, + Mixture { + kind: MixtureKind, + schedule: MixtureScheduleMode, + alpha: f64, + decay: Option, + experts: Box<[RateBackendPlanExpert]>, + }, + Particle { + spec: ParticleSpec, + }, + Calibrated { + context: CalibrationContextKind, + bins: usize, + learning_rate: f64, + bias_clip: f64, + base: Arc, + }, +} + +impl RateBackendPlan { + pub(crate) fn kind(&self) -> crate::runtime::RateBackendKind { + match self { + RateBackendPlan::RosaPlus { .. } => crate::runtime::RateBackendKind::RosaPlus, + RateBackendPlan::Match { .. } => crate::runtime::RateBackendKind::Match, + RateBackendPlan::SparseMatch { .. } => crate::runtime::RateBackendKind::SparseMatch, + RateBackendPlan::Ppmd { .. } => crate::runtime::RateBackendKind::Ppmd, + RateBackendPlan::Sequitur { .. } => crate::runtime::RateBackendKind::Sequitur, + RateBackendPlan::Ctw { .. } => crate::runtime::RateBackendKind::Ctw, + RateBackendPlan::FacCtw { .. } => crate::runtime::RateBackendKind::FacCtw, + RateBackendPlan::Zpaq { .. } => crate::runtime::RateBackendKind::Zpaq, + #[cfg(feature = "backend-mamba")] + RateBackendPlan::Mamba { .. } => crate::runtime::RateBackendKind::Mamba, + #[cfg(feature = "backend-rwkv")] + RateBackendPlan::Rwkv7 { .. } => crate::runtime::RateBackendKind::Rwkv7, + RateBackendPlan::Mixture { .. } => crate::runtime::RateBackendKind::Mixture, + RateBackendPlan::Particle { .. } => crate::runtime::RateBackendKind::Particle, + RateBackendPlan::Calibrated { .. } => crate::runtime::RateBackendKind::Calibrated, + } + } +} + +#[derive(Clone, Debug)] +pub(crate) enum CompressionBackendPlan { + Zpaq { + method: String, + threads: usize, + }, + #[cfg(feature = "backend-rwkv")] + Rwkv7 { + method: String, + parsed_method: crate::rwkvzip::MethodSpec, + asset: Option, + coder: CoderType, + }, + Rate { + rate_backend: Arc, + coder: CoderType, + framing: FramingMode, + }, +} + +impl CompressionBackendPlan { + pub(crate) fn kind(&self) -> crate::runtime::CompressionBackendKind { + match self { + CompressionBackendPlan::Zpaq { .. } => crate::runtime::CompressionBackendKind::Zpaq, + #[cfg(feature = "backend-rwkv")] + CompressionBackendPlan::Rwkv7 { .. } => crate::runtime::CompressionBackendKind::Rwkv7, + CompressionBackendPlan::Rate { + coder: CoderType::AC, + .. + } => crate::runtime::CompressionBackendKind::RateAc, + CompressionBackendPlan::Rate { + coder: CoderType::RANS, + .. + } => crate::runtime::CompressionBackendKind::RateRans, + } + } +} + +/// Canonicalized and feature-validated rate backend wrapper. +#[derive(Clone)] +pub struct ValidatedRateBackend { + canonical_spec: Arc, + canonical_bytes: CanonicalBytes, + capabilities: RateBackendCapabilities, + plan: Arc, +} + +impl ValidatedRateBackend { + /// Canonical wrapper AST for this backend. + pub fn canonical_spec(&self) -> &RateBackend { + self.canonical_spec.as_ref() + } + + /// Deterministic binary canonical code for this backend. + pub fn canonical_bytes(&self) -> &CanonicalBytes { + &self.canonical_bytes + } + + /// Capability metadata derived from the canonicalized backend graph. + pub fn capabilities(&self) -> &RateBackendCapabilities { + &self.capabilities + } + + /// Human-readable backend label derived from the compiled plan. + pub fn display_label(&self) -> String { + rate_backend_plan_display_label(self.plan.as_ref()) + } + + /// Short default backend name for logs, diagnostics, and model labels. + pub fn default_name(&self) -> String { + rate_backend_plan_default_name(self.plan.as_ref()) + } + + /// Compile the validated spec into an immutable runtime plan. + pub fn compile(&self) -> SpecResult { + Ok(CompiledRateBackend { + canonical_spec: self.canonical_spec.clone(), + canonical_bytes: self.canonical_bytes.clone(), + capabilities: self.capabilities.clone(), + plan: self.plan.clone(), + }) + } +} + +/// Canonicalized and feature-validated compression backend wrapper. +#[derive(Clone)] +pub struct ValidatedCompressionBackend { + canonical_spec: Arc, + canonical_bytes: CanonicalBytes, + capabilities: CompressionBackendCapabilities, + plan: Arc, +} + +impl ValidatedCompressionBackend { + /// Canonical wrapper AST for this backend. + pub fn canonical_spec(&self) -> &CompressionBackend { + self.canonical_spec.as_ref() + } + + /// Deterministic binary canonical code for this backend. + pub fn canonical_bytes(&self) -> &CanonicalBytes { + &self.canonical_bytes + } + + /// Capability metadata derived from the canonicalized backend graph. + pub fn capabilities(&self) -> &CompressionBackendCapabilities { + &self.capabilities + } + + /// Compile the validated spec into an immutable runtime plan. + pub fn compile(&self) -> SpecResult { + Ok(CompiledCompressionBackend { + canonical_spec: self.canonical_spec.clone(), + canonical_bytes: self.canonical_bytes.clone(), + capabilities: self.capabilities.clone(), + plan: self.plan.clone(), + }) + } +} + +/// Compiled immutable rate-backend runtime plan. +#[derive(Clone)] +pub struct CompiledRateBackend { + canonical_spec: Arc, + canonical_bytes: CanonicalBytes, + capabilities: RateBackendCapabilities, + plan: Arc, +} + +impl CompiledRateBackend { + /// Canonical wrapper AST for this backend. + pub fn canonical_spec(&self) -> &RateBackend { + self.canonical_spec.as_ref() + } + + /// Deterministic binary canonical code for this backend. + pub fn canonical_bytes(&self) -> &CanonicalBytes { + &self.canonical_bytes + } + + /// Capability metadata derived from the compiled backend graph. + pub fn capabilities(&self) -> &RateBackendCapabilities { + &self.capabilities + } + + /// Compile-friendly backend display label. + pub fn display_label(&self) -> String { + rate_backend_plan_display_label(self.plan.as_ref()) + } + + /// Short default backend name for logs, diagnostics, and model labels. + pub fn default_name(&self) -> String { + rate_backend_plan_default_name(self.plan.as_ref()) + } + + /// Canonical backend family name. + pub fn canonical_name(&self) -> &'static str { + self.capabilities.canonical_name + } + + /// Whether the compiled backend graph contains any ZPAQ component. + pub fn contains_zpaq(&self) -> bool { + self.capabilities.contains_zpaq + } + + /// Whether this backend supports generic frozen conditioning. + pub fn supports_frozen_conditioning(&self) -> bool { + self.capabilities.supports_frozen_conditioning + } + + /// Whether this backend supports generic rate-coded compression wrapping. + pub fn supports_rate_coded_compression(&self) -> bool { + self.capabilities.supports_rate_coded_compression + } + + /// Whether this backend has a direct native bit predictor rather than a + /// byte-symbol adaptation over `{0,1}`. + pub fn supports_native_bit_prediction(&self) -> bool { + self.capabilities.supports_native_bit_prediction + } + + /// Whether this backend can expose its byte PDF as a lazy binary prefix mass. + pub fn supports_byte_prefix_mass(&self) -> bool { + self.capabilities.supports_byte_prefix_mass + } + + /// Whether this backend can support repeated byte-packed bit-session prefix + /// queries without pathological replay-heavy fallback. + pub fn supports_efficient_byte_packed_bit_sessions(&self) -> bool { + self.capabilities + .supports_efficient_byte_packed_bit_sessions + } + + /// Whether bit observations can be undone exactly after update. + pub fn supports_reversible_bit_updates(&self) -> bool { + self.capabilities.supports_reversible_bit_updates + } + + pub(crate) fn plan(&self) -> &RateBackendPlan { + self.plan.as_ref() + } + + #[allow(dead_code)] + pub(crate) fn method_string(&self) -> Option<&str> { + match self.plan() { + #[cfg(feature = "backend-mamba")] + RateBackendPlan::Mamba { method, .. } => Some(method.as_str()), + #[cfg(feature = "backend-rwkv")] + RateBackendPlan::Rwkv7 { method, .. } => Some(method.as_str()), + _ => None, + } + } +} + +/// Compiled immutable compression-backend runtime plan. +#[derive(Clone)] +pub struct CompiledCompressionBackend { + canonical_spec: Arc, + canonical_bytes: CanonicalBytes, + capabilities: CompressionBackendCapabilities, + plan: Arc, +} + +impl CompiledCompressionBackend { + /// Canonical wrapper AST for this backend. + pub fn canonical_spec(&self) -> &CompressionBackend { + self.canonical_spec.as_ref() + } + + /// Deterministic binary canonical code for this backend. + pub fn canonical_bytes(&self) -> &CanonicalBytes { + &self.canonical_bytes + } + + /// Capability metadata derived from the compiled backend graph. + pub fn capabilities(&self) -> &CompressionBackendCapabilities { + &self.capabilities + } + + pub(crate) fn plan(&self) -> &CompressionBackendPlan { + self.plan.as_ref() + } +} + +pub(crate) fn validate_rate_backend_in( + backend: &RateBackend, + env: &SpecEnvironment, +) -> SpecResult { + let plan = Arc::new(build_rate_plan(backend, env, MAX_MIXTURE_NESTING)?); + let canonical_spec = Arc::new(rate_plan_to_wrapper(plan.as_ref())); + crate::api::validate_rate_backend(canonical_spec.as_ref()) + .map_err(|err| SpecError::new(err.to_string()))?; + Ok(ValidatedRateBackend { + canonical_spec, + canonical_bytes: encode_rate_backend_plan(plan.as_ref()), + capabilities: rate_backend_capabilities(plan.as_ref()), + plan, + }) +} + +pub(crate) fn validate_compression_backend_in( + backend: &CompressionBackend, + env: &SpecEnvironment, +) -> SpecResult { + let plan = Arc::new(build_compression_plan(backend, env)?); + let canonical_spec = Arc::new(compression_plan_to_wrapper(plan.as_ref())); + crate::api::validate_compression_backend(canonical_spec.as_ref()) + .map_err(|err| SpecError::new(err.to_string()))?; + Ok(ValidatedCompressionBackend { + canonical_spec, + canonical_bytes: encode_compression_backend_plan(plan.as_ref()), + capabilities: compression_backend_capabilities(plan.as_ref()), + plan, + }) +} + +pub(crate) fn compiled_rate_backend_from_plan( + plan: Arc, +) -> SpecResult { + let canonical_spec = Arc::new(rate_plan_to_wrapper(plan.as_ref())); + crate::api::validate_rate_backend(canonical_spec.as_ref()) + .map_err(|err| SpecError::new(err.to_string()))?; + Ok(CompiledRateBackend { + canonical_bytes: encode_rate_backend_plan(plan.as_ref()), + capabilities: rate_backend_capabilities(plan.as_ref()), + canonical_spec, + plan, + }) +} + +pub(crate) fn compiled_compression_backend_from_plan( + plan: Arc, +) -> SpecResult { + let canonical_spec = Arc::new(compression_plan_to_wrapper(plan.as_ref())); + crate::api::validate_compression_backend(canonical_spec.as_ref()) + .map_err(|err| SpecError::new(err.to_string()))?; + Ok(CompiledCompressionBackend { + canonical_bytes: encode_compression_backend_plan(plan.as_ref()), + capabilities: compression_backend_capabilities(plan.as_ref()), + canonical_spec, + plan, + }) +} + +fn build_rate_plan( + backend: &RateBackend, + env: &SpecEnvironment, + depth: usize, +) -> SpecResult { + if depth == 0 { + return Err(SpecError::new("backend spec nesting too deep")); + } + + let descriptor = backend + .descriptor() + .map_err(|err| SpecError::new(format!("{err} (while validating rate backend)")))?; + if !descriptor.enabled { + let Some(feature) = descriptor.feature else { + return Err(SpecError::new(format!( + "internal backend registry mismatch: disabled backend '{}' is missing required feature metadata", + descriptor.canonical + ))); + }; + return Err(SpecError::new(format!( + "backend '{}' requires infotheory feature '{}'", + descriptor.canonical, feature + ))); + } + + crate::runtime::compile_rate_backend_plan_via_kernel(descriptor.kind, backend, env, depth) +} + +fn build_compression_plan( + backend: &CompressionBackend, + env: &SpecEnvironment, +) -> SpecResult { + let descriptor = backend + .descriptor() + .map_err(|err| SpecError::new(format!("{err} (while validating compression backend)")))?; + if !descriptor.enabled { + let Some(feature) = descriptor.feature else { + return Err(SpecError::new(format!( + "internal backend registry mismatch: disabled compression backend '{}' is missing required feature metadata", + descriptor.canonical + ))); + }; + return Err(SpecError::new(format!( + "compression backend '{}' requires infotheory feature '{}'", + descriptor.canonical, feature + ))); + } + + crate::runtime::compile_compression_backend_plan_via_kernel(descriptor.kind, backend, env) +} + +pub(crate) fn rate_plan_to_wrapper(plan: &RateBackendPlan) -> RateBackend { + crate::runtime::rate_backend_wrapper_via_kernel(plan) +} + +fn compression_plan_to_wrapper(plan: &CompressionBackendPlan) -> CompressionBackend { + crate::runtime::compression_backend_wrapper_via_kernel(plan) +} + +pub(crate) fn rate_backend_capabilities(plan: &RateBackendPlan) -> RateBackendCapabilities { + crate::runtime::rate_backend_capabilities_via_kernel(plan) +} + +fn compression_backend_capabilities( + plan: &CompressionBackendPlan, +) -> CompressionBackendCapabilities { + crate::runtime::compression_backend_capabilities_via_kernel(plan) +} + +fn rate_plan_contains_zpaq(plan: &RateBackendPlan) -> bool { + (crate::runtime::rate_backend_kernel(plan.kind()).contains_zpaq)(plan) +} + +fn rate_backend_plan_display_label(plan: &RateBackendPlan) -> String { + crate::runtime::rate_backend_display_label_via_kernel(plan) +} + +fn rate_backend_plan_default_name(plan: &RateBackendPlan) -> String { + crate::runtime::rate_backend_default_name_via_kernel(plan) +} + +#[cfg(feature = "backend-rwkv")] +fn rwkv_asset_ref(spec: &crate::rwkvzip::MethodSpec) -> Option { + match spec { + crate::rwkvzip::MethodSpec::File { path, .. } => Some(AssetRef::Filesystem(path.clone())), + crate::rwkvzip::MethodSpec::Online { .. } => None, + } +} + +#[cfg(feature = "backend-mamba")] +fn mamba_asset_ref(spec: &crate::mambazip::MethodSpec) -> Option { + match spec { + crate::mambazip::MethodSpec::File { path, .. } => Some(AssetRef::Filesystem(path.clone())), + crate::mambazip::MethodSpec::Online { .. } => None, + } +} + +pub(crate) fn encode_rate_backend_plan(plan: &RateBackendPlan) -> CanonicalBytes { + let mut payload = Vec::new(); + payload.extend_from_slice(b"itrb"); + payload.push(1); + encode_rate_backend_payload(plan, &mut payload); + frame_canonical_payload(payload) +} + +fn encode_compression_backend_plan(plan: &CompressionBackendPlan) -> CanonicalBytes { + let mut payload = Vec::new(); + payload.extend_from_slice(b"itcb"); + payload.push(1); + encode_compression_backend_payload(plan, &mut payload); + frame_canonical_payload(payload) +} + +fn frame_canonical_payload(payload: Vec) -> CanonicalBytes { + let mut framed = Vec::with_capacity(payload.len() + 10); + push_varint(&mut framed, payload.len() as u64); + framed.extend_from_slice(&payload); + CanonicalBytes::from(framed) +} + +fn encode_rate_backend_payload(plan: &RateBackendPlan, out: &mut Vec) { + crate::runtime::encode_rate_backend_payload_via_kernel(plan, out) +} + +fn encode_compression_backend_payload(plan: &CompressionBackendPlan, out: &mut Vec) { + crate::runtime::encode_compression_backend_payload_via_kernel(plan, out) +} + +fn encode_particle_spec(spec: &ParticleSpec, out: &mut Vec) { + push_usize(out, spec.num_particles); + push_usize(out, spec.context_window); + push_usize(out, spec.unroll_steps); + push_usize(out, spec.num_cells); + push_usize(out, spec.cell_dim); + push_usize(out, spec.num_rules); + push_usize(out, spec.selector_hidden); + push_usize(out, spec.rule_hidden); + push_usize(out, spec.noise_dim); + push_bool(out, spec.deterministic); + push_bool(out, spec.enable_noise); + push_f64(out, spec.noise_scale); + push_usize(out, spec.noise_anneal_steps); + push_f64(out, spec.learning_rate_readout); + push_f64(out, spec.learning_rate_selector); + push_f64(out, spec.learning_rate_rule); + push_usize(out, spec.bptt_depth); + push_f64(out, spec.optimizer_momentum); + push_f64(out, spec.grad_clip); + push_f64(out, spec.state_clip); + push_f64(out, spec.forget_lambda); + push_f64(out, spec.resample_threshold); + push_f64(out, spec.mutate_fraction); + push_f64(out, spec.mutate_scale); + push_bool(out, spec.mutate_model_params); + push_usize(out, spec.diagnostics_interval); + push_f64(out, spec.min_prob); + push_varint(out, spec.seed); +} + +fn push_varint(out: &mut Vec, mut value: u64) { + while value >= 0x80 { + out.push((value as u8) | 0x80); + value >>= 7; + } + out.push(value as u8); +} + +fn push_i64(out: &mut Vec, value: i64) { + let zigzag = ((value << 1) ^ (value >> 63)) as u64; + push_varint(out, zigzag); +} + +fn push_usize(out: &mut Vec, value: usize) { + push_varint(out, value as u64); +} + +fn push_bool(out: &mut Vec, value: bool) { + out.push(u8::from(value)); +} + +fn push_f64(out: &mut Vec, value: f64) { + out.extend_from_slice(&value.to_bits().to_le_bytes()); +} + +fn push_string(out: &mut Vec, value: &str) { + push_varint(out, value.len() as u64); + out.extend_from_slice(value.as_bytes()); +} + +fn push_option_string(out: &mut Vec, value: Option<&str>) { + match value { + Some(value) => { + out.push(1); + push_string(out, value); + } + None => out.push(0), + } +} + +fn push_option_f64(out: &mut Vec, value: Option) { + match value { + Some(value) => { + out.push(1); + push_f64(out, value); + } + None => out.push(0), + } +} + +#[cfg(any(feature = "backend-mamba", feature = "backend-rwkv"))] +fn push_asset_ref(out: &mut Vec, value: Option<&AssetRef>) { + match value { + Some(AssetRef::Filesystem(path)) => { + out.push(1); + push_string(out, &path.to_string_lossy()); + } + None => out.push(0), + } +} + +fn coder_tag(coder: CoderType) -> u8 { + match coder { + CoderType::AC => 0, + CoderType::RANS => 1, + } +} + +fn framing_tag(framing: FramingMode) -> u8 { + match framing { + FramingMode::Raw => 0, + FramingMode::Framed => 1, + } +} + +fn mixture_kind_tag(kind: MixtureKind) -> u8 { + match kind { + MixtureKind::Bayes => 0, + MixtureKind::FadingBayes => 1, + MixtureKind::Switching => 2, + MixtureKind::Convex => 3, + MixtureKind::Mdl => 4, + MixtureKind::Neural => 5, + } +} + +fn mixture_schedule_tag(schedule: MixtureScheduleMode) -> u8 { + match schedule { + MixtureScheduleMode::Default => 0, + MixtureScheduleMode::Theorem => 1, + } +} + +fn calibration_context_tag(context: CalibrationContextKind) -> u8 { + match context { + CalibrationContextKind::Global => 0, + CalibrationContextKind::ByteClass => 1, + CalibrationContextKind::Text => 2, + CalibrationContextKind::Repeat => 3, + CalibrationContextKind::TextRepeat => 4, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn assert_not_prefix(a: &CanonicalBytes, b: &CanonicalBytes) { + assert!( + !b.as_slice().starts_with(a.as_slice()), + "canonical encoding must be prefix-free" + ); + } + + fn read_varint_prefix(bytes: &[u8]) -> Result<(u64, usize), String> { + let mut shift: u32 = 0; + let mut out: u64 = 0; + let mut index: usize = 0; + loop { + let byte = *bytes.get(index).ok_or_else(|| { + "unexpected end while decoding canonical frame length".to_string() + })?; + out |= ((byte & 0x7f) as u64) << shift; + index = index.saturating_add(1); + if byte & 0x80 == 0 { + return Ok((out, index)); + } + shift = shift.saturating_add(7); + if shift > 63 { + return Err("invalid canonical frame varint length".to_string()); + } + } + } + + fn parse_framed_canonical_payload(bytes: &[u8]) -> Result<&[u8], String> { + let (declared_len_u64, header_len) = read_varint_prefix(bytes)?; + let declared_len = usize::try_from(declared_len_u64) + .map_err(|_| "canonical frame length exceeds usize::MAX".to_string())?; + let payload_end = header_len + .checked_add(declared_len) + .ok_or_else(|| "canonical frame length overflow".to_string())?; + if bytes.len() < payload_end { + return Err("truncated canonical frame payload".to_string()); + } + if bytes.len() > payload_end { + return Err("canonical frame has trailing bytes".to_string()); + } + Ok(&bytes[header_len..payload_end]) + } + + fn sample_compression_backend_corpus() -> Vec { + let mut out = Vec::::new(); + let leaf = crate::runtime::first_enabled_default_rate_backend_spec(); + for descriptor in crate::runtime::COMPRESSION_BACKEND_REGISTRY { + if !descriptor.enabled { + continue; + } + match descriptor.kind { + crate::runtime::CompressionBackendKind::Zpaq => { + out.push(CompressionBackend::zpaq("5")); + } + crate::runtime::CompressionBackendKind::RateAc => { + if let Some(rate_backend) = leaf.clone() { + out.push(CompressionBackend::Rate { + rate_backend, + coder: crate::coders::CoderType::AC, + framing: crate::compression::FramingMode::Framed, + }); + } + } + crate::runtime::CompressionBackendKind::RateRans => { + if let Some(rate_backend) = leaf.clone() { + out.push(CompressionBackend::Rate { + rate_backend, + coder: crate::coders::CoderType::RANS, + framing: crate::compression::FramingMode::Raw, + }); + } + } + #[cfg(feature = "backend-rwkv")] + crate::runtime::CompressionBackendKind::Rwkv7 => { + let options = crate::spec::CompressionBackendShorthandOptions { + default_framing: crate::compression::FramingMode::Raw, + ..Default::default() + }; + let candidate = crate::spec::parse_compression_backend_name_method( + "rwkv7", + Some( + "cfg:hidden=64,intermediate=64,layers=1,train=sgd,lr=0.01;policy:schedule=0..100:infer", + ), + None, + &options, + ) + .expect("rwkv shorthand candidate"); + out.push(candidate); + } + #[cfg(not(feature = "backend-rwkv"))] + crate::runtime::CompressionBackendKind::Rwkv7 => {} + } + } + out + } + + #[test] + fn canonical_compression_code_frame_is_self_delimiting_and_typed() { + let env = SpecEnvironment::default(); + let corpus = sample_compression_backend_corpus(); + if corpus.is_empty() { + return; + } + for candidate in corpus { + let validated = + validate_compression_backend_in(&candidate, &env).expect("validate candidate"); + let bytes = validated.canonical_bytes.as_slice(); + let payload = parse_framed_canonical_payload(bytes).expect("valid canonical frame"); + assert!( + payload.len() >= 5, + "canonical payload must contain magic + version" + ); + assert_eq!(&payload[..4], b"itcb"); + assert_eq!(payload[4], 1_u8); + } + } + + #[test] + fn canonical_compression_code_frame_rejects_trailing_bytes_in_contract_parser() { + let env = SpecEnvironment::default(); + let Some(candidate) = sample_compression_backend_corpus().into_iter().next() else { + return; + }; + let validated = validate_compression_backend_in(&candidate, &env).expect("validate"); + let bytes = validated.canonical_bytes.as_slice().to_vec(); + parse_framed_canonical_payload(&bytes).expect("valid canonical frame"); + + let mut extended = bytes.clone(); + extended.extend_from_slice(&[0x00, 0x01]); + let err = parse_framed_canonical_payload(&extended) + .expect_err("trailing bytes must be rejected by canonical frame parser"); + assert!(err.contains("trailing bytes"), "{err}"); + } + + #[test] + fn canonical_rate_plan_bytes_are_prefix_free_for_sample_corpus() { + let env = SpecEnvironment::default(); + let samples: Vec<_> = crate::runtime::RATE_BACKEND_REGISTRY + .iter() + .filter(|descriptor| descriptor.enabled) + .filter_map(|descriptor| crate::runtime::default_rate_backend_spec(descriptor.kind)) + .take(4) + .collect(); + if samples.len() < 2 { + return; + } + let encodings: Vec<_> = samples + .iter() + .map(|backend| { + validate_rate_backend_in(backend, &env) + .unwrap() + .canonical_bytes + .clone() + }) + .collect(); + for (i, left) in encodings.iter().enumerate() { + for (j, right) in encodings.iter().enumerate() { + if i != j { + assert_not_prefix(left, right); + } + } + } + } + + #[test] + fn compiled_rate_backend_clone_is_o1_arc_backed() { + let Some(default_backend) = crate::runtime::first_enabled_default_rate_backend_spec() + else { + return; + }; + let compiled = validate_rate_backend_in(&default_backend, &SpecEnvironment::default()) + .unwrap() + .compile() + .unwrap(); + let cloned = compiled.clone(); + assert!(Arc::ptr_eq(&compiled.plan, &cloned.plan)); + assert!(Arc::ptr_eq( + &compiled.canonical_spec, + &cloned.canonical_spec + )); + } + + #[cfg(feature = "backend-ctw")] + #[test] + fn compiled_ctw_preserves_family_identity() { + let compiled = + validate_rate_backend_in(&RateBackend::Ctw { depth: 7 }, &SpecEnvironment::default()) + .unwrap() + .compile() + .unwrap(); + assert!(matches!( + compiled.canonical_spec(), + RateBackend::Ctw { depth: 7 } + )); + } + + #[cfg(feature = "backend-mixture")] + #[cfg(feature = "backend-zpaq")] + #[test] + fn compiled_capabilities_track_nested_zpaq_components() { + let backend = RateBackend::Mixture { + spec: Arc::new(MixtureSpec::new( + MixtureKind::Bayes, + vec![ + MixtureExpertSpec { + name: Some("ctw".to_string()), + log_prior: 0.0, + backend: RateBackend::Ctw { depth: 6 }, + }, + MixtureExpertSpec { + name: Some("zpaq".to_string()), + log_prior: -0.1, + backend: RateBackend::Zpaq { + method: crate::api::ZpaqMethodSpec::literal("1"), + }, + }, + ], + )), + }; + let compiled = validate_rate_backend_in(&backend, &SpecEnvironment::default()) + .unwrap() + .compile() + .unwrap(); + assert!(compiled.contains_zpaq()); + } +} diff --git a/crates/infotheory/src/spec/core/adapters.rs b/crates/infotheory/src/spec/core/adapters.rs new file mode 100644 index 00000000..98ca607e --- /dev/null +++ b/crates/infotheory/src/spec/core/adapters.rs @@ -0,0 +1,751 @@ +use super::*; + +pub(crate) fn rate_plan_to_wrapper_rosa(plan: &RateBackendPlan) -> RateBackend { + let RateBackendPlan::RosaPlus { max_order } = plan else { + unreachable!("rosa wrapper kernel used with non-rosa plan"); + }; + RateBackend::RosaPlus { + max_order: *max_order, + } +} + +pub(crate) fn rate_plan_to_wrapper_match(plan: &RateBackendPlan) -> RateBackend { + let RateBackendPlan::Match { + hash_bits, + min_len, + max_len, + base_mix, + confidence_scale, + } = plan + else { + unreachable!("match wrapper kernel used with non-match plan"); + }; + RateBackend::Match { + hash_bits: *hash_bits, + min_len: *min_len, + max_len: *max_len, + base_mix: *base_mix, + confidence_scale: *confidence_scale, + } +} + +pub(crate) fn rate_plan_to_wrapper_sparse_match(plan: &RateBackendPlan) -> RateBackend { + let RateBackendPlan::SparseMatch { + hash_bits, + min_len, + max_len, + gap_min, + gap_max, + base_mix, + confidence_scale, + } = plan + else { + unreachable!("sparse-match wrapper kernel used with non-sparse-match plan"); + }; + RateBackend::SparseMatch { + hash_bits: *hash_bits, + min_len: *min_len, + max_len: *max_len, + gap_min: *gap_min, + gap_max: *gap_max, + base_mix: *base_mix, + confidence_scale: *confidence_scale, + } +} + +pub(crate) fn rate_plan_to_wrapper_ppmd(plan: &RateBackendPlan) -> RateBackend { + let RateBackendPlan::Ppmd { order, memory_mb } = plan else { + unreachable!("ppmd wrapper kernel used with non-ppmd plan"); + }; + RateBackend::Ppmd { + order: *order, + memory_mb: *memory_mb, + } +} + +pub(crate) fn rate_plan_to_wrapper_sequitur(plan: &RateBackendPlan) -> RateBackend { + let RateBackendPlan::Sequitur { context_bytes } = plan else { + unreachable!("sequitur wrapper kernel used with non-sequitur plan"); + }; + RateBackend::Sequitur { + context_bytes: *context_bytes, + } +} + +pub(crate) fn rate_plan_to_wrapper_ctw(plan: &RateBackendPlan) -> RateBackend { + let RateBackendPlan::Ctw { depth } = plan else { + unreachable!("ctw wrapper kernel used with non-ctw plan"); + }; + RateBackend::Ctw { depth: *depth } +} + +pub(crate) fn rate_plan_to_wrapper_fac_ctw(plan: &RateBackendPlan) -> RateBackend { + let RateBackendPlan::FacCtw { + base_depth, + num_percept_bits, + encoding_bits, + msb_first, + } = plan + else { + unreachable!("fac-ctw wrapper kernel used with non-fac-ctw plan"); + }; + RateBackend::FacCtw { + base_depth: *base_depth, + num_percept_bits: *num_percept_bits, + encoding_bits: *encoding_bits, + msb_first: Some(*msb_first), + } +} + +pub(crate) fn rate_plan_to_wrapper_zpaq(plan: &RateBackendPlan) -> RateBackend { + let RateBackendPlan::Zpaq { method } = plan else { + unreachable!("zpaq wrapper kernel used with non-zpaq plan"); + }; + RateBackend::Zpaq { + method: crate::api::ZpaqMethodSpec::literal(method), + } +} + +#[cfg(feature = "backend-mamba")] +pub(crate) fn rate_plan_to_wrapper_mamba(plan: &RateBackendPlan) -> RateBackend { + let RateBackendPlan::Mamba { method, .. } = plan else { + unreachable!("mamba wrapper kernel used with non-mamba plan"); + }; + RateBackend::MambaMethod { + method: crate::mambazip::parse_method_spec(method) + .expect("compiled mamba plan must retain a valid canonical method"), + } +} + +#[cfg(feature = "backend-rwkv")] +pub(crate) fn rate_plan_to_wrapper_rwkv7(plan: &RateBackendPlan) -> RateBackend { + let RateBackendPlan::Rwkv7 { method, .. } = plan else { + unreachable!("rwkv7 wrapper kernel used with non-rwkv7 plan"); + }; + RateBackend::Rwkv7Method { + method: crate::rwkvzip::parse_method_spec(method) + .expect("compiled rwkv plan must retain a valid canonical method"), + } +} + +pub(crate) fn rate_plan_to_wrapper_mixture(plan: &RateBackendPlan) -> RateBackend { + let RateBackendPlan::Mixture { + kind, + schedule, + alpha, + decay, + experts, + } = plan + else { + unreachable!("mixture wrapper kernel used with non-mixture plan"); + }; + RateBackend::Mixture { + spec: Arc::new(MixtureSpec { + kind: *kind, + schedule: *schedule, + alpha: *alpha, + decay: *decay, + experts: experts + .iter() + .map(|expert| MixtureExpertSpec { + name: expert.name.clone(), + log_prior: expert.log_prior, + backend: rate_plan_to_wrapper(expert.backend.as_ref()), + }) + .collect(), + }), + } +} + +pub(crate) fn rate_plan_to_wrapper_particle(plan: &RateBackendPlan) -> RateBackend { + let RateBackendPlan::Particle { spec } = plan else { + unreachable!("particle wrapper kernel used with non-particle plan"); + }; + RateBackend::Particle { + spec: Arc::new(spec.clone()), + } +} + +pub(crate) fn rate_plan_to_wrapper_calibrated(plan: &RateBackendPlan) -> RateBackend { + let RateBackendPlan::Calibrated { + context, + bins, + learning_rate, + bias_clip, + base, + } = plan + else { + unreachable!("calibrated wrapper kernel used with non-calibrated plan"); + }; + RateBackend::Calibrated { + spec: Arc::new(CalibratedSpec { + base: rate_plan_to_wrapper(base.as_ref()), + context: *context, + bins: *bins, + learning_rate: *learning_rate, + bias_clip: *bias_clip, + }), + } +} + +pub(crate) fn compression_plan_to_wrapper_zpaq( + plan: &CompressionBackendPlan, +) -> CompressionBackend { + let CompressionBackendPlan::Zpaq { method, threads } = plan else { + unreachable!("zpaq compression wrapper kernel used with non-zpaq plan"); + }; + CompressionBackend::Zpaq { + method: crate::api::ZpaqMethodSpec::literal(method), + threads: std::num::NonZeroUsize::new(*threads) + .expect("compiled zpaq compression plan must retain non-zero thread count"), + } +} + +#[cfg(feature = "backend-rwkv")] +pub(crate) fn compression_plan_to_wrapper_rwkv7( + plan: &CompressionBackendPlan, +) -> CompressionBackend { + let CompressionBackendPlan::Rwkv7 { method, coder, .. } = plan else { + unreachable!("rwkv7 compression wrapper kernel used with non-rwkv7 plan"); + }; + CompressionBackend::Rwkv7 { + method: crate::rwkvzip::parse_method_spec(method) + .expect("compiled rwkv compression plan must retain a valid canonical method"), + coder: *coder, + } +} + +pub(crate) fn compression_plan_to_wrapper_rate( + plan: &CompressionBackendPlan, +) -> CompressionBackend { + let CompressionBackendPlan::Rate { + rate_backend, + coder, + framing, + } = plan + else { + unreachable!("rate compression wrapper kernel used with non-rate plan"); + }; + CompressionBackend::Rate { + rate_backend: rate_plan_to_wrapper(rate_backend.as_ref()), + coder: *coder, + framing: *framing, + } +} + +pub(crate) fn rate_plan_contains_zpaq_false(_plan: &RateBackendPlan) -> bool { + false +} + +pub(crate) fn rate_plan_contains_zpaq_true(_plan: &RateBackendPlan) -> bool { + true +} + +pub(crate) fn rate_plan_contains_zpaq_mixture(plan: &RateBackendPlan) -> bool { + let RateBackendPlan::Mixture { experts, .. } = plan else { + unreachable!("mixture zpaq kernel used with non-mixture plan"); + }; + experts + .iter() + .any(|expert| rate_plan_contains_zpaq(expert.backend.as_ref())) +} + +pub(crate) fn rate_plan_contains_zpaq_calibrated(plan: &RateBackendPlan) -> bool { + let RateBackendPlan::Calibrated { base, .. } = plan else { + unreachable!("calibrated zpaq kernel used with non-calibrated plan"); + }; + rate_plan_contains_zpaq(base.as_ref()) +} + +pub(crate) fn rate_plan_display_label_rosa(plan: &RateBackendPlan) -> String { + let RateBackendPlan::RosaPlus { max_order } = plan else { + unreachable!("rosa label kernel used with non-rosa plan"); + }; + format!("rosaplus(max_order={max_order})") +} + +pub(crate) fn rate_plan_default_name_rosa(plan: &RateBackendPlan) -> String { + let RateBackendPlan::RosaPlus { max_order } = plan else { + unreachable!("rosa default-name kernel used with non-rosa plan"); + }; + format!("rosa(mo={max_order})") +} + +pub(crate) fn rate_plan_display_label_match(plan: &RateBackendPlan) -> String { + let RateBackendPlan::Match { + hash_bits, + min_len, + max_len, + base_mix, + confidence_scale, + } = plan + else { + unreachable!("match label kernel used with non-match plan"); + }; + format!( + "match(hash_bits={hash_bits},min_len={min_len},max_len={max_len},base_mix={base_mix},confidence_scale={confidence_scale})" + ) +} + +pub(crate) fn rate_plan_default_name_match(plan: &RateBackendPlan) -> String { + let RateBackendPlan::Match { .. } = plan else { + unreachable!("match default-name kernel used with non-match plan"); + }; + "match".to_string() +} + +pub(crate) fn rate_plan_display_label_sparse_match(plan: &RateBackendPlan) -> String { + let RateBackendPlan::SparseMatch { + hash_bits, + min_len, + max_len, + gap_min, + gap_max, + base_mix, + confidence_scale, + } = plan + else { + unreachable!("sparse-match label kernel used with non-sparse-match plan"); + }; + format!( + "sparse-match(hash_bits={hash_bits},min_len={min_len},max_len={max_len},gap_min={gap_min},gap_max={gap_max},base_mix={base_mix},confidence_scale={confidence_scale})" + ) +} + +pub(crate) fn rate_plan_default_name_sparse_match(plan: &RateBackendPlan) -> String { + let RateBackendPlan::SparseMatch { .. } = plan else { + unreachable!("sparse-match default-name kernel used with non-sparse-match plan"); + }; + "sparse-match".to_string() +} + +pub(crate) fn rate_plan_display_label_ppmd(plan: &RateBackendPlan) -> String { + let RateBackendPlan::Ppmd { order, memory_mb } = plan else { + unreachable!("ppmd label kernel used with non-ppmd plan"); + }; + format!("ppmd(order={order},memory_mb={memory_mb})") +} + +pub(crate) fn rate_plan_default_name_ppmd(plan: &RateBackendPlan) -> String { + let RateBackendPlan::Ppmd { order, memory_mb } = plan else { + unreachable!("ppmd default-name kernel used with non-ppmd plan"); + }; + format!("ppmd(o={order},m={memory_mb}MiB)") +} + +pub(crate) fn rate_plan_display_label_sequitur(plan: &RateBackendPlan) -> String { + let RateBackendPlan::Sequitur { context_bytes } = plan else { + unreachable!("sequitur label kernel used with non-sequitur plan"); + }; + format!("sequitur(context_bytes={context_bytes})") +} + +pub(crate) fn rate_plan_default_name_sequitur(plan: &RateBackendPlan) -> String { + let RateBackendPlan::Sequitur { context_bytes } = plan else { + unreachable!("sequitur default-name kernel used with non-sequitur plan"); + }; + format!("sequitur(ctx={context_bytes})") +} + +pub(crate) fn rate_plan_display_label_ctw(plan: &RateBackendPlan) -> String { + let RateBackendPlan::Ctw { depth } = plan else { + unreachable!("ctw label kernel used with non-ctw plan"); + }; + format!("ctw(depth={depth})") +} + +pub(crate) fn rate_plan_default_name_ctw(plan: &RateBackendPlan) -> String { + let RateBackendPlan::Ctw { depth } = plan else { + unreachable!("ctw default-name kernel used with non-ctw plan"); + }; + format!("ctw(d={depth})") +} + +pub(crate) fn rate_plan_display_label_fac_ctw(plan: &RateBackendPlan) -> String { + let RateBackendPlan::FacCtw { + base_depth, + num_percept_bits, + encoding_bits, + msb_first, + } = plan + else { + unreachable!("fac-ctw label kernel used with non-fac-ctw plan"); + }; + format!( + "fac-ctw(base_depth={base_depth},num_percept_bits={num_percept_bits},encoding_bits={encoding_bits},msb_first={msb_first})" + ) +} + +pub(crate) fn rate_plan_default_name_fac_ctw(plan: &RateBackendPlan) -> String { + let RateBackendPlan::FacCtw { + base_depth, + encoding_bits, + msb_first, + .. + } = plan + else { + unreachable!("fac-ctw default-name kernel used with non-fac-ctw plan"); + }; + let order = if *msb_first { "msb" } else { "lsb" }; + format!("fac-ctw(d={base_depth},b={encoding_bits},{order})") +} + +pub(crate) fn rate_plan_display_label_zpaq(plan: &RateBackendPlan) -> String { + let RateBackendPlan::Zpaq { method } = plan else { + unreachable!("zpaq label kernel used with non-zpaq plan"); + }; + format!("zpaq(method={method})") +} + +pub(crate) fn rate_plan_default_name_zpaq(plan: &RateBackendPlan) -> String { + let RateBackendPlan::Zpaq { method } = plan else { + unreachable!("zpaq default-name kernel used with non-zpaq plan"); + }; + format!("zpaq(m={method})") +} + +#[cfg(feature = "backend-mamba")] +pub(crate) fn rate_plan_display_label_mamba(plan: &RateBackendPlan) -> String { + let RateBackendPlan::Mamba { method, .. } = plan else { + unreachable!("mamba label kernel used with non-mamba plan"); + }; + format!("mamba(method={method})") +} + +#[cfg(feature = "backend-mamba")] +pub(crate) fn rate_plan_default_name_mamba(plan: &RateBackendPlan) -> String { + let RateBackendPlan::Mamba { method, .. } = plan else { + unreachable!("mamba default-name kernel used with non-mamba plan"); + }; + format!("mamba({method})") +} + +#[cfg(feature = "backend-rwkv")] +pub(crate) fn rate_plan_display_label_rwkv7(plan: &RateBackendPlan) -> String { + let RateBackendPlan::Rwkv7 { method, .. } = plan else { + unreachable!("rwkv7 label kernel used with non-rwkv7 plan"); + }; + format!("rwkv7(method={method})") +} + +#[cfg(feature = "backend-rwkv")] +pub(crate) fn rate_plan_default_name_rwkv7(plan: &RateBackendPlan) -> String { + let RateBackendPlan::Rwkv7 { method, .. } = plan else { + unreachable!("rwkv7 default-name kernel used with non-rwkv7 plan"); + }; + format!("rwkv7({method})") +} + +pub(crate) fn rate_plan_display_label_mixture(plan: &RateBackendPlan) -> String { + let RateBackendPlan::Mixture { kind, .. } = plan else { + unreachable!("mixture label kernel used with non-mixture plan"); + }; + match kind { + MixtureKind::Bayes => "mixture:bayes".to_string(), + MixtureKind::FadingBayes => "mixture:fading-bayes".to_string(), + MixtureKind::Switching => "mixture:switching".to_string(), + MixtureKind::Convex => "mixture:convex".to_string(), + MixtureKind::Mdl => "mixture:mdl".to_string(), + MixtureKind::Neural => "mixture:neural".to_string(), + } +} + +pub(crate) fn rate_plan_default_name_mixture(plan: &RateBackendPlan) -> String { + let RateBackendPlan::Mixture { kind, .. } = plan else { + unreachable!("mixture default-name kernel used with non-mixture plan"); + }; + let kind = match kind { + MixtureKind::Bayes => "bayes", + MixtureKind::FadingBayes => "fading", + MixtureKind::Switching => "switch", + MixtureKind::Convex => "convex", + MixtureKind::Mdl => "mdl", + MixtureKind::Neural => "neural", + }; + format!("mix({kind})") +} + +pub(crate) fn rate_plan_display_label_particle(plan: &RateBackendPlan) -> String { + let RateBackendPlan::Particle { spec } = plan else { + unreachable!("particle label kernel used with non-particle plan"); + }; + format!( + "particle(num_particles={},num_cells={})", + spec.num_particles, spec.num_cells + ) +} + +pub(crate) fn rate_plan_default_name_particle(plan: &RateBackendPlan) -> String { + let RateBackendPlan::Particle { spec } = plan else { + unreachable!("particle default-name kernel used with non-particle plan"); + }; + format!("particle(n={},c={})", spec.num_particles, spec.num_cells) +} + +pub(crate) fn rate_plan_display_label_calibrated(plan: &RateBackendPlan) -> String { + let RateBackendPlan::Calibrated { + context, + bins, + learning_rate, + bias_clip, + .. + } = plan + else { + unreachable!("calibrated label kernel used with non-calibrated plan"); + }; + format!( + "calibrated(context={context:?},bins={bins},learning_rate={learning_rate},bias_clip={bias_clip})" + ) +} + +pub(crate) fn rate_plan_default_name_calibrated(plan: &RateBackendPlan) -> String { + let RateBackendPlan::Calibrated { base, .. } = plan else { + unreachable!("calibrated default-name kernel used with non-calibrated plan"); + }; + format!( + "calibrated({})", + rate_backend_plan_default_name(base.as_ref()) + ) +} + +pub(crate) fn compression_plan_display_label_zpaq(plan: &CompressionBackendPlan) -> String { + let CompressionBackendPlan::Zpaq { method, threads } = plan else { + unreachable!("zpaq compression label kernel used with non-zpaq plan"); + }; + format!("zpaq(method={method},threads={threads})") +} + +#[cfg(feature = "backend-rwkv")] +pub(crate) fn compression_plan_display_label_rwkv7(plan: &CompressionBackendPlan) -> String { + let CompressionBackendPlan::Rwkv7 { method, coder, .. } = plan else { + unreachable!("rwkv7 compression label kernel used with non-rwkv7 plan"); + }; + format!("rwkv7(coder={coder:?},method={method})") +} + +pub(crate) fn compression_plan_display_label_rate(plan: &CompressionBackendPlan) -> String { + let CompressionBackendPlan::Rate { + rate_backend, + coder, + framing, + } = plan + else { + unreachable!("rate compression label kernel used with non-rate plan"); + }; + format!( + "{}(coder={coder:?},framing={framing:?})", + crate::runtime::rate_backend_canonical_name(rate_backend.kind()) + ) +} + +pub(crate) fn encode_rate_payload_rosa(plan: &RateBackendPlan, out: &mut Vec) { + let RateBackendPlan::RosaPlus { max_order } = plan else { + unreachable!("rosa encoder kernel used with non-rosa plan"); + }; + out.push(0); + push_i64(out, *max_order); +} + +pub(crate) fn encode_rate_payload_match(plan: &RateBackendPlan, out: &mut Vec) { + let RateBackendPlan::Match { + hash_bits, + min_len, + max_len, + base_mix, + confidence_scale, + } = plan + else { + unreachable!("match encoder kernel used with non-match plan"); + }; + out.push(1); + push_usize(out, *hash_bits); + push_usize(out, *min_len); + push_usize(out, *max_len); + push_f64(out, *base_mix); + push_f64(out, *confidence_scale); +} + +pub(crate) fn encode_rate_payload_sparse_match(plan: &RateBackendPlan, out: &mut Vec) { + let RateBackendPlan::SparseMatch { + hash_bits, + min_len, + max_len, + gap_min, + gap_max, + base_mix, + confidence_scale, + } = plan + else { + unreachable!("sparse-match encoder kernel used with non-sparse-match plan"); + }; + out.push(2); + push_usize(out, *hash_bits); + push_usize(out, *min_len); + push_usize(out, *max_len); + push_usize(out, *gap_min); + push_usize(out, *gap_max); + push_f64(out, *base_mix); + push_f64(out, *confidence_scale); +} + +pub(crate) fn encode_rate_payload_ppmd(plan: &RateBackendPlan, out: &mut Vec) { + let RateBackendPlan::Ppmd { order, memory_mb } = plan else { + unreachable!("ppmd encoder kernel used with non-ppmd plan"); + }; + out.push(3); + push_usize(out, *order); + push_usize(out, *memory_mb); +} + +pub(crate) fn encode_rate_payload_sequitur(plan: &RateBackendPlan, out: &mut Vec) { + let RateBackendPlan::Sequitur { context_bytes } = plan else { + unreachable!("sequitur encoder kernel used with non-sequitur plan"); + }; + out.push(4); + push_usize(out, *context_bytes); +} + +pub(crate) fn encode_rate_payload_ctw(plan: &RateBackendPlan, out: &mut Vec) { + let RateBackendPlan::Ctw { depth } = plan else { + unreachable!("ctw encoder kernel used with non-ctw plan"); + }; + out.push(5); + push_usize(out, *depth); +} + +pub(crate) fn encode_rate_payload_fac_ctw(plan: &RateBackendPlan, out: &mut Vec) { + let RateBackendPlan::FacCtw { + base_depth, + num_percept_bits, + encoding_bits, + msb_first, + } = plan + else { + unreachable!("fac-ctw encoder kernel used with non-fac-ctw plan"); + }; + out.push(6); + push_usize(out, *base_depth); + push_usize(out, *num_percept_bits); + push_usize(out, *encoding_bits); + out.push(u8::from(*msb_first)); +} + +pub(crate) fn encode_rate_payload_zpaq(plan: &RateBackendPlan, out: &mut Vec) { + let RateBackendPlan::Zpaq { method } = plan else { + unreachable!("zpaq encoder kernel used with non-zpaq plan"); + }; + out.push(7); + push_string(out, method); +} + +#[cfg(feature = "backend-mamba")] +pub(crate) fn encode_rate_payload_mamba(plan: &RateBackendPlan, out: &mut Vec) { + let RateBackendPlan::Mamba { method, asset, .. } = plan else { + unreachable!("mamba encoder kernel used with non-mamba plan"); + }; + out.push(8); + push_string(out, method); + push_asset_ref(out, asset.as_ref()); +} + +#[cfg(feature = "backend-rwkv")] +pub(crate) fn encode_rate_payload_rwkv7(plan: &RateBackendPlan, out: &mut Vec) { + let RateBackendPlan::Rwkv7 { method, asset, .. } = plan else { + unreachable!("rwkv7 encoder kernel used with non-rwkv7 plan"); + }; + out.push(9); + push_string(out, method); + push_asset_ref(out, asset.as_ref()); +} + +pub(crate) fn encode_rate_payload_mixture(plan: &RateBackendPlan, out: &mut Vec) { + let RateBackendPlan::Mixture { + kind, + schedule, + alpha, + decay, + experts, + } = plan + else { + unreachable!("mixture encoder kernel used with non-mixture plan"); + }; + out.push(10); + out.push(mixture_kind_tag(*kind)); + out.push(mixture_schedule_tag(*schedule)); + push_f64(out, *alpha); + push_option_f64(out, *decay); + push_varint(out, experts.len() as u64); + for expert in experts.iter() { + push_option_string(out, expert.name.as_deref()); + push_f64(out, expert.log_prior); + encode_rate_backend_payload(expert.backend.as_ref(), out); + } +} + +pub(crate) fn encode_rate_payload_particle(plan: &RateBackendPlan, out: &mut Vec) { + let RateBackendPlan::Particle { spec } = plan else { + unreachable!("particle encoder kernel used with non-particle plan"); + }; + out.push(11); + encode_particle_spec(spec, out); +} + +pub(crate) fn encode_rate_payload_calibrated(plan: &RateBackendPlan, out: &mut Vec) { + let RateBackendPlan::Calibrated { + context, + bins, + learning_rate, + bias_clip, + base, + } = plan + else { + unreachable!("calibrated encoder kernel used with non-calibrated plan"); + }; + out.push(12); + out.push(calibration_context_tag(*context)); + push_usize(out, *bins); + push_f64(out, *learning_rate); + push_f64(out, *bias_clip); + encode_rate_backend_payload(base.as_ref(), out); +} + +pub(crate) fn encode_compression_payload_zpaq(plan: &CompressionBackendPlan, out: &mut Vec) { + let CompressionBackendPlan::Zpaq { method, threads } = plan else { + unreachable!("zpaq compression encoder kernel used with non-zpaq plan"); + }; + out.push(0); + push_string(out, method); + push_usize(out, *threads); +} + +#[cfg(feature = "backend-rwkv")] +pub(crate) fn encode_compression_payload_rwkv7(plan: &CompressionBackendPlan, out: &mut Vec) { + let CompressionBackendPlan::Rwkv7 { + method, + asset, + coder, + .. + } = plan + else { + unreachable!("rwkv7 compression encoder kernel used with non-rwkv7 plan"); + }; + out.push(1); + push_string(out, method); + push_asset_ref(out, asset.as_ref()); + out.push(coder_tag(*coder)); +} + +pub(crate) fn encode_compression_payload_rate(plan: &CompressionBackendPlan, out: &mut Vec) { + let CompressionBackendPlan::Rate { + rate_backend, + coder, + framing, + } = plan + else { + unreachable!("rate compression encoder kernel used with non-rate plan"); + }; + out.push(2); + out.push(coder_tag(*coder)); + out.push(framing_tag(*framing)); + encode_rate_backend_payload(rate_backend.as_ref(), out); +} diff --git a/crates/infotheory/src/spec/core/compile.rs b/crates/infotheory/src/spec/core/compile.rs new file mode 100644 index 00000000..d03d0fc9 --- /dev/null +++ b/crates/infotheory/src/spec/core/compile.rs @@ -0,0 +1,334 @@ +use super::*; + +pub(crate) fn compile_rate_plan_rosa( + backend: &RateBackend, + _env: &SpecEnvironment, + _depth: usize, +) -> SpecResult { + match backend { + RateBackend::RosaPlus { max_order } => Ok(RateBackendPlan::RosaPlus { + max_order: *max_order, + }), + _ => unreachable!("rosa kernel used with non-rosa backend"), + } +} + +pub(crate) fn compile_rate_plan_match( + backend: &RateBackend, + _env: &SpecEnvironment, + _depth: usize, +) -> SpecResult { + match backend { + RateBackend::Match { + hash_bits, + min_len, + max_len, + base_mix, + confidence_scale, + } => Ok(RateBackendPlan::Match { + hash_bits: *hash_bits, + min_len: *min_len, + max_len: *max_len, + base_mix: *base_mix, + confidence_scale: *confidence_scale, + }), + _ => unreachable!("match kernel used with non-match backend"), + } +} + +pub(crate) fn compile_rate_plan_sparse_match( + backend: &RateBackend, + _env: &SpecEnvironment, + _depth: usize, +) -> SpecResult { + match backend { + RateBackend::SparseMatch { + hash_bits, + min_len, + max_len, + gap_min, + gap_max, + base_mix, + confidence_scale, + } => Ok(RateBackendPlan::SparseMatch { + hash_bits: *hash_bits, + min_len: *min_len, + max_len: *max_len, + gap_min: *gap_min, + gap_max: *gap_max, + base_mix: *base_mix, + confidence_scale: *confidence_scale, + }), + _ => unreachable!("sparse-match kernel used with non-sparse-match backend"), + } +} + +pub(crate) fn compile_rate_plan_ppmd( + backend: &RateBackend, + _env: &SpecEnvironment, + _depth: usize, +) -> SpecResult { + match backend { + RateBackend::Ppmd { order, memory_mb } => Ok(RateBackendPlan::Ppmd { + order: *order, + memory_mb: *memory_mb, + }), + _ => unreachable!("ppmd kernel used with non-ppmd backend"), + } +} + +pub(crate) fn compile_rate_plan_sequitur( + backend: &RateBackend, + _env: &SpecEnvironment, + _depth: usize, +) -> SpecResult { + match backend { + RateBackend::Sequitur { context_bytes } => { + if *context_bytes < 2 { + return Err(SpecError::new("sequitur context_bytes must be >= 2")); + } + Ok(RateBackendPlan::Sequitur { + context_bytes: *context_bytes, + }) + } + _ => unreachable!("sequitur kernel used with non-sequitur backend"), + } +} + +pub(crate) fn compile_rate_plan_ctw( + backend: &RateBackend, + _env: &SpecEnvironment, + _depth: usize, +) -> SpecResult { + match backend { + RateBackend::Ctw { depth } => Ok(RateBackendPlan::Ctw { depth: *depth }), + _ => unreachable!("ctw kernel used with non-ctw backend"), + } +} + +pub(crate) fn compile_rate_plan_fac_ctw( + backend: &RateBackend, + _env: &SpecEnvironment, + _depth: usize, +) -> SpecResult { + match backend { + RateBackend::FacCtw { + base_depth, + num_percept_bits, + encoding_bits, + msb_first, + } => { + if !(1..=8).contains(encoding_bits) { + return Err(SpecError::new(format!( + "fac-ctw encoding_bits must be in 1..=8, got {encoding_bits}" + ))); + } + Ok(RateBackendPlan::FacCtw { + base_depth: *base_depth, + num_percept_bits: *num_percept_bits, + encoding_bits: *encoding_bits, + msb_first: msb_first.unwrap_or(*encoding_bits == 8), + }) + } + _ => unreachable!("fac-ctw kernel used with non-fac-ctw backend"), + } +} + +pub(crate) fn compile_rate_plan_zpaq( + backend: &RateBackend, + _env: &SpecEnvironment, + _depth: usize, +) -> SpecResult { + match backend { + RateBackend::Zpaq { method } => { + crate::validate_zpaq_rate_method(method.value()) + .map_err(|err| SpecError::new(err.to_string()))?; + Ok(RateBackendPlan::Zpaq { + method: method.value().to_string(), + }) + } + _ => unreachable!("zpaq kernel used with non-zpaq backend"), + } +} + +#[cfg(feature = "backend-mamba")] +pub(crate) fn compile_rate_plan_mamba( + backend: &RateBackend, + env: &SpecEnvironment, + _depth: usize, +) -> SpecResult { + match backend { + RateBackend::MambaMethod { method } => { + let parsed_method = + crate::spec::normalize_mamba_method_spec_for_base_dir(env.base_dir(), method)?; + let method = crate::mambazip::canonical_method_string(&parsed_method) + .map_err(|err| SpecError::new(err.to_string()))?; + Ok(RateBackendPlan::Mamba { + method, + asset: mamba_asset_ref(&parsed_method), + parsed_method, + }) + } + _ => unreachable!("mamba kernel used with non-mamba backend"), + } +} + +#[cfg(feature = "backend-rwkv")] +pub(crate) fn compile_rate_plan_rwkv7( + backend: &RateBackend, + env: &SpecEnvironment, + _depth: usize, +) -> SpecResult { + match backend { + RateBackend::Rwkv7Method { method } => { + let parsed_method = + crate::spec::normalize_rwkv_method_spec_for_base_dir(env.base_dir(), method)?; + let method = crate::rwkvzip::canonical_method_string(&parsed_method) + .map_err(|err| SpecError::new(err.to_string()))?; + Ok(RateBackendPlan::Rwkv7 { + method, + asset: rwkv_asset_ref(&parsed_method), + parsed_method, + }) + } + _ => unreachable!("rwkv7 kernel used with non-rwkv7 backend"), + } +} + +pub(crate) fn compile_rate_plan_mixture( + backend: &RateBackend, + env: &SpecEnvironment, + depth: usize, +) -> SpecResult { + let RateBackend::Mixture { spec } = backend else { + unreachable!("mixture kernel used with non-mixture backend"); + }; + let experts: Vec<_> = spec + .experts + .iter() + .map(|expert| { + Ok(RateBackendPlanExpert { + name: expert.name.clone(), + log_prior: expert.log_prior, + backend: Arc::new(build_rate_plan(&expert.backend, env, depth - 1)?), + }) + }) + .collect::>()?; + let canonical = MixtureSpec { + kind: spec.kind, + schedule: spec.schedule, + alpha: spec.alpha, + decay: spec.decay, + experts: experts + .iter() + .map(|expert| MixtureExpertSpec { + name: expert.name.clone(), + log_prior: expert.log_prior, + backend: rate_plan_to_wrapper(expert.backend.as_ref()), + }) + .collect(), + }; + canonical + .validate() + .map_err(|err| SpecError::new(err.to_string()))?; + Ok(RateBackendPlan::Mixture { + kind: canonical.kind, + schedule: canonical.schedule, + alpha: canonical.alpha, + decay: canonical.decay, + experts: experts.into_boxed_slice(), + }) +} + +pub(crate) fn compile_rate_plan_particle( + backend: &RateBackend, + _env: &SpecEnvironment, + _depth: usize, +) -> SpecResult { + let RateBackend::Particle { spec } = backend else { + unreachable!("particle kernel used with non-particle backend"); + }; + spec.validate() + .map_err(|err| SpecError::new(err.to_string()))?; + Ok(RateBackendPlan::Particle { + spec: spec.as_ref().clone(), + }) +} + +pub(crate) fn compile_rate_plan_calibrated( + backend: &RateBackend, + env: &SpecEnvironment, + depth: usize, +) -> SpecResult { + let RateBackend::Calibrated { spec } = backend else { + unreachable!("calibrated kernel used with non-calibrated backend"); + }; + Ok(RateBackendPlan::Calibrated { + context: spec.context, + bins: spec.bins, + learning_rate: spec.learning_rate, + bias_clip: spec.bias_clip, + base: Arc::new(build_rate_plan(&spec.base, env, depth - 1)?), + }) +} + +pub(crate) fn compile_compression_plan_zpaq( + backend: &CompressionBackend, + _env: &SpecEnvironment, +) -> SpecResult { + match backend { + CompressionBackend::Zpaq { method, threads } => { + crate::zpaq_compress_to_vec(&[], method.value()).map_err(|err| { + SpecError::new(format!( + "invalid zpaq compression method '{}': {err}", + method.value() + )) + })?; + Ok(CompressionBackendPlan::Zpaq { + method: method.value().to_string(), + threads: threads.get(), + }) + } + _ => unreachable!("zpaq compression kernel used with non-zpaq backend"), + } +} + +#[cfg(feature = "backend-rwkv")] +pub(crate) fn compile_compression_plan_rwkv7( + backend: &CompressionBackend, + env: &SpecEnvironment, +) -> SpecResult { + match backend { + CompressionBackend::Rwkv7 { method, coder } => { + let parsed_method = + crate::spec::normalize_rwkv_method_spec_for_base_dir(env.base_dir(), method)?; + let method = crate::rwkvzip::canonical_method_string(&parsed_method) + .map_err(|err| SpecError::new(err.to_string()))?; + Ok(CompressionBackendPlan::Rwkv7 { + method, + asset: rwkv_asset_ref(&parsed_method), + parsed_method, + coder: *coder, + }) + } + _ => unreachable!("rwkv7 compression kernel used with non-rwkv7 backend"), + } +} + +pub(crate) fn compile_compression_plan_rate( + backend: &CompressionBackend, + env: &SpecEnvironment, +) -> SpecResult { + match backend { + CompressionBackend::Rate { + rate_backend, + coder, + framing, + } => Ok(CompressionBackendPlan::Rate { + rate_backend: Arc::new(build_rate_plan(rate_backend, env, MAX_MIXTURE_NESTING)?), + coder: *coder, + framing: *framing, + }), + _ => unreachable!("rate compression kernel used with non-rate backend"), + } +} diff --git a/crates/infotheory/src/spec/document/binary.rs b/crates/infotheory/src/spec/document/binary.rs new file mode 100644 index 00000000..9ac0fc5c --- /dev/null +++ b/crates/infotheory/src/spec/document/binary.rs @@ -0,0 +1,2192 @@ +//! Binary envelope codec for canonical top-level spec documents. + +use super::*; +use crate::api::{BitOrder, BitStreamSemantics}; +use std::path::Path; +use std::sync::Arc; + +pub(super) fn encode_spec_document_payload(doc: &SpecDocument) -> Vec { + let mut out = Vec::new(); + out.extend_from_slice(DOCUMENT_MAGIC); + out.push(DOCUMENT_BINARY_VERSION); + match doc { + SpecDocument::PlannerRun(spec) => { + out.push(0); + encode_planner_run(spec, &mut out); + } + #[cfg(feature = "tuner")] + SpecDocument::Tune(spec) => { + out.push(1); + encode_tune_spec(spec, &mut out); + } + SpecDocument::RateBackend(backend) => { + out.push(2); + encode_rate_backend(&mut out, backend); + } + SpecDocument::CompressionBackend(backend) => { + out.push(3); + encode_compression_backend(&mut out, backend); + } + } + out +} + +pub(super) fn decode_spec_document(bytes: &[u8], base_dir: &Path) -> SpecResult { + let mut cursor = Cursor::new(bytes); + let magic = cursor.read_exact(4)?; + if magic != DOCUMENT_MAGIC { + return Err(SpecError::new("invalid spec document magic")); + } + let version = cursor.read_u8()?; + if version != DOCUMENT_BINARY_VERSION { + return Err(SpecError::new(format!( + "unsupported spec document binary version '{version}'" + ))); + } + let document = match cursor.read_u8()? { + 0 => Ok(SpecDocument::PlannerRun(decode_planner_run( + &mut cursor, + base_dir, + )?)), + #[cfg(feature = "tuner")] + 1 => Ok(SpecDocument::Tune(decode_tune_spec(&mut cursor, base_dir)?)), + #[cfg(not(feature = "tuner"))] + 1 => Err(SpecError::new( + "tune binary documents require infotheory built with feature 'tuner'", + )), + 2 => Ok(SpecDocument::RateBackend(decode_rate_backend( + &mut cursor, + base_dir, + )?)), + 3 => Ok(SpecDocument::CompressionBackend( + decode_compression_backend(&mut cursor, base_dir)?, + )), + tag => Err(SpecError::new(format!("unknown spec document tag '{tag}'"))), + }?; + if cursor.has_remaining() { + return Err(SpecError::new("unexpected trailing bytes in spec document")); + } + Ok(document) +} + +fn encode_planner_run(spec: &PlannerRunSpec, out: &mut Vec) { + encode_assets(&spec.assets, out); + encode_environment_spec(&spec.environment, out); + encode_interface_spec(&spec.interface, out); + encode_controller_spec(&spec.controller, out); + encode_runtime_spec(&spec.runtime, out); +} + +fn decode_planner_run(cursor: &mut Cursor<'_>, base_dir: &Path) -> SpecResult { + let spec = PlannerRunSpec { + assets: decode_assets(cursor)?, + environment: decode_environment_spec(cursor, base_dir)?, + interface: decode_interface_spec(cursor)?, + controller: decode_controller_spec(cursor, base_dir)?, + runtime: decode_runtime_spec(cursor)?, + }; + if matches!( + spec.environment, + EnvironmentSpec::Builtin { + builtin: BuiltinEnvironmentSpec::TunerBridge + } + ) { + return Err(SpecError::new( + "builtin environment 'tuner_bridge' is an internal tuner planner bridge and is not accepted in public binary planner-run documents", + )); + } + Ok(spec) +} + +#[cfg(feature = "tuner")] +fn encode_tune_spec(spec: &TuneSpec, out: &mut Vec) { + encode_assets(&spec.assets, out); + push_string(out, &spec.input_asset); + encode_compression_backend(out, &spec.baseline_candidate); + encode_tune_controller(&spec.controller, out); + encode_tune_bounds(&spec.bounds, out); + push_f64(out, spec.eval_time_limit_seconds); + push_f64(out, spec.time_budget_seconds); + push_f64(out, spec.min_throughput_bytes_per_second); + push_u64(out, spec.max_memory_bytes); + push_string(out, &spec.output_config_path); + push_u64(out, spec.seed); + push_option_string(out, spec.report_path.as_deref()); +} + +#[cfg(feature = "tuner")] +fn decode_tune_spec(cursor: &mut Cursor<'_>, base_dir: &Path) -> SpecResult { + let assets = decode_assets(cursor)?; + let input_asset = cursor.read_string()?; + Ok(TuneSpec { + assets, + input_asset, + baseline_candidate: decode_compression_backend(cursor, base_dir)?, + controller: decode_tune_controller(cursor)?, + bounds: decode_tune_bounds(cursor)?, + eval_time_limit_seconds: cursor.read_f64()?, + time_budget_seconds: cursor.read_f64()?, + min_throughput_bytes_per_second: cursor.read_f64()?, + max_memory_bytes: cursor.read_u64()?, + output_config_path: cursor.read_string()?, + seed: cursor.read_u64()?, + report_path: cursor.read_option_string()?, + }) +} + +fn encode_assets(assets: &[AssetBinding], out: &mut Vec) { + push_u64(out, assets.len() as u64); + for asset in assets { + push_string(out, &asset.id); + push_string(out, &asset.path); + } +} + +fn decode_assets(cursor: &mut Cursor<'_>) -> SpecResult> { + let len = cursor.read_u64()? as usize; + let mut assets = Vec::with_capacity(len); + for _ in 0..len { + assets.push(AssetBinding { + id: cursor.read_string()?, + path: cursor.read_string()?, + }); + } + Ok(assets) +} + +fn encode_zpaq_method_spec(out: &mut Vec, method: &crate::api::ZpaqMethodSpec) { + match method { + crate::api::ZpaqMethodSpec::Literal { value } => { + out.push(0); + push_string(out, value); + } + } +} + +fn decode_zpaq_method_spec(cursor: &mut Cursor<'_>) -> SpecResult { + match cursor.read_u8()? { + 0 => Ok(crate::api::ZpaqMethodSpec::literal(cursor.read_string()?)), + tag => Err(SpecError::new(format!( + "unknown zpaq method spec tag '{tag}'" + ))), + } +} + +#[cfg(any(feature = "backend-rwkv", feature = "backend-mamba"))] +fn encode_llm_position_expr(out: &mut Vec, expr: &crate::backends::llm_policy::PositionExpr) { + match expr { + crate::backends::llm_policy::PositionExpr::Bytes(value) => { + out.push(0); + push_u64(out, *value); + } + crate::backends::llm_policy::PositionExpr::Percent(value) => { + out.push(1); + push_f64(out, *value); + } + } +} + +#[cfg(any(feature = "backend-rwkv", feature = "backend-mamba"))] +fn decode_llm_position_expr( + cursor: &mut Cursor<'_>, +) -> SpecResult { + match cursor.read_u8()? { + 0 => Ok(crate::backends::llm_policy::PositionExpr::Bytes( + cursor.read_u64()?, + )), + 1 => Ok(crate::backends::llm_policy::PositionExpr::Percent( + cursor.read_f64()?, + )), + tag => Err(SpecError::new(format!( + "unknown llm position expr tag '{tag}'" + ))), + } +} + +#[cfg(any(feature = "backend-rwkv", feature = "backend-mamba"))] +fn encode_optimizer_kind(out: &mut Vec, kind: crate::backends::llm_policy::OptimizerKind) { + out.push(match kind { + crate::backends::llm_policy::OptimizerKind::Sgd => 0, + crate::backends::llm_policy::OptimizerKind::Adam => 1, + }); +} + +#[cfg(any(feature = "backend-rwkv", feature = "backend-mamba"))] +fn decode_optimizer_kind( + cursor: &mut Cursor<'_>, +) -> SpecResult { + match cursor.read_u8()? { + 0 => Ok(crate::backends::llm_policy::OptimizerKind::Sgd), + 1 => Ok(crate::backends::llm_policy::OptimizerKind::Adam), + tag => Err(SpecError::new(format!( + "unknown optimizer kind tag '{tag}'" + ))), + } +} + +#[cfg(any(feature = "backend-rwkv", feature = "backend-mamba"))] +fn encode_train_scope_set(out: &mut Vec, scope: &crate::backends::llm_policy::TrainScopeSet) { + push_bool(out, scope.all); + push_string_list(out, &scope.names); +} + +#[cfg(any(feature = "backend-rwkv", feature = "backend-mamba"))] +fn decode_train_scope_set( + cursor: &mut Cursor<'_>, +) -> SpecResult { + Ok(crate::backends::llm_policy::TrainScopeSet { + all: cursor.read_bool()?, + names: cursor.read_string_list()?, + }) +} + +#[cfg(any(feature = "backend-rwkv", feature = "backend-mamba"))] +fn encode_policy_action(out: &mut Vec, action: &crate::backends::llm_policy::PolicyAction) { + match action { + crate::backends::llm_policy::PolicyAction::Infer => out.push(0), + crate::backends::llm_policy::PolicyAction::Train(train) => { + out.push(1); + encode_train_scope_set(out, &train.scope); + encode_optimizer_kind(out, train.optimizer); + push_f64(out, train.hyper.lr as f64); + push_u64(out, train.hyper.stride as u64); + push_u64(out, train.hyper.bptt as u64); + push_f64(out, train.hyper.clip as f64); + push_f64(out, train.hyper.momentum as f64); + } + } +} + +#[cfg(any(feature = "backend-rwkv", feature = "backend-mamba"))] +fn decode_policy_action( + cursor: &mut Cursor<'_>, +) -> SpecResult { + match cursor.read_u8()? { + 0 => Ok(crate::backends::llm_policy::PolicyAction::Infer), + 1 => Ok(crate::backends::llm_policy::PolicyAction::Train( + crate::backends::llm_policy::TrainAction { + scope: decode_train_scope_set(cursor)?, + optimizer: decode_optimizer_kind(cursor)?, + hyper: crate::backends::llm_policy::OptimizerHyperParams { + lr: cursor.read_f64()? as f32, + stride: cursor.read_u64()? as usize, + bptt: cursor.read_u64()? as usize, + clip: cursor.read_f64()? as f32, + momentum: cursor.read_f64()? as f32, + }, + }, + )), + tag => Err(SpecError::new(format!("unknown policy action tag '{tag}'"))), + } +} + +#[cfg(any(feature = "backend-rwkv", feature = "backend-mamba"))] +fn encode_llm_policy(out: &mut Vec, policy: Option<&crate::backends::llm_policy::LlmPolicy>) { + match policy { + Some(policy) => { + out.push(1); + push_option_string( + out, + policy + .load_from + .as_ref() + .map(|path| path.to_string_lossy()) + .as_deref(), + ); + push_u64(out, policy.schedule.len() as u64); + for rule in &policy.schedule { + match rule { + crate::backends::llm_policy::ScheduleRule::Interval(rule) => { + out.push(0); + encode_llm_position_expr(out, &rule.start); + encode_llm_position_expr(out, &rule.end); + encode_policy_action(out, &rule.action); + } + crate::backends::llm_policy::ScheduleRule::Repeat(rule) => { + out.push(1); + encode_llm_position_expr(out, &rule.start); + encode_llm_position_expr(out, &rule.end); + encode_llm_position_expr(out, &rule.period); + push_u64(out, rule.pattern.len() as u64); + for segment in &rule.pattern { + encode_llm_position_expr(out, &segment.span); + encode_policy_action(out, &segment.action); + } + } + } + } + } + None => out.push(0), + } +} + +#[cfg(any(feature = "backend-rwkv", feature = "backend-mamba"))] +fn decode_llm_policy( + cursor: &mut Cursor<'_>, +) -> SpecResult> { + if cursor.read_u8()? == 0 { + return Ok(None); + } + let load_from = cursor.read_option_string()?.map(std::path::PathBuf::from); + let rule_len = cursor.read_u64()? as usize; + let mut schedule = Vec::with_capacity(rule_len); + for _ in 0..rule_len { + match cursor.read_u8()? { + 0 => schedule.push(crate::backends::llm_policy::ScheduleRule::Interval( + crate::backends::llm_policy::PolicyRule { + start: decode_llm_position_expr(cursor)?, + end: decode_llm_position_expr(cursor)?, + action: decode_policy_action(cursor)?, + }, + )), + 1 => { + let start = decode_llm_position_expr(cursor)?; + let end = decode_llm_position_expr(cursor)?; + let period = decode_llm_position_expr(cursor)?; + let pattern_len = cursor.read_u64()? as usize; + let mut pattern = Vec::with_capacity(pattern_len); + for _ in 0..pattern_len { + pattern.push(crate::backends::llm_policy::RepeatSegment { + span: decode_llm_position_expr(cursor)?, + action: decode_policy_action(cursor)?, + }); + } + schedule.push(crate::backends::llm_policy::ScheduleRule::Repeat( + crate::backends::llm_policy::RepeatRule { + start, + end, + period, + pattern, + }, + )); + } + tag => { + return Err(SpecError::new(format!( + "unknown llm policy schedule tag '{tag}'" + ))); + } + } + } + Ok(Some(crate::backends::llm_policy::LlmPolicy { + load_from, + schedule, + })) +} + +#[cfg(feature = "backend-rwkv")] +fn encode_rwkv_method_spec(out: &mut Vec, method: &crate::rwkvzip::MethodSpec) { + match method { + crate::rwkvzip::MethodSpec::File { path, policy } => { + out.push(0); + push_string(out, &path.to_string_lossy()); + encode_llm_policy(out, policy.as_ref()); + } + crate::rwkvzip::MethodSpec::Online { cfg, policy } => { + out.push(1); + push_u64(out, cfg.hidden as u64); + push_u64(out, cfg.layers as u64); + push_u64(out, cfg.intermediate as u64); + push_u64(out, cfg.decay_rank as u64); + push_u64(out, cfg.a_rank as u64); + push_u64(out, cfg.v_rank as u64); + push_u64(out, cfg.g_rank as u64); + push_u64(out, cfg.seed); + out.push(match cfg.train_mode { + crate::rwkvzip::OnlineTrainMode::None => 0, + crate::rwkvzip::OnlineTrainMode::Sgd => 1, + crate::rwkvzip::OnlineTrainMode::Adam => 2, + }); + push_f64(out, cfg.lr as f64); + push_u64(out, cfg.stride as u64); + encode_llm_policy(out, policy.as_ref()); + } + } +} + +#[cfg(feature = "backend-rwkv")] +fn decode_rwkv_method_spec(cursor: &mut Cursor<'_>) -> SpecResult { + match cursor.read_u8()? { + 0 => Ok(crate::rwkvzip::MethodSpec::File { + path: std::path::PathBuf::from(cursor.read_string()?), + policy: decode_llm_policy(cursor)?, + }), + 1 => Ok(crate::rwkvzip::MethodSpec::Online { + cfg: crate::rwkvzip::OnlineConfig { + hidden: cursor.read_u64()? as usize, + layers: cursor.read_u64()? as usize, + intermediate: cursor.read_u64()? as usize, + decay_rank: cursor.read_u64()? as usize, + a_rank: cursor.read_u64()? as usize, + v_rank: cursor.read_u64()? as usize, + g_rank: cursor.read_u64()? as usize, + seed: cursor.read_u64()?, + train_mode: match cursor.read_u8()? { + 0 => crate::rwkvzip::OnlineTrainMode::None, + 1 => crate::rwkvzip::OnlineTrainMode::Sgd, + 2 => crate::rwkvzip::OnlineTrainMode::Adam, + tag => { + return Err(SpecError::new(format!( + "unknown rwkv online train mode tag '{tag}'" + ))); + } + }, + lr: cursor.read_f64()? as f32, + stride: cursor.read_u64()? as usize, + }, + policy: decode_llm_policy(cursor)?, + }), + tag => Err(SpecError::new(format!( + "unknown rwkv method spec tag '{tag}'" + ))), + } +} + +#[cfg(feature = "backend-mamba")] +fn encode_mamba_method_spec(out: &mut Vec, method: &crate::mambazip::MethodSpec) { + match method { + crate::mambazip::MethodSpec::File { path, policy } => { + out.push(0); + push_string(out, &path.to_string_lossy()); + encode_llm_policy(out, policy.as_ref()); + } + crate::mambazip::MethodSpec::Online { cfg, policy } => { + out.push(1); + push_u64(out, cfg.hidden as u64); + push_u64(out, cfg.layers as u64); + push_u64(out, cfg.intermediate as u64); + push_u64(out, cfg.state as u64); + push_u64(out, cfg.conv as u64); + push_u64(out, cfg.dt_rank as u64); + push_u64(out, cfg.seed); + out.push(match cfg.train_mode { + crate::mambazip::OnlineTrainMode::None => 0, + crate::mambazip::OnlineTrainMode::Sgd => 1, + crate::mambazip::OnlineTrainMode::Adam => 2, + }); + push_f64(out, cfg.lr as f64); + push_u64(out, cfg.stride as u64); + encode_llm_policy(out, policy.as_ref()); + } + } +} + +#[cfg(feature = "backend-mamba")] +fn decode_mamba_method_spec(cursor: &mut Cursor<'_>) -> SpecResult { + match cursor.read_u8()? { + 0 => Ok(crate::mambazip::MethodSpec::File { + path: std::path::PathBuf::from(cursor.read_string()?), + policy: decode_llm_policy(cursor)?, + }), + 1 => Ok(crate::mambazip::MethodSpec::Online { + cfg: crate::mambazip::OnlineConfig { + hidden: cursor.read_u64()? as usize, + layers: cursor.read_u64()? as usize, + intermediate: cursor.read_u64()? as usize, + state: cursor.read_u64()? as usize, + conv: cursor.read_u64()? as usize, + dt_rank: cursor.read_u64()? as usize, + seed: cursor.read_u64()?, + train_mode: match cursor.read_u8()? { + 0 => crate::mambazip::OnlineTrainMode::None, + 1 => crate::mambazip::OnlineTrainMode::Sgd, + 2 => crate::mambazip::OnlineTrainMode::Adam, + tag => { + return Err(SpecError::new(format!( + "unknown mamba online train mode tag '{tag}'" + ))); + } + }, + lr: cursor.read_f64()? as f32, + stride: cursor.read_u64()? as usize, + }, + policy: decode_llm_policy(cursor)?, + }), + tag => Err(SpecError::new(format!( + "unknown mamba method spec tag '{tag}'" + ))), + } +} + +fn encode_rate_backend(out: &mut Vec, backend: &RateBackend) { + match backend { + RateBackend::RosaPlus { max_order } => { + out.push(0); + push_i64(out, *max_order); + } + RateBackend::Match { + hash_bits, + min_len, + max_len, + base_mix, + confidence_scale, + } => { + out.push(1); + push_u64(out, *hash_bits as u64); + push_u64(out, *min_len as u64); + push_u64(out, *max_len as u64); + push_f64(out, *base_mix); + push_f64(out, *confidence_scale); + } + RateBackend::SparseMatch { + hash_bits, + min_len, + max_len, + gap_min, + gap_max, + base_mix, + confidence_scale, + } => { + out.push(2); + push_u64(out, *hash_bits as u64); + push_u64(out, *min_len as u64); + push_u64(out, *max_len as u64); + push_u64(out, *gap_min as u64); + push_u64(out, *gap_max as u64); + push_f64(out, *base_mix); + push_f64(out, *confidence_scale); + } + RateBackend::Ppmd { order, memory_mb } => { + out.push(3); + push_u64(out, *order as u64); + push_u64(out, *memory_mb as u64); + } + RateBackend::Sequitur { context_bytes } => { + out.push(4); + push_u64(out, *context_bytes as u64); + } + RateBackend::Ctw { depth } => { + out.push(5); + push_u64(out, *depth as u64); + } + RateBackend::FacCtw { + base_depth, + num_percept_bits, + encoding_bits, + msb_first, + } => { + out.push(6); + push_u64(out, *base_depth as u64); + push_u64(out, *num_percept_bits as u64); + push_u64(out, *encoding_bits as u64); + push_option_bool(out, *msb_first); + } + RateBackend::Zpaq { method } => { + out.push(7); + encode_zpaq_method_spec(out, method); + } + #[cfg(feature = "backend-mamba")] + RateBackend::MambaMethod { method } => { + out.push(8); + encode_mamba_method_spec(out, method); + } + #[cfg(feature = "backend-rwkv")] + RateBackend::Rwkv7Method { method } => { + out.push(9); + encode_rwkv_method_spec(out, method); + } + RateBackend::Mixture { spec } => { + out.push(10); + out.push(mixture_kind_tag(spec.kind)); + out.push(mixture_schedule_tag(spec.schedule)); + push_f64(out, spec.alpha); + push_option_f64(out, spec.decay); + push_u64(out, spec.experts.len() as u64); + for expert in &spec.experts { + push_option_string(out, expert.name.as_deref()); + push_f64(out, expert.log_prior); + encode_rate_backend(out, &expert.backend); + } + } + RateBackend::Particle { spec } => { + out.push(11); + encode_particle_spec(out, spec.as_ref()); + } + RateBackend::Calibrated { spec } => { + out.push(12); + out.push(calibration_context_tag(spec.context)); + push_u64(out, spec.bins as u64); + push_f64(out, spec.learning_rate); + push_f64(out, spec.bias_clip); + encode_rate_backend(out, &spec.base); + } + } +} + +fn decode_rate_backend(cursor: &mut Cursor<'_>, base_dir: &Path) -> SpecResult { + let _ = base_dir; + match cursor.read_u8()? { + 0 => Ok(RateBackend::RosaPlus { + max_order: cursor.read_i64()?, + }), + 1 => Ok(RateBackend::Match { + hash_bits: cursor.read_u64()? as usize, + min_len: cursor.read_u64()? as usize, + max_len: cursor.read_u64()? as usize, + base_mix: cursor.read_f64()?, + confidence_scale: cursor.read_f64()?, + }), + 2 => Ok(RateBackend::SparseMatch { + hash_bits: cursor.read_u64()? as usize, + min_len: cursor.read_u64()? as usize, + max_len: cursor.read_u64()? as usize, + gap_min: cursor.read_u64()? as usize, + gap_max: cursor.read_u64()? as usize, + base_mix: cursor.read_f64()?, + confidence_scale: cursor.read_f64()?, + }), + 3 => Ok(RateBackend::Ppmd { + order: cursor.read_u64()? as usize, + memory_mb: cursor.read_u64()? as usize, + }), + 4 => Ok(RateBackend::Sequitur { + context_bytes: cursor.read_u64()? as usize, + }), + 5 => Ok(RateBackend::Ctw { + depth: cursor.read_u64()? as usize, + }), + 6 => Ok(RateBackend::FacCtw { + base_depth: cursor.read_u64()? as usize, + num_percept_bits: cursor.read_u64()? as usize, + encoding_bits: cursor.read_u64()? as usize, + msb_first: cursor.read_option_bool()?, + }), + 7 => Ok(RateBackend::Zpaq { + method: decode_zpaq_method_spec(cursor)?, + }), + #[cfg(feature = "backend-mamba")] + 8 => Ok(RateBackend::MambaMethod { + method: decode_mamba_method_spec(cursor)?, + }), + #[cfg(not(feature = "backend-mamba"))] + 8 => Err(SpecError::new( + "binary mamba backend requires the 'backend-mamba' feature", + )), + #[cfg(feature = "backend-rwkv")] + 9 => Ok(RateBackend::Rwkv7Method { + method: decode_rwkv_method_spec(cursor)?, + }), + #[cfg(not(feature = "backend-rwkv"))] + 9 => Err(SpecError::new( + "binary rwkv backend requires the 'backend-rwkv' feature", + )), + 10 => { + let kind = decode_mixture_kind(cursor.read_u8()?)?; + let schedule = decode_mixture_schedule(cursor.read_u8()?)?; + let alpha = cursor.read_f64()?; + let decay = cursor.read_option_f64()?; + let expert_len = cursor.read_u64()? as usize; + let mut experts = Vec::with_capacity(expert_len); + for _ in 0..expert_len { + experts.push(crate::api::MixtureExpertSpec { + name: cursor.read_option_string()?, + log_prior: cursor.read_f64()?, + backend: decode_rate_backend(cursor, base_dir)?, + }); + } + Ok(RateBackend::Mixture { + spec: Arc::new(crate::api::MixtureSpec { + kind, + schedule, + alpha, + decay, + experts, + }), + }) + } + 11 => Ok(RateBackend::Particle { + spec: Arc::new(decode_particle_spec(cursor)?), + }), + 12 => Ok(RateBackend::Calibrated { + spec: Arc::new(crate::api::CalibratedSpec { + context: decode_calibration_context(cursor.read_u8()?)?, + bins: cursor.read_u64()? as usize, + learning_rate: cursor.read_f64()?, + bias_clip: cursor.read_f64()?, + base: decode_rate_backend(cursor, base_dir)?, + }), + }), + tag => Err(SpecError::new(format!("unknown rate backend tag '{tag}'"))), + } +} + +fn encode_compression_backend(out: &mut Vec, backend: &CompressionBackend) { + match backend { + CompressionBackend::Zpaq { method, threads } => { + out.push(0); + encode_zpaq_method_spec(out, method); + push_u64(out, threads.get() as u64); + } + #[cfg(feature = "backend-rwkv")] + CompressionBackend::Rwkv7 { method, coder } => { + out.push(1); + encode_rwkv_method_spec(out, method); + out.push(coder_tag(*coder)); + } + CompressionBackend::Rate { + rate_backend, + coder, + framing, + } => { + out.push(2); + out.push(coder_tag(*coder)); + out.push(framing_tag(*framing)); + encode_rate_backend(out, rate_backend); + } + } +} + +fn decode_compression_backend( + cursor: &mut Cursor<'_>, + base_dir: &Path, +) -> SpecResult { + match cursor.read_u8()? { + 0 => Ok(CompressionBackend::Zpaq { + method: decode_zpaq_method_spec(cursor)?, + threads: std::num::NonZeroUsize::new( + usize::try_from(cursor.read_u64()?) + .map_err(|_| SpecError::new("zpaq compression threads exceeds usize::MAX"))?, + ) + .ok_or_else(|| SpecError::new("zpaq compression threads must be >= 1"))?, + }), + #[cfg(feature = "backend-rwkv")] + 1 => Ok(CompressionBackend::Rwkv7 { + method: decode_rwkv_method_spec(cursor)?, + coder: decode_coder(cursor.read_u8()?)?, + }), + #[cfg(not(feature = "backend-rwkv"))] + 1 => Err(SpecError::new( + "binary rwkv compression backend requires the 'backend-rwkv' feature", + )), + 2 => Ok(CompressionBackend::Rate { + coder: decode_coder(cursor.read_u8()?)?, + framing: decode_framing(cursor.read_u8()?)?, + rate_backend: decode_rate_backend(cursor, base_dir)?, + }), + tag => Err(SpecError::new(format!( + "unknown compression backend tag '{tag}'" + ))), + } +} + +fn encode_particle_spec(out: &mut Vec, spec: &crate::api::ParticleSpec) { + push_u64(out, spec.num_particles as u64); + push_u64(out, spec.context_window as u64); + push_u64(out, spec.unroll_steps as u64); + push_u64(out, spec.num_cells as u64); + push_u64(out, spec.cell_dim as u64); + push_u64(out, spec.num_rules as u64); + push_u64(out, spec.selector_hidden as u64); + push_u64(out, spec.rule_hidden as u64); + push_u64(out, spec.noise_dim as u64); + push_bool(out, spec.deterministic); + push_bool(out, spec.enable_noise); + push_f64(out, spec.noise_scale); + push_u64(out, spec.noise_anneal_steps as u64); + push_f64(out, spec.learning_rate_readout); + push_f64(out, spec.learning_rate_selector); + push_f64(out, spec.learning_rate_rule); + push_u64(out, spec.bptt_depth as u64); + push_f64(out, spec.optimizer_momentum); + push_f64(out, spec.grad_clip); + push_f64(out, spec.state_clip); + push_f64(out, spec.forget_lambda); + push_f64(out, spec.resample_threshold); + push_f64(out, spec.mutate_fraction); + push_f64(out, spec.mutate_scale); + push_bool(out, spec.mutate_model_params); + push_u64(out, spec.diagnostics_interval as u64); + push_f64(out, spec.min_prob); + push_u64(out, spec.seed); +} + +fn decode_particle_spec(cursor: &mut Cursor<'_>) -> SpecResult { + Ok(crate::api::ParticleSpec { + num_particles: cursor.read_u64()? as usize, + context_window: cursor.read_u64()? as usize, + unroll_steps: cursor.read_u64()? as usize, + num_cells: cursor.read_u64()? as usize, + cell_dim: cursor.read_u64()? as usize, + num_rules: cursor.read_u64()? as usize, + selector_hidden: cursor.read_u64()? as usize, + rule_hidden: cursor.read_u64()? as usize, + noise_dim: cursor.read_u64()? as usize, + deterministic: cursor.read_bool()?, + enable_noise: cursor.read_bool()?, + noise_scale: cursor.read_f64()?, + noise_anneal_steps: cursor.read_u64()? as usize, + learning_rate_readout: cursor.read_f64()?, + learning_rate_selector: cursor.read_f64()?, + learning_rate_rule: cursor.read_f64()?, + bptt_depth: cursor.read_u64()? as usize, + optimizer_momentum: cursor.read_f64()?, + grad_clip: cursor.read_f64()?, + state_clip: cursor.read_f64()?, + forget_lambda: cursor.read_f64()?, + resample_threshold: cursor.read_f64()?, + mutate_fraction: cursor.read_f64()?, + mutate_scale: cursor.read_f64()?, + mutate_model_params: cursor.read_bool()?, + diagnostics_interval: cursor.read_u64()? as usize, + min_prob: cursor.read_f64()?, + seed: cursor.read_u64()?, + }) +} + +fn encode_environment_spec(spec: &EnvironmentSpec, out: &mut Vec) { + match spec { + EnvironmentSpec::Builtin { builtin } => { + out.push(0); + out.push(builtin_environment_tag(*builtin)); + } + #[cfg(feature = "vm")] + EnvironmentSpec::NyxVm(vm) => { + out.push(1); + push_string(out, &vm.firecracker_config_asset); + push_string(out, &vm.instance_id); + push_string(out, &vm.shared_region_name); + push_u64(out, vm.shared_region_size as u64); + out.push(shared_memory_policy_tag(vm.shared_memory_policy)); + push_u64(out, vm.step_timeout_ms); + push_u64(out, vm.boot_timeout_ms); + push_u64(out, vm.episode_steps as u64); + push_i64(out, vm.step_cost); + out.push(vm_observation_policy_tag(vm.observation_policy)); + push_u64(out, vm.observation_bits as u64); + push_u64(out, vm.observation_stream_len as u64); + out.push(vm_observation_stream_mode_tag(vm.observation_stream_mode)); + out.push(vm.observation_pad_byte); + push_u64(out, vm.reward_bits as u64); + encode_vm_reward_policy(&vm.reward_policy, out); + match &vm.reward_shaping { + Some(shape) => { + out.push(1); + encode_vm_reward_shaping(shape, out); + } + None => out.push(0), + } + encode_vm_action_source(&vm.action_source, out); + match &vm.action_filter { + Some(filter) => { + out.push(1); + encode_vm_action_filter(filter, out); + } + None => out.push(0), + } + push_string(out, &vm.action_prefix); + push_string(out, &vm.action_suffix); + push_string(out, &vm.obs_prefix); + push_string(out, &vm.rew_prefix); + push_string(out, &vm.done_prefix); + push_string(out, &vm.data_prefix); + out.push(vm_payload_encoding_tag(vm.wire_encoding)); + encode_rate_backend(out, &vm.stats_backend); + match &vm.trace { + Some(trace) => { + out.push(1); + encode_vm_trace(trace, out); + } + None => out.push(0), + } + push_bool(out, vm.debug_mode); + push_option_string(out, vm.crash_log.as_deref()); + } + } +} + +fn decode_environment_spec( + cursor: &mut Cursor<'_>, + base_dir: &Path, +) -> SpecResult { + #[cfg(not(feature = "vm"))] + let _ = base_dir; + match cursor.read_u8()? { + 0 => Ok(EnvironmentSpec::Builtin { + builtin: decode_builtin_environment(cursor.read_u8()?)?, + }), + #[cfg(feature = "vm")] + 1 => { + let baseline = cursor.read_string()?; + let instance_id = cursor.read_string()?; + let shared_region_name = cursor.read_string()?; + let shared_region_size = cursor.read_u64()? as usize; + let shared_memory_policy = decode_shared_memory_policy(cursor.read_u8()?)?; + let step_timeout_ms = cursor.read_u64()?; + let boot_timeout_ms = cursor.read_u64()?; + let episode_steps = cursor.read_u64()? as usize; + let step_cost = cursor.read_i64()?; + let observation_policy = decode_vm_observation_policy(cursor.read_u8()?)?; + let observation_bits = cursor.read_u64()? as usize; + let observation_stream_len = cursor.read_u64()? as usize; + let observation_stream_mode = decode_vm_observation_stream_mode(cursor.read_u8()?)?; + let observation_pad_byte = cursor.read_u8()?; + let reward_bits = cursor.read_u64()? as usize; + let reward_policy = decode_vm_reward_policy(cursor)?; + let reward_shaping = if cursor.read_u8()? == 1 { + Some(decode_vm_reward_shaping(cursor)?) + } else { + None + }; + let action_source = decode_vm_action_source(cursor)?; + let action_filter = if cursor.read_u8()? == 1 { + Some(decode_vm_action_filter(cursor)?) + } else { + None + }; + let action_prefix = cursor.read_string()?; + let action_suffix = cursor.read_string()?; + let obs_prefix = cursor.read_string()?; + let rew_prefix = cursor.read_string()?; + let done_prefix = cursor.read_string()?; + let data_prefix = cursor.read_string()?; + let wire_encoding = decode_vm_payload_encoding(cursor.read_u8()?)?; + let stats_backend = decode_rate_backend(cursor, base_dir)?; + let trace = if cursor.read_u8()? == 1 { + Some(decode_vm_trace(cursor)?) + } else { + None + }; + let debug_mode = cursor.read_bool()?; + let crash_log = if cursor.has_remaining() { + cursor.read_option_string()? + } else { + None + }; + Ok(EnvironmentSpec::NyxVm(VmEnvironmentSpec { + firecracker_config_asset: baseline, + instance_id, + shared_region_name, + shared_region_size, + shared_memory_policy, + step_timeout_ms, + boot_timeout_ms, + episode_steps, + step_cost, + observation_policy, + observation_bits, + observation_stream_len, + observation_stream_mode, + observation_pad_byte, + reward_bits, + reward_policy, + reward_shaping, + action_source, + action_filter, + action_prefix, + action_suffix, + obs_prefix, + rew_prefix, + done_prefix, + data_prefix, + wire_encoding, + stats_backend, + trace, + debug_mode, + crash_log, + })) + } + #[cfg(not(feature = "vm"))] + 1 => Err(SpecError::new( + "binary nyx_vm environment requires the 'vm' feature", + )), + tag => Err(SpecError::new(format!("unknown environment tag '{tag}'"))), + } +} + +fn encode_interface_spec(spec: &PlannerInterfaceSpec, out: &mut Vec) { + push_u64(out, spec.observation_bits as u64); + push_u64(out, spec.observation_stream_len as u64); + out.push(observation_key_mode_tag(spec.observation_key_mode)); + push_u64(out, spec.reward_bits as u64); + push_u64(out, spec.agent_actions.get() as u64); +} + +fn decode_interface_spec(cursor: &mut Cursor<'_>) -> SpecResult { + let observation_bits = decode_usize_field(cursor, "interface.observation_bits")?; + let observation_stream_len = decode_usize_field(cursor, "interface.observation_stream_len")?; + let observation_key_mode = decode_observation_key_mode(cursor.read_u8()?)?; + let reward_bits = decode_usize_field(cursor, "interface.reward_bits")?; + let agent_actions_raw = decode_usize_field(cursor, "interface.agent_actions")?; + let agent_actions = crate::aixi::common::ActionAlphabet::try_from_usize(agent_actions_raw) + .map_err(|_| SpecError::new("binary interface.agent_actions must be >= 1"))?; + Ok(PlannerInterfaceSpec { + observation_bits, + observation_stream_len, + observation_key_mode, + reward_bits, + agent_actions, + }) +} + +#[cfg(feature = "tuner")] +fn encode_tune_interface_spec(spec: &TunePlannerInterfaceSpec, out: &mut Vec) { + push_u64(out, spec.observation_bits as u64); + push_u64(out, spec.observation_stream_len as u64); + out.push(observation_key_mode_tag(spec.observation_key_mode)); + push_u64(out, spec.reward_bits as u64); + push_u64(out, spec.agent_actions.get() as u64); +} + +#[cfg(feature = "tuner")] +fn decode_tune_interface_spec(cursor: &mut Cursor<'_>) -> SpecResult { + let observation_bits = decode_usize_field(cursor, "interface.observation_bits")?; + let observation_stream_len = decode_usize_field(cursor, "interface.observation_stream_len")?; + let observation_key_mode = decode_observation_key_mode(cursor.read_u8()?)?; + let reward_bits = decode_usize_field(cursor, "interface.reward_bits")?; + let agent_actions_raw = decode_usize_field(cursor, "interface.agent_actions")?; + let agent_actions = crate::aixi::common::ActionAlphabet::try_from_usize(agent_actions_raw) + .map_err(|_| SpecError::new("binary interface.agent_actions must be >= 1"))?; + Ok(TunePlannerInterfaceSpec { + observation_bits, + observation_stream_len, + observation_key_mode, + reward_bits, + agent_actions, + }) +} + +fn decode_usize_field(cursor: &mut Cursor<'_>, label: &str) -> SpecResult { + let raw = cursor.read_u64()?; + usize::try_from(raw).map_err(|_| SpecError::new(format!("{label} exceeds usize::MAX"))) +} + +fn encode_controller_spec(spec: &ControllerSpec, out: &mut Vec) { + match spec { + ControllerSpec::McAixi(inner) => { + out.push(0); + encode_rate_backend(out, &inner.predictor); + encode_bit_stream_semantics(out, inner.bit_stream_semantics); + push_u64(out, inner.agent_horizon as u64); + push_u64(out, inner.num_simulations as u64); + encode_mcts_strategy(out, inner.mcts_strategy); + push_f64(out, inner.exploration_exploitation_ratio); + push_f64(out, inner.discount_gamma); + } + ControllerSpec::AiqiDiscounted(inner) => { + out.push(1); + encode_rate_backend(out, &inner.predictor); + encode_bit_stream_semantics(out, inner.bit_stream_semantics); + push_f64(out, inner.discount_gamma); + push_u64(out, inner.return_horizon as u64); + push_u64(out, inner.return_bins as u64); + push_u64(out, inner.augmentation_period as u64); + push_option_u64(out, inner.history_prune_keep_steps.map(|n| n as u64)); + push_f64(out, inner.baseline_exploration); + } + #[cfg(feature = "aixi")] + ControllerSpec::AiqiWarmstartExactJh(inner) => { + out.push(2); + encode_rate_backend(out, &inner.predictor); + encode_bit_stream_semantics(out, inner.bit_stream_semantics); + push_u64(out, inner.return_horizon as u64); + push_u64(out, inner.return_bins as u64); + push_u64(out, inner.label_phase_period as u64); + push_string(out, &inner.teacher_dataset_asset); + push_u64(out, inner.planner_simulations_per_step as u64); + } + } +} + +fn decode_controller_spec(cursor: &mut Cursor<'_>, base_dir: &Path) -> SpecResult { + match cursor.read_u8()? { + 0 => Ok(ControllerSpec::McAixi(McAixiControllerSpec { + predictor: decode_rate_backend(cursor, base_dir)?, + bit_stream_semantics: decode_bit_stream_semantics(cursor)?, + agent_horizon: cursor.read_u64()? as usize, + num_simulations: cursor.read_u64()? as usize, + mcts_strategy: decode_mcts_strategy(cursor)?, + exploration_exploitation_ratio: cursor.read_f64()?, + discount_gamma: cursor.read_f64()?, + })), + 1 => Ok(ControllerSpec::AiqiDiscounted( + AiqiDiscountedControllerSpec { + predictor: decode_rate_backend(cursor, base_dir)?, + bit_stream_semantics: decode_bit_stream_semantics(cursor)?, + discount_gamma: cursor.read_f64()?, + return_horizon: cursor.read_u64()? as usize, + return_bins: cursor.read_u64()? as usize, + augmentation_period: cursor.read_u64()? as usize, + history_prune_keep_steps: cursor.read_option_u64()?.map(|n| n as usize), + baseline_exploration: cursor.read_f64()?, + }, + )), + #[cfg(feature = "aixi")] + 2 => Ok(ControllerSpec::AiqiWarmstartExactJh( + WarmStartExactJhControllerSpec { + predictor: decode_rate_backend(cursor, base_dir)?, + bit_stream_semantics: decode_bit_stream_semantics(cursor)?, + return_horizon: cursor.read_u64()? as usize, + return_bins: cursor.read_u64()? as usize, + label_phase_period: cursor.read_u64()? as usize, + teacher_dataset_asset: cursor.read_string()?, + planner_simulations_per_step: cursor.read_u64()? as usize, + }, + )), + #[cfg(not(feature = "aixi"))] + 2 => Err(SpecError::new( + "binary controller tag 2 (aiqi_warmstart_exact_jh) requires the 'aixi' feature", + )), + tag => Err(SpecError::new(format!("unknown controller tag '{tag}'"))), + } +} + +fn encode_bit_stream_semantics(out: &mut Vec, semantics: BitStreamSemantics) { + match semantics { + BitStreamSemantics::BytePacked { order } => { + out.push(0); + match order { + BitOrder::MsbFirst => out.push(0), + BitOrder::LsbFirst => out.push(1), + } + } + BitStreamSemantics::BinaryTokens => out.push(1), + } +} + +fn decode_bit_stream_semantics(cursor: &mut Cursor<'_>) -> SpecResult { + match cursor.read_u8()? { + 0 => { + let order = match cursor.read_u8()? { + 0 => BitOrder::MsbFirst, + 1 => BitOrder::LsbFirst, + tag => return Err(SpecError::new(format!("unknown bit order tag '{tag}'"))), + }; + Ok(BitStreamSemantics::BytePacked { order }) + } + 1 => Ok(BitStreamSemantics::BinaryTokens), + tag => Err(SpecError::new(format!( + "unknown bit stream semantics tag '{tag}'" + ))), + } +} + +fn encode_mcts_strategy(out: &mut Vec, strategy: crate::aixi::common::MctsStrategy) { + use crate::aixi::common::MctsStrategy; + match strategy { + MctsStrategy::RhoUct => out.push(0), + MctsStrategy::ParallelUct { + workers, + bu_uct_m_max, + } => { + out.push(1); + push_u64(out, workers.get() as u64); + push_option_f64(out, bu_uct_m_max); + } + } +} + +fn decode_mcts_strategy(cursor: &mut Cursor<'_>) -> SpecResult { + use crate::aixi::common::MctsStrategy; + use std::num::NonZeroUsize; + match cursor.read_u8()? { + 0 => Ok(MctsStrategy::RhoUct), + 1 => { + let workers_raw = cursor.read_u64()?; + let workers = NonZeroUsize::new(workers_raw as usize).ok_or_else(|| { + SpecError::new("binary mcts_strategy parallel_uct workers must be >= 1") + })?; + Ok(MctsStrategy::ParallelUct { + workers, + bu_uct_m_max: cursor.read_option_f64()?, + }) + } + tag => Err(SpecError::new(format!("unknown MCTS strategy tag '{tag}'"))), + } +} + +fn encode_runtime_spec(spec: &PlannerRuntimeSpec, out: &mut Vec) { + push_option_u64(out, spec.random_seed); + push_option_u64(out, spec.learn_cycles.map(|n| n as u64)); + push_option_u64(out, spec.eval_cycles.map(|n| n as u64)); + push_u64(out, spec.terminate_lifetime as u64); + push_u64(out, spec.log_every as u64); + push_bool(out, spec.perf); + push_bool(out, spec.vm_perf_only); + push_f64(out, spec.explore_epsilon); + push_f64(out, spec.explore_gamma); +} + +fn decode_runtime_spec(cursor: &mut Cursor<'_>) -> SpecResult { + Ok(PlannerRuntimeSpec { + random_seed: cursor.read_option_u64()?, + learn_cycles: cursor.read_option_u64()?.map(|n| n as usize), + eval_cycles: cursor.read_option_u64()?.map(|n| n as usize), + terminate_lifetime: cursor.read_u64()? as usize, + log_every: cursor.read_u64()? as usize, + perf: cursor.read_bool()?, + vm_perf_only: cursor.read_bool()?, + explore_epsilon: cursor.read_f64()?, + explore_gamma: cursor.read_f64()?, + }) +} + +#[cfg(feature = "tuner")] +fn encode_tune_bounds(bounds: &TuneBoundsSpec, out: &mut Vec) { + push_string_list(out, &bounds.allowed_backends); + push_string_list(out, &bounds.forbidden_backends); + push_u64(out, bounds.parameter_ranges.len() as u64); + for range in &bounds.parameter_ranges { + push_string(out, &range.parameter); + push_f64(out, range.min); + push_f64(out, range.max); + } + push_u64(out, bounds.max_experts as u64); + push_u64(out, bounds.max_mixture_nesting_depth as u64); + push_option_u64(out, bounds.min_experts.map(|n| n as u64)); + match bounds.allow_duplicate_experts { + Some(value) => { + out.push(1); + push_bool(out, value); + } + None => out.push(0), + } + push_string_list(out, &bounds.required_experts); + push_u64(out, bounds.forbidden_expert_pairs.len() as u64); + for (left, right) in &bounds.forbidden_expert_pairs { + push_string(out, left); + push_string(out, right); + } +} + +#[cfg(feature = "tuner")] +fn decode_tune_bounds(cursor: &mut Cursor<'_>) -> SpecResult { + let allowed_backends = cursor.read_string_list()?; + let forbidden_backends = cursor.read_string_list()?; + let range_len = cursor.read_u64()? as usize; + let mut parameter_ranges = Vec::with_capacity(range_len); + for _ in 0..range_len { + parameter_ranges.push(TuneParameterRangeSpec { + parameter: cursor.read_string()?, + min: cursor.read_f64()?, + max: cursor.read_f64()?, + }); + } + let max_experts = cursor.read_u64()? as usize; + let max_mixture_nesting_depth = cursor.read_u64()? as usize; + let min_experts = cursor.read_option_u64()?.map(|n| n as usize); + let allow_duplicate_experts = if cursor.read_u8()? == 1 { + Some(cursor.read_bool()?) + } else { + None + }; + let required_experts = cursor.read_string_list()?; + let pair_len = cursor.read_u64()? as usize; + let mut forbidden_expert_pairs = Vec::with_capacity(pair_len); + for _ in 0..pair_len { + forbidden_expert_pairs.push((cursor.read_string()?, cursor.read_string()?)); + } + Ok(TuneBoundsSpec { + allowed_backends, + forbidden_backends, + parameter_ranges, + max_experts, + max_mixture_nesting_depth, + min_experts, + allow_duplicate_experts, + required_experts, + forbidden_expert_pairs, + }) +} + +#[cfg(feature = "tuner")] +fn encode_tune_controller(spec: &TuneControllerSpec, out: &mut Vec) { + match spec { + TuneControllerSpec::AnnealedHillClimbing(inner) => { + out.push(tune_controller_kind_tag( + TuneControllerKind::AnnealedHillClimbing, + )); + push_u64(out, inner.max_mutation_radius as u64); + } + TuneControllerSpec::McAixiFacCtw(inner) => { + out.push(tune_controller_kind_tag(TuneControllerKind::McAixiFacCtw)); + encode_tune_interface_spec(&inner.interface, out); + push_u64(out, inner.planner_simulations_per_step as u64); + } + TuneControllerSpec::AiqiDiscounted(inner) => { + out.push(tune_controller_kind_tag(TuneControllerKind::AiqiDiscounted)); + encode_tune_interface_spec(&inner.interface, out); + push_u64(out, inner.planner_simulations_per_step as u64); + push_u64(out, inner.return_horizon as u64); + push_u64(out, inner.return_bins as u64); + push_f64(out, inner.discount_factor); + push_f64(out, inner.min_improvement); + push_f64(out, inner.max_improvement); + } + #[cfg(feature = "aixi")] + TuneControllerSpec::AiqiWarmstartExactJh(inner) => { + out.push(tune_controller_kind_tag( + TuneControllerKind::AiqiWarmstartExactJh, + )); + encode_tune_interface_spec(&inner.interface, out); + push_u64(out, inner.planner_simulations_per_step as u64); + push_u64(out, inner.return_horizon as u64); + push_string(out, &inner.warmstart_teacher_dataset_asset); + push_u64(out, inner.label_phase_period as u64); + } + } +} + +#[cfg(feature = "tuner")] +fn decode_tune_controller(cursor: &mut Cursor<'_>) -> SpecResult { + match decode_tune_controller_kind(cursor.read_u8()?)? { + TuneControllerKind::AnnealedHillClimbing => Ok(TuneControllerSpec::AnnealedHillClimbing( + AnnealedHillClimbingTuneControllerSpec { + max_mutation_radius: cursor.read_u64()? as usize, + }, + )), + TuneControllerKind::McAixiFacCtw => Ok(TuneControllerSpec::McAixiFacCtw( + McAixiFacCtwTuneControllerSpec { + interface: decode_tune_interface_spec(cursor)?, + planner_simulations_per_step: cursor.read_u64()? as usize, + }, + )), + TuneControllerKind::AiqiDiscounted => Ok(TuneControllerSpec::AiqiDiscounted( + AiqiDiscountedTuneControllerSpec { + interface: decode_tune_interface_spec(cursor)?, + planner_simulations_per_step: cursor.read_u64()? as usize, + return_horizon: cursor.read_u64()? as usize, + return_bins: cursor.read_u64()? as usize, + discount_factor: cursor.read_f64()?, + min_improvement: cursor.read_f64()?, + max_improvement: cursor.read_f64()?, + }, + )), + #[cfg(feature = "aixi")] + TuneControllerKind::AiqiWarmstartExactJh => Ok(TuneControllerSpec::AiqiWarmstartExactJh( + WarmStartExactJhTuneControllerSpec { + interface: decode_tune_interface_spec(cursor)?, + planner_simulations_per_step: cursor.read_u64()? as usize, + return_horizon: cursor.read_u64()? as usize, + warmstart_teacher_dataset_asset: cursor.read_string()?, + label_phase_period: cursor.read_u64()? as usize, + }, + )), + } +} + +#[cfg(feature = "vm")] +fn encode_vm_reward_policy(policy: &VmRewardPolicySpec, out: &mut Vec) { + match policy { + VmRewardPolicySpec::FromGuest => out.push(0), + VmRewardPolicySpec::Pattern { + pattern, + base_reward, + bonus_reward, + } => { + out.push(1); + push_string(out, pattern); + push_i64(out, *base_reward); + push_i64(out, *bonus_reward); + } + } +} + +#[cfg(feature = "vm")] +fn decode_vm_reward_policy(cursor: &mut Cursor<'_>) -> SpecResult { + match cursor.read_u8()? { + 0 => Ok(VmRewardPolicySpec::FromGuest), + 1 => Ok(VmRewardPolicySpec::Pattern { + pattern: cursor.read_string()?, + base_reward: cursor.read_i64()?, + bonus_reward: cursor.read_i64()?, + }), + tag => Err(SpecError::new(format!( + "unknown vm reward policy tag '{tag}'" + ))), + } +} + +#[cfg(feature = "vm")] +fn encode_vm_reward_shaping(spec: &VmRewardShapingSpec, out: &mut Vec) { + match spec { + VmRewardShapingSpec::EntropyReduction { + baseline_asset, + scale, + crash_bonus, + timeout_bonus, + } => { + out.push(0); + push_string(out, baseline_asset); + push_f64(out, *scale); + push_option_i64(out, *crash_bonus); + push_option_i64(out, *timeout_bonus); + } + VmRewardShapingSpec::TraceEntropy { scale, normalize } => { + out.push(1); + push_f64(out, *scale); + push_bool(out, *normalize); + } + } +} + +#[cfg(feature = "vm")] +fn decode_vm_reward_shaping(cursor: &mut Cursor<'_>) -> SpecResult { + match cursor.read_u8()? { + 0 => Ok(VmRewardShapingSpec::EntropyReduction { + baseline_asset: cursor.read_string()?, + scale: cursor.read_f64()?, + crash_bonus: cursor.read_option_i64()?, + timeout_bonus: cursor.read_option_i64()?, + }), + 1 => Ok(VmRewardShapingSpec::TraceEntropy { + scale: cursor.read_f64()?, + normalize: cursor.read_bool()?, + }), + tag => Err(SpecError::new(format!( + "unknown vm reward shaping tag '{tag}'" + ))), + } +} + +#[cfg(feature = "vm")] +fn encode_vm_action_source(spec: &VmRuntimeActionSourceSpec, out: &mut Vec) { + match spec { + VmRuntimeActionSourceSpec::Literal { + names, + payloads, + encoding, + } => { + out.push(0); + out.push(vm_payload_encoding_tag(*encoding)); + push_u64(out, payloads.len() as u64); + for (idx, payload) in payloads.iter().enumerate() { + push_option_string(out, names.get(idx).cloned().flatten().as_deref()); + push_string(out, payload); + } + } + VmRuntimeActionSourceSpec::Fuzz { + seeds, + encoding, + mutators, + min_len, + max_len, + dictionary, + rng_seed, + } => { + out.push(1); + out.push(vm_payload_encoding_tag(*encoding)); + push_string_list(out, seeds); + push_u64(out, mutators.len() as u64); + for mutator in mutators { + out.push(vm_fuzz_mutator_tag(*mutator)); + } + push_u64(out, *min_len as u64); + push_u64(out, *max_len as u64); + push_string_list(out, dictionary); + push_u64(out, *rng_seed); + } + } +} + +#[cfg(feature = "vm")] +fn decode_vm_action_source(cursor: &mut Cursor<'_>) -> SpecResult { + match cursor.read_u8()? { + 0 => { + let encoding = decode_vm_payload_encoding(cursor.read_u8()?)?; + let len = cursor.read_u64()? as usize; + let mut names = Vec::with_capacity(len); + let mut payloads = Vec::with_capacity(len); + for _ in 0..len { + names.push(cursor.read_option_string()?); + payloads.push(cursor.read_string()?); + } + Ok(VmRuntimeActionSourceSpec::Literal { + names, + payloads, + encoding, + }) + } + 1 => { + let encoding = decode_vm_payload_encoding(cursor.read_u8()?)?; + let seeds = cursor.read_string_list()?; + let mutators_len = cursor.read_u64()? as usize; + let mut mutators = Vec::with_capacity(mutators_len); + for _ in 0..mutators_len { + mutators.push(decode_vm_fuzz_mutator(cursor.read_u8()?)?); + } + Ok(VmRuntimeActionSourceSpec::Fuzz { + seeds, + encoding, + mutators, + min_len: cursor.read_u64()? as usize, + max_len: cursor.read_u64()? as usize, + dictionary: cursor.read_string_list()?, + rng_seed: cursor.read_u64()?, + }) + } + tag => Err(SpecError::new(format!( + "unknown vm action source tag '{tag}'" + ))), + } +} + +#[cfg(feature = "vm")] +fn encode_vm_action_filter(spec: &VmActionFilterSpec, out: &mut Vec) { + push_option_f64(out, spec.min_entropy); + push_option_f64(out, spec.max_entropy); + push_option_f64(out, spec.min_intrinsic_dependence); + push_option_f64(out, spec.min_novelty); + push_option_string(out, spec.novelty_prior_asset.as_deref()); + push_option_i64(out, spec.reject_reward); +} + +#[cfg(feature = "vm")] +fn decode_vm_action_filter(cursor: &mut Cursor<'_>) -> SpecResult { + Ok(VmActionFilterSpec { + min_entropy: cursor.read_option_f64()?, + max_entropy: cursor.read_option_f64()?, + min_intrinsic_dependence: cursor.read_option_f64()?, + min_novelty: cursor.read_option_f64()?, + novelty_prior_asset: cursor.read_option_string()?, + reject_reward: cursor.read_option_i64()?, + }) +} + +#[cfg(feature = "vm")] +fn encode_vm_trace(spec: &VmTraceSpec, out: &mut Vec) { + push_option_string(out, spec.shared_region_name.as_deref()); + push_u64(out, spec.max_bytes as u64); + push_bool(out, spec.reset_on_episode); +} + +#[cfg(feature = "vm")] +fn decode_vm_trace(cursor: &mut Cursor<'_>) -> SpecResult { + Ok(VmTraceSpec { + shared_region_name: cursor.read_option_string()?, + max_bytes: cursor.read_u64()? as usize, + reset_on_episode: cursor.read_bool()?, + }) +} + +pub(super) fn builtin_environment_name(env: BuiltinEnvironmentSpec) -> &'static str { + match env { + BuiltinEnvironmentSpec::TunerBridge => "tuner_bridge", + BuiltinEnvironmentSpec::CoinFlip => "coin_flip", + BuiltinEnvironmentSpec::BiasedRockPaperScissor => "biased_rock_paper_scissor", + BuiltinEnvironmentSpec::KuhnPoker => "kuhn_poker", + BuiltinEnvironmentSpec::ExtendedTiger => "extended_tiger", + BuiltinEnvironmentSpec::TicTacToe => "tic_tac_toe", + BuiltinEnvironmentSpec::Blackjack => "blackjack", + BuiltinEnvironmentSpec::Platformer => "platformer", + } +} + +fn coder_tag(coder: crate::coders::CoderType) -> u8 { + match coder { + crate::coders::CoderType::AC => 0, + crate::coders::CoderType::RANS => 1, + } +} + +fn decode_coder(tag: u8) -> SpecResult { + match tag { + 0 => Ok(crate::coders::CoderType::AC), + 1 => Ok(crate::coders::CoderType::RANS), + _ => Err(SpecError::new(format!("unknown coder tag '{tag}'"))), + } +} + +fn framing_tag(framing: crate::compression::FramingMode) -> u8 { + match framing { + crate::compression::FramingMode::Raw => 0, + crate::compression::FramingMode::Framed => 1, + } +} + +fn decode_framing(tag: u8) -> SpecResult { + match tag { + 0 => Ok(crate::compression::FramingMode::Raw), + 1 => Ok(crate::compression::FramingMode::Framed), + _ => Err(SpecError::new(format!("unknown framing tag '{tag}'"))), + } +} + +fn mixture_kind_tag(kind: crate::api::MixtureKind) -> u8 { + match kind { + crate::api::MixtureKind::Bayes => 0, + crate::api::MixtureKind::FadingBayes => 1, + crate::api::MixtureKind::Switching => 2, + crate::api::MixtureKind::Convex => 3, + crate::api::MixtureKind::Mdl => 4, + crate::api::MixtureKind::Neural => 5, + } +} + +fn decode_mixture_kind(tag: u8) -> SpecResult { + match tag { + 0 => Ok(crate::api::MixtureKind::Bayes), + 1 => Ok(crate::api::MixtureKind::FadingBayes), + 2 => Ok(crate::api::MixtureKind::Switching), + 3 => Ok(crate::api::MixtureKind::Convex), + 4 => Ok(crate::api::MixtureKind::Mdl), + 5 => Ok(crate::api::MixtureKind::Neural), + _ => Err(SpecError::new(format!("unknown mixture kind tag '{tag}'"))), + } +} + +fn mixture_schedule_tag(schedule: crate::api::MixtureScheduleMode) -> u8 { + match schedule { + crate::api::MixtureScheduleMode::Default => 0, + crate::api::MixtureScheduleMode::Theorem => 1, + } +} + +fn decode_mixture_schedule(tag: u8) -> SpecResult { + match tag { + 0 => Ok(crate::api::MixtureScheduleMode::Default), + 1 => Ok(crate::api::MixtureScheduleMode::Theorem), + _ => Err(SpecError::new(format!( + "unknown mixture schedule tag '{tag}'" + ))), + } +} + +fn calibration_context_tag(context: crate::api::CalibrationContextKind) -> u8 { + match context { + crate::api::CalibrationContextKind::Global => 0, + crate::api::CalibrationContextKind::ByteClass => 1, + crate::api::CalibrationContextKind::Text => 2, + crate::api::CalibrationContextKind::Repeat => 3, + crate::api::CalibrationContextKind::TextRepeat => 4, + } +} + +fn decode_calibration_context(tag: u8) -> SpecResult { + match tag { + 0 => Ok(crate::api::CalibrationContextKind::Global), + 1 => Ok(crate::api::CalibrationContextKind::ByteClass), + 2 => Ok(crate::api::CalibrationContextKind::Text), + 3 => Ok(crate::api::CalibrationContextKind::Repeat), + 4 => Ok(crate::api::CalibrationContextKind::TextRepeat), + _ => Err(SpecError::new(format!( + "unknown calibration context tag '{tag}'" + ))), + } +} + +fn builtin_environment_tag(env: BuiltinEnvironmentSpec) -> u8 { + match env { + BuiltinEnvironmentSpec::TunerBridge => 8, + BuiltinEnvironmentSpec::CoinFlip => 0, + BuiltinEnvironmentSpec::ExtendedTiger => 2, + BuiltinEnvironmentSpec::TicTacToe => 3, + BuiltinEnvironmentSpec::BiasedRockPaperScissor => 4, + BuiltinEnvironmentSpec::KuhnPoker => 5, + BuiltinEnvironmentSpec::Blackjack => 6, + BuiltinEnvironmentSpec::Platformer => 7, + } +} + +fn decode_builtin_environment(tag: u8) -> SpecResult { + match tag { + 8 => Ok(BuiltinEnvironmentSpec::TunerBridge), + 0 => Ok(BuiltinEnvironmentSpec::CoinFlip), + 1 => Err(SpecError::new( + "builtin environment tag '1' (ctw_test) is no longer supported", + )), + 2 => Ok(BuiltinEnvironmentSpec::ExtendedTiger), + 3 => Ok(BuiltinEnvironmentSpec::TicTacToe), + 4 => Ok(BuiltinEnvironmentSpec::BiasedRockPaperScissor), + 5 => Ok(BuiltinEnvironmentSpec::KuhnPoker), + 6 => Ok(BuiltinEnvironmentSpec::Blackjack), + 7 => Ok(BuiltinEnvironmentSpec::Platformer), + _ => Err(SpecError::new(format!( + "unknown builtin environment tag '{tag}'" + ))), + } +} + +#[cfg(feature = "vm")] +pub(super) fn shared_memory_policy_name(policy: SharedMemoryPolicySpec) -> &'static str { + match policy { + SharedMemoryPolicySpec::Preserve => "preserve", + SharedMemoryPolicySpec::Snapshot => "snapshot", + } +} + +#[cfg(feature = "vm")] +fn shared_memory_policy_tag(policy: SharedMemoryPolicySpec) -> u8 { + match policy { + SharedMemoryPolicySpec::Preserve => 0, + SharedMemoryPolicySpec::Snapshot => 1, + } +} + +#[cfg(feature = "vm")] +fn decode_shared_memory_policy(tag: u8) -> SpecResult { + match tag { + 0 => Ok(SharedMemoryPolicySpec::Preserve), + 1 => Ok(SharedMemoryPolicySpec::Snapshot), + _ => Err(SpecError::new(format!( + "unknown shared memory policy tag '{tag}'" + ))), + } +} + +#[cfg(feature = "vm")] +pub(super) fn vm_observation_policy_name(policy: VmObservationPolicySpec) -> &'static str { + match policy { + VmObservationPolicySpec::FromGuest => "from_guest", + VmObservationPolicySpec::OutputHash => "output_hash", + VmObservationPolicySpec::RawOutput => "raw_output", + VmObservationPolicySpec::SharedMemory => "shared_memory", + } +} + +#[cfg(feature = "vm")] +fn vm_observation_policy_tag(policy: VmObservationPolicySpec) -> u8 { + match policy { + VmObservationPolicySpec::FromGuest => 0, + VmObservationPolicySpec::OutputHash => 1, + VmObservationPolicySpec::RawOutput => 2, + VmObservationPolicySpec::SharedMemory => 3, + } +} + +#[cfg(feature = "vm")] +fn decode_vm_observation_policy(tag: u8) -> SpecResult { + match tag { + 0 => Ok(VmObservationPolicySpec::FromGuest), + 1 => Ok(VmObservationPolicySpec::OutputHash), + 2 => Ok(VmObservationPolicySpec::RawOutput), + 3 => Ok(VmObservationPolicySpec::SharedMemory), + _ => Err(SpecError::new(format!( + "unknown VM observation policy tag '{tag}'" + ))), + } +} + +#[cfg(feature = "vm")] +pub(super) fn vm_observation_stream_mode_name(mode: VmObservationStreamModeSpec) -> &'static str { + match mode { + VmObservationStreamModeSpec::PadTruncate => "pad_truncate", + VmObservationStreamModeSpec::Pad => "pad", + VmObservationStreamModeSpec::Truncate => "truncate", + } +} + +#[cfg(feature = "vm")] +fn vm_observation_stream_mode_tag(mode: VmObservationStreamModeSpec) -> u8 { + match mode { + VmObservationStreamModeSpec::PadTruncate => 0, + VmObservationStreamModeSpec::Pad => 1, + VmObservationStreamModeSpec::Truncate => 2, + } +} + +#[cfg(feature = "vm")] +fn decode_vm_observation_stream_mode(tag: u8) -> SpecResult { + match tag { + 0 => Ok(VmObservationStreamModeSpec::PadTruncate), + 1 => Ok(VmObservationStreamModeSpec::Pad), + 2 => Ok(VmObservationStreamModeSpec::Truncate), + _ => Err(SpecError::new(format!( + "unknown VM observation stream mode tag '{tag}'" + ))), + } +} + +#[cfg(feature = "vm")] +pub(super) fn vm_payload_encoding_name(encoding: VmPayloadEncodingSpec) -> &'static str { + match encoding { + VmPayloadEncodingSpec::Utf8 => "utf8", + VmPayloadEncodingSpec::Hex => "hex", + } +} + +#[cfg(feature = "vm")] +fn vm_payload_encoding_tag(encoding: VmPayloadEncodingSpec) -> u8 { + match encoding { + VmPayloadEncodingSpec::Utf8 => 0, + VmPayloadEncodingSpec::Hex => 1, + } +} + +#[cfg(feature = "vm")] +fn decode_vm_payload_encoding(tag: u8) -> SpecResult { + match tag { + 0 => Ok(VmPayloadEncodingSpec::Utf8), + 1 => Ok(VmPayloadEncodingSpec::Hex), + _ => Err(SpecError::new(format!( + "unknown VM payload encoding tag '{tag}'" + ))), + } +} + +#[cfg(feature = "vm")] +pub(super) fn vm_fuzz_mutator_name(mutator: VmFuzzMutatorSpec) -> &'static str { + match mutator { + VmFuzzMutatorSpec::FlipBit => "flip_bit", + VmFuzzMutatorSpec::FlipByte => "flip_byte", + VmFuzzMutatorSpec::InsertByte => "insert_byte", + VmFuzzMutatorSpec::DeleteByte => "delete_byte", + VmFuzzMutatorSpec::SpliceSeed => "splice_seed", + VmFuzzMutatorSpec::ResetSeed => "reset_seed", + VmFuzzMutatorSpec::Havoc => "havoc", + } +} + +#[cfg(feature = "vm")] +fn vm_fuzz_mutator_tag(mutator: VmFuzzMutatorSpec) -> u8 { + match mutator { + VmFuzzMutatorSpec::FlipBit => 0, + VmFuzzMutatorSpec::FlipByte => 1, + VmFuzzMutatorSpec::InsertByte => 2, + VmFuzzMutatorSpec::DeleteByte => 3, + VmFuzzMutatorSpec::SpliceSeed => 4, + VmFuzzMutatorSpec::ResetSeed => 5, + VmFuzzMutatorSpec::Havoc => 6, + } +} + +#[cfg(feature = "vm")] +fn decode_vm_fuzz_mutator(tag: u8) -> SpecResult { + match tag { + 0 => Ok(VmFuzzMutatorSpec::FlipBit), + 1 => Ok(VmFuzzMutatorSpec::FlipByte), + 2 => Ok(VmFuzzMutatorSpec::InsertByte), + 3 => Ok(VmFuzzMutatorSpec::DeleteByte), + 4 => Ok(VmFuzzMutatorSpec::SpliceSeed), + 5 => Ok(VmFuzzMutatorSpec::ResetSeed), + 6 => Ok(VmFuzzMutatorSpec::Havoc), + _ => Err(SpecError::new(format!( + "unknown VM fuzz mutator tag '{tag}'" + ))), + } +} + +pub(super) fn observation_key_mode_name(mode: ObservationKeyMode) -> &'static str { + match mode { + ObservationKeyMode::First => "first", + ObservationKeyMode::Last => "last", + ObservationKeyMode::StreamHash => "stream_hash", + ObservationKeyMode::FullStream => "full_stream", + } +} + +fn observation_key_mode_tag(mode: ObservationKeyMode) -> u8 { + match mode { + ObservationKeyMode::First => 0, + ObservationKeyMode::Last => 1, + ObservationKeyMode::StreamHash => 2, + ObservationKeyMode::FullStream => 3, + } +} + +fn decode_observation_key_mode(tag: u8) -> SpecResult { + match tag { + 0 => Ok(ObservationKeyMode::First), + 1 => Ok(ObservationKeyMode::Last), + 2 => Ok(ObservationKeyMode::StreamHash), + 3 => Ok(ObservationKeyMode::FullStream), + _ => Err(SpecError::new(format!( + "unknown observation key mode tag '{tag}'" + ))), + } +} + +#[cfg(feature = "tuner")] +fn tune_controller_kind_tag(kind: TuneControllerKind) -> u8 { + match kind { + TuneControllerKind::AnnealedHillClimbing => 0, + TuneControllerKind::McAixiFacCtw => 1, + TuneControllerKind::AiqiDiscounted => 2, + #[cfg(feature = "aixi")] + TuneControllerKind::AiqiWarmstartExactJh => 3, + } +} + +#[cfg(feature = "tuner")] +fn decode_tune_controller_kind(tag: u8) -> SpecResult { + match tag { + 0 => Ok(TuneControllerKind::AnnealedHillClimbing), + 1 => Ok(TuneControllerKind::McAixiFacCtw), + 2 => Ok(TuneControllerKind::AiqiDiscounted), + #[cfg(feature = "aixi")] + 3 => Ok(TuneControllerKind::AiqiWarmstartExactJh), + #[cfg(not(feature = "aixi"))] + 3 => Err(SpecError::new( + "binary tune controller tag 3 (aiqi_warmstart_exact_jh) requires the 'aixi' feature", + )), + _ => Err(SpecError::new(format!( + "unknown tune controller tag '{tag}'" + ))), + } +} + +fn push_bool(out: &mut Vec, value: bool) { + out.push(u8::from(value)); +} + +fn push_u64(out: &mut Vec, mut value: u64) { + while value >= 0x80 { + out.push((value as u8) | 0x80); + value >>= 7; + } + out.push(value as u8); +} + +fn push_i64(out: &mut Vec, value: i64) { + push_u64(out, ((value << 1) ^ (value >> 63)) as u64); +} + +fn push_f64(out: &mut Vec, value: f64) { + out.extend_from_slice(&value.to_le_bytes()); +} + +fn push_string(out: &mut Vec, value: &str) { + push_u64(out, value.len() as u64); + out.extend_from_slice(value.as_bytes()); +} + +fn push_option_string(out: &mut Vec, value: Option<&str>) { + match value { + Some(value) => { + out.push(1); + push_string(out, value); + } + None => out.push(0), + } +} + +fn push_option_u64(out: &mut Vec, value: Option) { + match value { + Some(value) => { + out.push(1); + push_u64(out, value); + } + None => out.push(0), + } +} + +fn push_option_bool(out: &mut Vec, value: Option) { + match value { + Some(value) => { + out.push(1); + push_bool(out, value); + } + None => out.push(0), + } +} + +#[cfg(feature = "vm")] +fn push_option_i64(out: &mut Vec, value: Option) { + match value { + Some(value) => { + out.push(1); + push_i64(out, value); + } + None => out.push(0), + } +} + +fn push_option_f64(out: &mut Vec, value: Option) { + match value { + Some(value) => { + out.push(1); + push_f64(out, value); + } + None => out.push(0), + } +} + +#[cfg(any( + feature = "tuner", + feature = "vm", + feature = "backend-rwkv", + feature = "backend-mamba" +))] +fn push_string_list(out: &mut Vec, items: &[String]) { + push_u64(out, items.len() as u64); + for item in items { + push_string(out, item); + } +} + +struct Cursor<'a> { + bytes: &'a [u8], + pos: usize, +} + +impl<'a> Cursor<'a> { + fn new(bytes: &'a [u8]) -> Self { + Self { bytes, pos: 0 } + } + + fn read_exact(&mut self, len: usize) -> SpecResult<&'a [u8]> { + if self.pos + len > self.bytes.len() { + return Err(SpecError::new("unexpected end of spec document")); + } + let start = self.pos; + self.pos += len; + Ok(&self.bytes[start..self.pos]) + } + + fn read_u8(&mut self) -> SpecResult { + Ok(self.read_exact(1)?[0]) + } + + fn read_bool(&mut self) -> SpecResult { + Ok(self.read_u8()? != 0) + } + + fn read_u64(&mut self) -> SpecResult { + let mut shift = 0u32; + let mut out = 0u64; + loop { + let byte = self.read_u8()?; + out |= ((byte & 0x7f) as u64) << shift; + if byte & 0x80 == 0 { + return Ok(out); + } + shift += 7; + if shift > 63 { + return Err(SpecError::new("invalid varint in spec document")); + } + } + } + + fn read_i64(&mut self) -> SpecResult { + let value = self.read_u64()?; + Ok(((value >> 1) as i64) ^ (-((value & 1) as i64))) + } + + fn read_f64(&mut self) -> SpecResult { + let mut bytes = [0u8; 8]; + bytes.copy_from_slice(self.read_exact(8)?); + Ok(f64::from_le_bytes(bytes)) + } + + fn read_string(&mut self) -> SpecResult { + let len = self.read_u64()? as usize; + let bytes = self.read_exact(len)?; + String::from_utf8(bytes.to_vec()).map_err(|err| SpecError::new(err.to_string())) + } + + fn read_option_string(&mut self) -> SpecResult> { + if self.read_u8()? == 1 { + Ok(Some(self.read_string()?)) + } else { + Ok(None) + } + } + + fn read_option_u64(&mut self) -> SpecResult> { + if self.read_u8()? == 1 { + Ok(Some(self.read_u64()?)) + } else { + Ok(None) + } + } + + fn read_option_bool(&mut self) -> SpecResult> { + if self.read_u8()? == 1 { + Ok(Some(self.read_bool()?)) + } else { + Ok(None) + } + } + + #[cfg(feature = "vm")] + fn read_option_i64(&mut self) -> SpecResult> { + if self.read_u8()? == 1 { + Ok(Some(self.read_i64()?)) + } else { + Ok(None) + } + } + + fn read_option_f64(&mut self) -> SpecResult> { + if self.read_u8()? == 1 { + Ok(Some(self.read_f64()?)) + } else { + Ok(None) + } + } + + #[cfg(any( + feature = "tuner", + feature = "vm", + feature = "backend-rwkv", + feature = "backend-mamba" + ))] + fn read_string_list(&mut self) -> SpecResult> { + let len = self.read_u64()? as usize; + let mut items = Vec::with_capacity(len); + for _ in 0..len { + items.push(self.read_string()?); + } + Ok(items) + } + + fn has_remaining(&self) -> bool { + self.pos < self.bytes.len() + } +} + +#[cfg(all(test, feature = "tuner"))] +mod tests { + use super::*; + + fn encode_tune_interface( + observation_bits: u64, + observation_stream_len: u64, + reward_bits: u64, + agent_actions: u64, + ) -> Vec { + let mut bytes = Vec::new(); + push_u64(&mut bytes, observation_bits); + push_u64(&mut bytes, observation_stream_len); + bytes.push(observation_key_mode_tag(ObservationKeyMode::FullStream)); + push_u64(&mut bytes, reward_bits); + push_u64(&mut bytes, agent_actions); + bytes + } + + #[test] + fn decode_tune_interface_accepts_platform_usize_max() { + let platform_max: u64 = usize::MAX as u64; + let bytes = encode_tune_interface(platform_max, platform_max, platform_max, 2); + let mut cursor = Cursor::new(&bytes); + + let decoded = decode_tune_interface_spec(&mut cursor).expect("decode tune interface"); + + assert_eq!(decoded.observation_bits, usize::MAX); + assert_eq!(decoded.observation_stream_len, usize::MAX); + assert_eq!(decoded.reward_bits, usize::MAX); + assert_eq!(decoded.agent_actions.get(), 2); + } + + #[cfg(target_pointer_width = "32")] + #[test] + fn decode_tune_interface_rejects_observation_bits_exceeding_usize() { + let overflow: u64 = (u32::MAX as u64) + 1; + let bytes = encode_tune_interface(overflow, 1, 1, 2); + let mut cursor = Cursor::new(&bytes); + + let err = decode_tune_interface_spec(&mut cursor) + .expect_err("overflowing observation_bits must be rejected"); + + assert!( + err.to_string() + .contains("interface.observation_bits exceeds usize::MAX"), + "unexpected error: {err}" + ); + } + + #[cfg(target_pointer_width = "32")] + #[test] + fn decode_tune_interface_rejects_reward_bits_exceeding_usize() { + let overflow: u64 = (u32::MAX as u64) + 1; + let bytes = encode_tune_interface(1, 1, overflow, 2); + let mut cursor = Cursor::new(&bytes); + + let err = decode_tune_interface_spec(&mut cursor) + .expect_err("overflowing reward_bits must be rejected"); + + assert!( + err.to_string() + .contains("interface.reward_bits exceeds usize::MAX"), + "unexpected error: {err}" + ); + } +} diff --git a/crates/infotheory/src/spec/document/io.rs b/crates/infotheory/src/spec/document/io.rs new file mode 100644 index 00000000..87666674 --- /dev/null +++ b/crates/infotheory/src/spec/document/io.rs @@ -0,0 +1,123 @@ +//! Disk/document I/O policy for top-level spec documents. + +use super::{DOCUMENT_MAGIC, SpecDocument, SpecError, SpecResult}; +use std::path::Path; + +/// Load a spec document from disk. +/// +/// Parsing is format-directed: +/// - `.json` files are parsed as JSON documents only. +/// - `.itsd` files (or payloads with the `itsd` magic envelope) are decoded as +/// binary documents. +/// - Otherwise, JSON parsing is attempted and JSON errors are reported +/// directly without a binary fallthrough. +pub fn load_spec_document(path: &str) -> SpecResult { + let full = Path::new(path); + let base_dir = full.parent().unwrap_or_else(|| Path::new(".")); + let raw = std::fs::read(full)?; + + let extension = full.extension().and_then(|value| value.to_str()); + if extension.is_some_and(|value| value.eq_ignore_ascii_case("json")) { + let value = serde_json::from_slice::(&raw).map_err(|err| { + SpecError::new(format!( + "invalid JSON spec document '{}': {err}", + full.display() + )) + })?; + return SpecDocument::parse_json_value(&value, base_dir); + } + + if extension.is_some_and(|value| value.eq_ignore_ascii_case("itsd")) + || raw.starts_with(DOCUMENT_MAGIC) + { + return SpecDocument::from_binary(&raw, base_dir); + } + + let value = serde_json::from_slice::(&raw).map_err(|err| { + SpecError::new(format!( + "failed to parse spec document '{}': expected JSON or binary '{}' envelope; JSON parse error: {err}", + full.display(), + String::from_utf8_lossy(DOCUMENT_MAGIC) + )) + })?; + SpecDocument::parse_json_value(&value, base_dir) +} + +#[cfg(test)] +mod tests { + use super::*; + #[cfg(feature = "backend-ctw")] + use crate::api::RateBackend; + #[cfg(feature = "backend-ctw")] + use crate::spec::CanonicalJson; + use std::path::PathBuf; + use std::time::{SystemTime, UNIX_EPOCH}; + + fn temp_path(prefix: &str, ext: &str) -> PathBuf { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("clock before epoch") + .as_nanos(); + std::env::temp_dir().join(format!("infotheory-spec-io-{prefix}-{nanos}.{ext}")) + } + + #[cfg(feature = "backend-ctw")] + #[test] + fn load_spec_document_detects_json_extension() { + let path = temp_path("json", "json"); + let expected = SpecDocument::RateBackend(RateBackend::Ctw { depth: 8 }); + std::fs::write(&path, expected.to_canonical_json().expect("json")) + .expect("write json spec"); + + let parsed = load_spec_document(path.to_string_lossy().as_ref()).expect("load json spec"); + assert_eq!( + parsed.to_canonical_json().expect("parsed json"), + expected.to_canonical_json().expect("expected json") + ); + + let _ = std::fs::remove_file(path); + } + + #[cfg(feature = "backend-ctw")] + #[test] + fn load_spec_document_detects_binary_extension_and_magic() { + let path = temp_path("binary", "itsd"); + let expected = SpecDocument::RateBackend(RateBackend::Ctw { depth: 12 }); + std::fs::write(&path, expected.to_binary()).expect("write binary spec"); + + let parsed = + load_spec_document(path.to_string_lossy().as_ref()).expect("load binary extension"); + assert_eq!( + parsed.to_canonical_json().expect("parsed json"), + expected.to_canonical_json().expect("expected json") + ); + + let magic_path = temp_path("magic", "bin"); + std::fs::write(&magic_path, expected.to_binary()).expect("write magic-detected binary"); + let magic_parsed = + load_spec_document(magic_path.to_string_lossy().as_ref()).expect("load binary magic"); + assert_eq!( + magic_parsed.to_canonical_json().expect("parsed json"), + expected.to_canonical_json().expect("expected json") + ); + + let _ = std::fs::remove_file(path); + let _ = std::fs::remove_file(magic_path); + } + + #[test] + fn load_spec_document_reports_non_json_non_binary_payloads_directly() { + let path = temp_path("invalid", "txt"); + std::fs::write(&path, b"not json and not binary").expect("write invalid payload"); + + let err = match load_spec_document(path.to_string_lossy().as_ref()) { + Ok(_) => panic!("invalid payload must fail"), + Err(err) => err, + }; + let msg = err.to_string(); + assert!(msg.contains("expected JSON or binary 'itsd' envelope")); + assert!(msg.contains(path.to_string_lossy().as_ref())); + + let _ = std::fs::remove_file(path); + } +} diff --git a/crates/infotheory/src/spec/document/mod.rs b/crates/infotheory/src/spec/document/mod.rs new file mode 100644 index 00000000..941bb5c4 --- /dev/null +++ b/crates/infotheory/src/spec/document/mod.rs @@ -0,0 +1,566 @@ +//! Canonical top-level specification documents for planner runs and tuning. + +#[cfg(feature = "tuner")] +use super::core::CompiledCompressionBackend; +use super::core::{CanonicalBytes, CompiledRateBackend}; +use super::{ + SpecEnvironment, SpecError, SpecResult, compression_backend_to_json_value, + parse_compression_backend_json, parse_rate_backend_json, rate_backend_to_json_value, +}; +use crate::aixi::common::ObservationKeyMode; +use crate::aixi::common::resolve_random_seed; +use crate::api::{CompressionBackend, RateBackend}; +use crate::spec::CanonicalJson; +use std::fmt; +use std::path::Path; +use std::sync::Arc; + +/// Schema version for canonical top-level spec documents. +pub const SPEC_DOCUMENT_SCHEMA_VERSION: u32 = 1; + +const DOCUMENT_MAGIC: &[u8; 4] = b"itsd"; +const DOCUMENT_BINARY_VERSION: u8 = 1; +#[cfg(feature = "tuner")] +const TUNE_CANONICALIZATION_CLASSIFICATION_VERSION: &str = "bounds-v1"; + +#[cfg(feature = "tuner")] +pub(crate) const CANDIDATE_EXTERNAL_ASSET_FORBIDDEN: &str = "candidate_external_asset_forbidden"; + +#[cfg(feature = "tuner")] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum TuneInvalidReason { + CandidateExternalAssetForbidden, + CandidateOutOfBounds, + CandidateCompileError, + InvalidActionIndex, + InapplicableAction, +} + +#[cfg(feature = "tuner")] +impl TuneInvalidReason { + pub(crate) const fn as_str(self) -> &'static str { + match self { + Self::CandidateExternalAssetForbidden => CANDIDATE_EXTERNAL_ASSET_FORBIDDEN, + Self::CandidateOutOfBounds => "candidate_out_of_bounds", + Self::CandidateCompileError => "candidate_compile_error", + Self::InvalidActionIndex => "invalid_action_index", + Self::InapplicableAction => "inapplicable_action", + } + } +} + +mod binary; +mod io; +mod parser; +mod pipeline; +mod serializer; +mod types; + +use binary::{builtin_environment_name, observation_key_mode_name}; +#[cfg(feature = "vm")] +use binary::{ + shared_memory_policy_name, vm_fuzz_mutator_name, vm_observation_policy_name, + vm_observation_stream_mode_name, vm_payload_encoding_name, +}; +pub use io::load_spec_document; +pub use types::*; + +impl ValidatedPlannerRunSpec { + /// Canonical validated planner-run spec. + pub fn canonical_spec(&self) -> &PlannerRunSpec { + self.canonical_spec.as_ref() + } + + /// Deterministic binary representation of the canonical planner-run spec. + pub fn canonical_bytes(&self) -> &CanonicalBytes { + &self.canonical_bytes + } + + /// Compile the validated planner-run spec into resolved assets and compiled backends. + pub fn compile(&self) -> SpecResult { + pipeline::compile_validated_planner_run_spec(self) + } +} + +#[cfg(feature = "tuner")] +impl ValidatedTuneSpec { + /// Canonical validated tune spec. + pub fn canonical_spec(&self) -> &TuneSpec { + self.canonical_spec.as_ref() + } + + /// Deterministic binary representation of the canonical tune spec. + pub fn canonical_bytes(&self) -> &CanonicalBytes { + &self.canonical_bytes + } + + /// Compile the validated tune request into resolved assets and compiled backends. + pub fn compile(&self) -> SpecResult { + pipeline::compile_validated_tune_spec(self) + } +} + +impl ParsedSpecDocument { + /// Parsed document before validation. + pub fn document(&self) -> &SpecDocument { + &self.document + } + + /// Base directory captured at parse time. + pub fn base_dir(&self) -> &Path { + self.base_dir.as_path() + } + + /// Consume this stage wrapper and return the parsed document. + pub fn into_document(self) -> SpecDocument { + self.document + } + + /// Validate this parsed document in the captured parse environment. + pub fn validate(self) -> SpecResult { + let env = SpecEnvironment::new(self.base_dir); + self.document.validate_in(&env) + } + + /// Validate and compile this parsed document in the captured parse environment. + pub fn compile(self) -> SpecResult { + self.validate()?.compile() + } +} + +impl ValidatedSpecDocument { + /// Deterministic canonical bytes for this validated document. + pub fn canonical_bytes(&self) -> &CanonicalBytes { + match self { + Self::PlannerRun(validated) => validated.canonical_bytes(), + #[cfg(feature = "tuner")] + Self::Tune(validated) => validated.canonical_bytes(), + Self::RateBackend(validated) => validated.canonical_bytes(), + Self::CompressionBackend(validated) => validated.canonical_bytes(), + } + } + + /// Compile this validated document into runtime-ready form. + pub fn compile(&self) -> SpecResult { + match self { + Self::PlannerRun(validated) => { + Ok(CompiledSpecDocument::PlannerRun(validated.compile()?)) + } + #[cfg(feature = "tuner")] + Self::Tune(validated) => Ok(CompiledSpecDocument::Tune(validated.compile()?)), + Self::RateBackend(validated) => { + Ok(CompiledSpecDocument::RateBackend(validated.compile()?)) + } + Self::CompressionBackend(validated) => Ok(CompiledSpecDocument::CompressionBackend( + validated.compile()?, + )), + } + } +} + +impl CompiledSpecDocument { + /// Deterministic canonical bytes for this compiled document. + pub fn canonical_bytes(&self) -> &CanonicalBytes { + match self { + Self::PlannerRun(compiled) => compiled.canonical_bytes(), + #[cfg(feature = "tuner")] + Self::Tune(compiled) => compiled.canonical_bytes(), + Self::RateBackend(compiled) => compiled.canonical_bytes(), + Self::CompressionBackend(compiled) => compiled.canonical_bytes(), + } + } +} + +impl PlannerRunSpec { + /// Validate this planner-run spec and return its canonical binary encoding. + pub fn validate_in(&self, env: &SpecEnvironment) -> SpecResult { + let canonical = pipeline::canonicalize_planner_run(self, env)?; + Ok(ValidatedPlannerRunSpec { + canonical_bytes: CanonicalBytes::from(binary::encode_spec_document_payload( + &SpecDocument::PlannerRun(canonical.clone()), + )), + canonical_spec: Arc::new(canonical), + base_dir: env.base_dir().to_path_buf(), + }) + } + + /// Validate this planner-run spec using the default compilation environment. + pub fn validate(&self) -> SpecResult { + self.validate_in(&SpecEnvironment::default()) + } + + /// Validate and compile this planner-run spec using the supplied environment. + pub fn compile_in(&self, env: &SpecEnvironment) -> SpecResult { + pipeline::compile_planner_run_spec(self, env.base_dir()) + } + + /// Validate and compile this planner-run spec using the default environment. + pub fn compile(&self) -> SpecResult { + self.compile_in(&SpecEnvironment::default()) + } +} + +impl EnvironmentSpec { + /// Validate and canonicalize this environment spec against the supplied asset bindings. + pub fn validate_in( + &self, + assets: &[AssetBinding], + env: &SpecEnvironment, + ) -> SpecResult { + pipeline::canonicalize_environment_spec(self, assets, env) + } + + /// Validate and canonicalize this environment spec using the default environment. + pub fn validate(&self, assets: &[AssetBinding]) -> SpecResult { + self.validate_in(assets, &SpecEnvironment::default()) + } + + /// Stable canonical kind name for this environment variant. + /// + /// Used by tooling that prints canonical names without dispatching on the + /// payload of each variant. + pub fn kind_str(&self) -> &'static str { + match self { + Self::Builtin { .. } => "builtin", + #[cfg(feature = "vm")] + Self::NyxVm(_) => "vm", + } + } +} + +impl BuiltinEnvironmentSpec { + /// Stable canonical name string for this builtin environment. + /// + /// This is the canonical document/CLI identifier used in serialized specs; + /// it does not change when new builtins are added. + pub fn canonical_name(&self) -> &'static str { + match self { + Self::TunerBridge => "tuner_bridge", + Self::CoinFlip => "coin_flip", + Self::BiasedRockPaperScissor => "biased_rock_paper_scissor", + Self::KuhnPoker => "kuhn_poker", + Self::ExtendedTiger => "extended_tiger", + Self::TicTacToe => "tic_tac_toe", + Self::Blackjack => "blackjack", + Self::Platformer => "platformer", + } + } +} + +#[cfg(feature = "tuner")] +impl TuneSpec { + /// Validate this tune request and return its canonical binary encoding. + pub fn validate_in(&self, env: &SpecEnvironment) -> SpecResult { + let canonical = pipeline::canonicalize_tune_spec(self, env)?; + Ok(ValidatedTuneSpec { + canonical_bytes: CanonicalBytes::from(binary::encode_spec_document_payload( + &SpecDocument::Tune(canonical.clone()), + )), + canonical_spec: Arc::new(canonical), + base_dir: env.base_dir().to_path_buf(), + }) + } + + /// Validate this tune request using the default compilation environment. + pub fn validate(&self) -> SpecResult { + self.validate_in(&SpecEnvironment::default()) + } + + /// Validate and compile this tune request using the supplied environment. + pub fn compile_in(&self, env: &SpecEnvironment) -> SpecResult { + pipeline::compile_tune_spec(self, env.base_dir()) + } + + /// Validate and compile this tune request using the default environment. + pub fn compile(&self) -> SpecResult { + self.compile_in(&SpecEnvironment::default()) + } +} + +impl SpecDocument { + /// Encode this document in the versioned binary document envelope. + pub fn to_binary(&self) -> Vec { + binary::encode_spec_document_payload(self) + } + + /// Parse a canonical JSON document from a raw JSON value. + pub fn parse_json_value(value: &serde_json::Value, base_dir: &Path) -> SpecResult { + parser::parse_spec_document_json_value(value, base_dir) + } + + /// Parse a canonical JSON document and return the parsed-stage wrapper. + pub fn parse_json_value_staged( + value: &serde_json::Value, + base_dir: &Path, + ) -> SpecResult { + let document = Self::parse_json_value(value, base_dir)?; + Ok(ParsedSpecDocument { + document, + base_dir: base_dir.to_path_buf(), + }) + } + + /// Decode a binary spec document. + pub fn from_binary(bytes: &[u8], base_dir: &Path) -> SpecResult { + binary::decode_spec_document(bytes, base_dir) + } + + /// Decode a binary spec document and return the parsed-stage wrapper. + pub fn from_binary_staged(bytes: &[u8], base_dir: &Path) -> SpecResult { + let document = Self::from_binary(bytes, base_dir)?; + Ok(ParsedSpecDocument { + document, + base_dir: base_dir.to_path_buf(), + }) + } + + /// Validate this top-level document in the supplied environment. + pub fn validate_in(&self, env: &SpecEnvironment) -> SpecResult { + match self { + Self::PlannerRun(spec) => Ok(ValidatedSpecDocument::PlannerRun(spec.validate_in(env)?)), + #[cfg(feature = "tuner")] + Self::Tune(spec) => Ok(ValidatedSpecDocument::Tune(spec.validate_in(env)?)), + Self::RateBackend(backend) => Ok(ValidatedSpecDocument::RateBackend( + backend.validate_in(env)?, + )), + Self::CompressionBackend(backend) => Ok(ValidatedSpecDocument::CompressionBackend( + backend.validate_in(env)?, + )), + } + } + + /// Validate this top-level document using the default compilation environment. + pub fn validate(&self) -> SpecResult { + self.validate_in(&SpecEnvironment::default()) + } + + /// Validate and compile this top-level document in the supplied environment. + pub fn compile_in(&self, env: &SpecEnvironment) -> SpecResult { + self.validate_in(env)?.compile() + } + + /// Validate and compile this top-level document using the default environment. + pub fn compile(&self) -> SpecResult { + self.compile_in(&SpecEnvironment::default()) + } + + /// Stable canonical kind name (matches the document's `"kind"` JSON field). + pub fn kind_str(&self) -> &'static str { + match self { + Self::PlannerRun(_) => "planner_run", + #[cfg(feature = "tuner")] + Self::Tune(_) => "tune", + Self::RateBackend(_) => "rate_backend", + Self::CompressionBackend(_) => "compression_backend", + } + } +} + +impl CanonicalJson for PlannerRunSpec { + fn to_canonical_json_value(&self) -> SpecResult { + serializer::planner_run_to_json_value(self) + } +} + +#[cfg(feature = "tuner")] +impl CanonicalJson for TuneSpec { + fn to_canonical_json_value(&self) -> SpecResult { + serializer::tune_spec_to_json_value(self) + } +} + +impl CanonicalJson for SpecDocument { + fn to_canonical_json_value(&self) -> SpecResult { + serializer::spec_document_to_json_value(self) + } +} + +impl CompiledPlannerController { + /// Compiled predictor backend used by this planner controller. + pub fn predictor(&self) -> &CompiledRateBackend { + match self { + Self::McAixi { predictor, .. } => predictor, + Self::AiqiDiscounted { predictor, .. } => predictor, + #[cfg(feature = "aixi")] + Self::AiqiWarmstartExactJh { predictor, .. } => predictor, + } + } + + /// Stable canonical kind name string for this controller variant. + pub fn kind_str(&self) -> &'static str { + match self { + Self::McAixi { .. } => "mc_aixi", + Self::AiqiDiscounted { .. } => "aiqi_discounted", + #[cfg(feature = "aixi")] + Self::AiqiWarmstartExactJh { .. } => "aiqi_warmstart_exact_jh", + } + } + + /// Human-readable predictor backend label. + /// + /// Backend-local algorithm parameters (such as ROSA's `max_order`) are + /// derived from the backend's variant directly. + pub fn backend_label(&self) -> String { + self.predictor().display_label() + } +} + +impl CompiledPlannerRunSpec { + /// Canonical planner-run spec used to build this compiled form. + pub fn canonical_spec(&self) -> &PlannerRunSpec { + self.canonical_spec.as_ref() + } + + /// Deterministic canonical bytes for the planner-run document. + pub fn canonical_bytes(&self) -> &CanonicalBytes { + &self.canonical_bytes + } + + /// Resolved assets used when compiling the planner-run document. + pub fn resolved_assets(&self) -> &[ResolvedAssetBinding] { + self.resolved_assets.as_ref() + } + + /// Canonical planner interface metadata. + pub fn interface(&self) -> &PlannerInterfaceSpec { + &self.interface + } + + /// Operational runtime controls. + pub fn runtime(&self) -> &PlannerRuntimeSpec { + &self.runtime + } + + /// Returns the canonical resolved planner runtime seed. + pub fn resolved_random_seed(&self) -> u64 { + resolve_random_seed(self.runtime.random_seed) + } + + /// Compiled planner controller. + pub fn controller(&self) -> &CompiledPlannerController { + &self.controller + } + + /// Number of bits required to encode agent actions. + pub fn action_bits(&self) -> usize { + self.action_bits + } +} + +#[cfg(feature = "tuner")] +impl CompiledTuneSpec { + /// Canonical tune request used to build this compiled form. + pub fn canonical_spec(&self) -> &TuneSpec { + self.canonical_spec.as_ref() + } + + /// Deterministic canonical bytes for the tune request document. + pub fn canonical_bytes(&self) -> &CanonicalBytes { + &self.canonical_bytes + } + + /// Base directory used to resolve relative paths during tune compilation. + pub fn base_dir(&self) -> &Path { + self.base_dir.as_path() + } + + /// Resolved assets used when compiling the tune request. + pub fn resolved_assets(&self) -> &[ResolvedAssetBinding] { + self.resolved_assets.as_ref() + } + + /// Compiled baseline candidate used by the future tuner. + pub fn baseline_candidate(&self) -> &CompiledCompressionBackend { + &self.baseline_candidate + } + + /// Runtime-selectable compiled controller settings. + pub fn controller(&self) -> &CompiledTuneController { + &self.controller + } + + /// Stable identifier for the current canonicalization classification rules. + pub fn candidate_canonicalization_version(&self) -> &'static str { + self.candidate_canonicalization_version + } + + /// Canonical byte length of the baseline candidate model code. + pub fn baseline_candidate_model_bytes(&self) -> usize { + self.baseline_candidate.canonical_bytes().len() + } +} + +#[cfg(feature = "vm")] +fn canonicalize_vm_observation_policy_name(name: &str) -> SpecResult { + match name { + "from_guest" => Ok(VmObservationPolicySpec::FromGuest), + "output_hash" => Ok(VmObservationPolicySpec::OutputHash), + "raw_output" => Ok(VmObservationPolicySpec::RawOutput), + "shared_memory" => Ok(VmObservationPolicySpec::SharedMemory), + other => Err(SpecError::new(format!( + "unknown VM observation_policy '{other}'" + ))), + } +} + +#[cfg(feature = "vm")] +fn canonicalize_vm_observation_stream_mode_name( + name: &str, +) -> SpecResult { + match name { + "pad_truncate" => Ok(VmObservationStreamModeSpec::PadTruncate), + "pad" => Ok(VmObservationStreamModeSpec::Pad), + "truncate" => Ok(VmObservationStreamModeSpec::Truncate), + other => Err(SpecError::new(format!( + "unknown VM observation_stream_mode '{other}'" + ))), + } +} + +#[cfg(feature = "vm")] +fn canonicalize_vm_payload_encoding( + name: &str, + field_name: &str, +) -> SpecResult { + match name { + "utf8" => Ok(VmPayloadEncodingSpec::Utf8), + "hex" => Ok(VmPayloadEncodingSpec::Hex), + other => Err(SpecError::new(format!( + "unknown VM payload encoding '{other}' for {field_name}" + ))), + } +} + +#[cfg(feature = "vm")] +fn canonicalize_vm_fuzz_mutator_name(name: &str) -> SpecResult { + match name { + "flip_bit" => Ok(VmFuzzMutatorSpec::FlipBit), + "flip_byte" => Ok(VmFuzzMutatorSpec::FlipByte), + "insert_byte" => Ok(VmFuzzMutatorSpec::InsertByte), + "delete_byte" => Ok(VmFuzzMutatorSpec::DeleteByte), + "splice_seed" => Ok(VmFuzzMutatorSpec::SpliceSeed), + "reset_seed" => Ok(VmFuzzMutatorSpec::ResetSeed), + "havoc" => Ok(VmFuzzMutatorSpec::Havoc), + other => Err(SpecError::new(format!("unknown VM fuzz mutator '{other}'"))), + } +} + +impl fmt::Debug for ValidatedPlannerRunSpec { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ValidatedPlannerRunSpec") + .field("canonical_bytes_len", &self.canonical_bytes.len()) + .finish() + } +} + +#[cfg(feature = "tuner")] +impl fmt::Debug for ValidatedTuneSpec { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ValidatedTuneSpec") + .field("canonical_bytes_len", &self.canonical_bytes.len()) + .finish() + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/infotheory/src/spec/document/parser.rs b/crates/infotheory/src/spec/document/parser.rs new file mode 100644 index 00000000..87a40882 --- /dev/null +++ b/crates/infotheory/src/spec/document/parser.rs @@ -0,0 +1,1564 @@ +//! JSON parsing for canonical top-level specification documents. + +#[cfg(feature = "aixi")] +use super::WarmStartExactJhControllerSpec; +use super::{ + AiqiDiscountedControllerSpec, AssetBinding, ControllerSpec, EnvironmentSpec, + McAixiControllerSpec, PlannerInterfaceSpec, PlannerRunSpec, PlannerRuntimeSpec, + SPEC_DOCUMENT_SCHEMA_VERSION, SpecDocument, SpecError, SpecResult, + parse_compression_backend_json, parse_rate_backend_json, +}; +#[cfg(feature = "tuner")] +use super::{ + AiqiDiscountedTuneControllerSpec, AnnealedHillClimbingTuneControllerSpec, + McAixiFacCtwTuneControllerSpec, TuneBoundsSpec, TuneControllerSpec, TuneInvalidReason, + TuneParameterRangeSpec, TunePlannerInterfaceSpec, TuneSpec, WarmStartExactJhTuneControllerSpec, + compression_backend_to_json_value, +}; +use crate::aixi::common::{ActionAlphabet, MctsStrategy}; +use crate::api::{BitOrder, BitStreamSemantics}; +use std::num::NonZeroUsize; + +#[cfg(feature = "vm")] +use super::{ + VmActionFilterSpec, VmEnvironmentSpec, VmRewardPolicySpec, VmRewardShapingSpec, + VmRuntimeActionSourceSpec, VmTraceSpec, +}; +use std::path::Path; + +pub(super) fn parse_spec_document_json_value( + value: &serde_json::Value, + base_dir: &Path, +) -> SpecResult { + let version = value["schema_version"].as_u64().unwrap_or(0); + if version != SPEC_DOCUMENT_SCHEMA_VERSION as u64 { + return Err(SpecError::new(format!( + "unsupported spec document schema_version '{version}'" + ))); + } + let kind = value["kind"] + .as_str() + .ok_or_else(|| SpecError::new("spec document kind is required"))?; + match kind { + "planner_run" => Ok(SpecDocument::PlannerRun(parse_planner_run_json_value( + value, base_dir, + )?)), + #[cfg(feature = "tuner")] + "tune" => Ok(SpecDocument::Tune(parse_tune_spec_json_value( + value, base_dir, + )?)), + #[cfg(not(feature = "tuner"))] + "tune" => Err(SpecError::new( + "tune documents require infotheory built with feature 'tuner'", + )), + "rate_backend" => Ok(SpecDocument::RateBackend(parse_rate_backend_json( + &value["backend"], + base_dir, + crate::api::MAX_MIXTURE_NESTING, + )?)), + "compression_backend" => Ok(SpecDocument::CompressionBackend( + parse_compression_backend_json( + &value["backend"], + base_dir, + None, + crate::compression::FramingMode::Framed, + )?, + )), + other => Err(SpecError::new(format!( + "unknown spec document kind '{other}'" + ))), + } +} + +fn parse_planner_run_json_value( + value: &serde_json::Value, + base_dir: &Path, +) -> SpecResult { + Ok(PlannerRunSpec { + assets: parse_asset_bindings(&value["assets"])?, + environment: parse_environment_spec(&value["environment"], base_dir)?, + interface: parse_interface_spec(&value["interface"])?, + controller: parse_controller_spec(&value["controller"], base_dir)?, + runtime: parse_runtime_spec(&value["runtime"])?, + }) +} + +#[cfg(feature = "tuner")] +fn parse_tune_spec_json_value(value: &serde_json::Value, base_dir: &Path) -> SpecResult { + ensure_known_fields( + value, + &[ + "schema_version", + "kind", + "assets", + "input_asset", + "baseline_candidate", + "controller", + "bounds", + "eval_time_limit_seconds", + "time_budget_seconds", + "min_throughput_bytes_per_second", + "max_memory_bytes", + "output_config_path", + "seed", + "report_path", + ], + "tune", + )?; + let baseline_candidate_value = value + .get("baseline_candidate") + .ok_or_else(|| SpecError::new("tune.baseline_candidate is required"))?; + reject_tune_candidate_local_external_refs(baseline_candidate_value)?; + let baseline_candidate = parse_compression_backend_json( + baseline_candidate_value, + base_dir, + None, + crate::compression::FramingMode::Framed, + )?; + ensure_tune_baseline_candidate_is_canonical_json( + baseline_candidate_value, + &baseline_candidate, + )?; + Ok(TuneSpec { + assets: parse_tune_asset_bindings( + value + .get("assets") + .ok_or_else(|| SpecError::new("tune.assets is required"))?, + )?, + input_asset: required_string(&value["input_asset"], "input_asset")?, + baseline_candidate, + controller: parse_tune_controller_spec(&value["controller"])?, + bounds: parse_tune_bounds_spec(&value["bounds"])?, + eval_time_limit_seconds: required_f64( + &value["eval_time_limit_seconds"], + "eval_time_limit_seconds", + )?, + time_budget_seconds: required_f64(&value["time_budget_seconds"], "time_budget_seconds")?, + min_throughput_bytes_per_second: required_f64( + &value["min_throughput_bytes_per_second"], + "min_throughput_bytes_per_second", + )?, + max_memory_bytes: required_u64(&value["max_memory_bytes"], "max_memory_bytes")?, + output_config_path: required_string(&value["output_config_path"], "output_config_path")?, + seed: required_u64(&value["seed"], "seed")?, + report_path: optional_string(&value["report_path"]), + }) +} + +#[cfg(feature = "tuner")] +fn reject_tune_candidate_local_external_refs(value: &serde_json::Value) -> SpecResult<()> { + fn external_asset_forbidden_error(detail: String) -> SpecError { + SpecError::new(format!( + "{}: {detail}", + TuneInvalidReason::CandidateExternalAssetForbidden.as_str() + )) + } + + fn visit(value: &serde_json::Value, path: &str) -> SpecResult<()> { + match value { + serde_json::Value::Object(object) => { + for (key, child) in object { + let next = if path.is_empty() { + key.clone() + } else { + format!("{path}.{key}") + }; + if matches!( + key.as_str(), + "spec_path" | "base_path" | "model_path" | "path" | "load_from" + ) { + return Err(external_asset_forbidden_error(format!( + "tune baseline_candidate contains candidate-local external asset field '{next}'" + ))); + } + visit(child, &next)?; + } + } + serde_json::Value::Array(items) => { + for (index, child) in items.iter().enumerate() { + visit(child, &format!("{path}[{index}]"))?; + } + } + serde_json::Value::String(raw) => { + let trimmed = raw.trim_start(); + if trimmed.starts_with("file:") || trimmed.contains("://") { + return Err(external_asset_forbidden_error(format!( + "tune baseline_candidate contains candidate-local external asset reference at '{path}'" + ))); + } + if raw.split(';').any(|segment| { + segment + .trim_start() + .strip_prefix("policy:") + .is_some_and(|policy| { + policy + .split(',') + .any(|part| part.trim_start().starts_with("load_from=")) + }) + }) { + return Err(external_asset_forbidden_error(format!( + "tune baseline_candidate contains candidate-local policy load_from at '{path}'" + ))); + } + } + _ => {} + } + Ok(()) + } + + visit(value, "baseline_candidate") +} + +#[cfg(feature = "tuner")] +fn ensure_tune_baseline_candidate_is_canonical_json( + source_value: &serde_json::Value, + parsed: &crate::api::CompressionBackend, +) -> SpecResult<()> { + let canonical_value = + compression_backend_to_json_value(parsed).map_err(|err| SpecError::new(err.to_string()))?; + if source_value != &canonical_value { + let mismatch = + first_json_mismatch_path(source_value, &canonical_value, "baseline_candidate"); + let mismatch_detail = mismatch + .map(|path| format!("; first mismatch at '{path}'")) + .unwrap_or_default(); + return Err(SpecError::new(format!( + "tune.baseline_candidate must be canonical compression backend JSON with no unknown or alias fields{mismatch_detail}" + ))); + } + Ok(()) +} + +#[cfg(feature = "tuner")] +fn first_json_mismatch_path( + observed: &serde_json::Value, + canonical: &serde_json::Value, + path: &str, +) -> Option { + match (observed, canonical) { + (serde_json::Value::Object(left), serde_json::Value::Object(right)) => { + for key in left.keys() { + if !right.contains_key(key) { + return Some(format!("{path}.{key}")); + } + } + for key in right.keys() { + let child_path = format!("{path}.{key}"); + match left.get(key) { + Some(left_child) => { + if let Some(mismatch) = + first_json_mismatch_path(left_child, &right[key], &child_path) + { + return Some(mismatch); + } + } + None => { + return Some(child_path); + } + } + } + None + } + (serde_json::Value::Array(left), serde_json::Value::Array(right)) => { + if left.len() != right.len() { + return Some(format!("{path}.len")); + } + for (index, (left_child, right_child)) in left.iter().zip(right.iter()).enumerate() { + let child_path = format!("{path}[{index}]"); + if let Some(mismatch) = + first_json_mismatch_path(left_child, right_child, &child_path) + { + return Some(mismatch); + } + } + None + } + _ => { + if observed == canonical { + None + } else { + Some(path.to_string()) + } + } + } +} + +fn ensure_known_fields(value: &serde_json::Value, allowed: &[&str], label: &str) -> SpecResult<()> { + let object = value + .as_object() + .ok_or_else(|| SpecError::new(format!("{label} document must be an object")))?; + for key in object.keys() { + if !allowed.contains(&key.as_str()) { + return Err(SpecError::new(format!("unknown {label} field '{key}'"))); + } + } + Ok(()) +} + +fn parse_asset_bindings(value: &serde_json::Value) -> SpecResult> { + let Some(items) = value.as_array() else { + return Ok(Vec::new()); + }; + items + .iter() + .map(|item| { + Ok(AssetBinding { + id: required_string(&item["id"], "assets[].id")?, + path: required_string(&item["path"], "assets[].path")?, + }) + }) + .collect() +} + +#[cfg(feature = "tuner")] +fn parse_tune_asset_bindings(value: &serde_json::Value) -> SpecResult> { + let items = value + .as_array() + .ok_or_else(|| SpecError::new("tune.assets must be an array"))?; + items + .iter() + .enumerate() + .map(|(index, item)| { + ensure_known_fields(item, &["id", "path"], &format!("tune.assets[{index}]"))?; + Ok(AssetBinding { + id: required_string(&item["id"], "assets[].id")?, + path: required_string(&item["path"], "assets[].path")?, + }) + }) + .collect() +} + +fn parse_environment_spec( + value: &serde_json::Value, + base_dir: &Path, +) -> SpecResult { + #[cfg(not(feature = "vm"))] + let _ = base_dir; + let kind = value["kind"] + .as_str() + .ok_or_else(|| SpecError::new("environment.kind is required"))?; + match kind { + "builtin" => Ok(EnvironmentSpec::Builtin { + builtin: parse_builtin_environment( + value["name"] + .as_str() + .ok_or_else(|| SpecError::new("environment.name is required"))?, + )?, + }), + #[cfg(feature = "vm")] + "nyx_vm" => Ok(EnvironmentSpec::NyxVm(VmEnvironmentSpec { + firecracker_config_asset: required_string( + &value["firecracker_config_asset"], + "environment.firecracker_config_asset", + )?, + instance_id: optional_string(&value["instance_id"]) + .unwrap_or_else(|| "aixi-nyx".to_string()), + shared_region_name: optional_string(&value["shared_region_name"]) + .unwrap_or_else(|| "shared".to_string()), + shared_region_size: default_usize( + &value["shared_region_size"], + 4096, + "environment.shared_region_size", + )?, + shared_memory_policy: parse_shared_memory_policy( + value["shared_memory_policy"].as_str().unwrap_or("snapshot"), + )?, + step_timeout_ms: value["step_timeout_ms"].as_u64().unwrap_or(100), + boot_timeout_ms: value["boot_timeout_ms"].as_u64().unwrap_or(30_000), + episode_steps: default_usize( + &value["episode_steps"], + 100, + "environment.episode_steps", + )?, + step_cost: value["step_cost"].as_i64().unwrap_or(0), + observation_policy: super::canonicalize_vm_observation_policy_name( + value["observation_policy"] + .as_str() + .unwrap_or("shared_memory"), + )?, + observation_bits: default_usize( + &value["observation_bits"], + 8, + "environment.observation_bits", + )?, + observation_stream_len: default_usize( + &value["observation_stream_len"], + 64, + "environment.observation_stream_len", + )?, + observation_stream_mode: super::canonicalize_vm_observation_stream_mode_name( + value["observation_stream_mode"] + .as_str() + .unwrap_or("pad_truncate"), + )?, + observation_pad_byte: default_u8( + &value["observation_pad_byte"], + 0, + "environment.observation_pad_byte", + )?, + reward_bits: default_usize(&value["reward_bits"], 8, "environment.reward_bits")?, + reward_policy: parse_vm_reward_policy(&value["reward_policy"])?, + reward_shaping: parse_optional_vm_reward_shaping(&value["reward_shaping"])?, + action_source: parse_vm_action_source(&value["action_source"])?, + action_filter: parse_optional_vm_action_filter(&value["action_filter"])?, + action_prefix: value["protocol"]["action_prefix"] + .as_str() + .unwrap_or("ACT ") + .to_string(), + action_suffix: value["protocol"]["action_suffix"] + .as_str() + .unwrap_or("\n") + .to_string(), + obs_prefix: value["protocol"]["obs_prefix"] + .as_str() + .unwrap_or("OBS ") + .to_string(), + rew_prefix: value["protocol"]["rew_prefix"] + .as_str() + .unwrap_or("REW ") + .to_string(), + done_prefix: value["protocol"]["done_prefix"] + .as_str() + .unwrap_or("DONE ") + .to_string(), + data_prefix: value["protocol"]["data_prefix"] + .as_str() + .unwrap_or("DATA ") + .to_string(), + wire_encoding: super::canonicalize_vm_payload_encoding( + value["protocol"]["wire_encoding"].as_str().unwrap_or("hex"), + "environment.protocol.wire_encoding", + )?, + stats_backend: parse_rate_backend_json( + &value["stats_backend"], + base_dir, + crate::api::MAX_MIXTURE_NESTING, + )?, + trace: parse_optional_vm_trace(&value["trace"])?, + debug_mode: value["debug_mode"].as_bool().unwrap_or(false), + crash_log: optional_string(&value["crash_log"]), + })), + #[cfg(not(feature = "vm"))] + "nyx_vm" => Err(SpecError::new( + "nyx_vm environment requires the 'vm' feature", + )), + other => Err(SpecError::new(format!( + "unknown environment kind '{other}'" + ))), + } +} + +fn parse_interface_spec(value: &serde_json::Value) -> SpecResult { + ensure_known_fields( + value, + &[ + "observation_bits", + "observation_stream_len", + "observation_key_mode", + "reward_bits", + "agent_actions", + ], + "interface", + )?; + let agent_actions_raw = required_usize(&value["agent_actions"], "interface.agent_actions")?; + let agent_actions = ActionAlphabet::try_from_usize(agent_actions_raw) + .map_err(|_| SpecError::new("interface.agent_actions must be >= 1"))?; + Ok(PlannerInterfaceSpec { + observation_bits: required_usize(&value["observation_bits"], "interface.observation_bits")?, + observation_stream_len: required_usize( + &value["observation_stream_len"], + "interface.observation_stream_len", + )?, + observation_key_mode: parse_observation_key_mode( + value["observation_key_mode"] + .as_str() + .unwrap_or("full_stream"), + )?, + reward_bits: required_usize(&value["reward_bits"], "interface.reward_bits")?, + agent_actions, + }) +} + +#[cfg(feature = "tuner")] +fn parse_tune_interface_spec(value: &serde_json::Value) -> SpecResult { + ensure_known_fields( + value, + &[ + "observation_bits", + "observation_stream_len", + "observation_key_mode", + "reward_bits", + "agent_actions", + ], + "controller.interface", + )?; + let agent_actions_raw = required_usize(&value["agent_actions"], "interface.agent_actions")?; + let agent_actions = ActionAlphabet::try_from_usize(agent_actions_raw) + .map_err(|_| SpecError::new("interface.agent_actions must be >= 1"))?; + Ok(TunePlannerInterfaceSpec { + observation_bits: required_usize(&value["observation_bits"], "interface.observation_bits")?, + observation_stream_len: required_usize( + &value["observation_stream_len"], + "interface.observation_stream_len", + )?, + observation_key_mode: parse_observation_key_mode( + value["observation_key_mode"].as_str().ok_or_else(|| { + SpecError::new("controller.interface.observation_key_mode is required") + })?, + )?, + reward_bits: required_usize(&value["reward_bits"], "interface.reward_bits")?, + agent_actions, + }) +} + +fn parse_controller_spec(value: &serde_json::Value, base_dir: &Path) -> SpecResult { + let kind = value["kind"] + .as_str() + .ok_or_else(|| SpecError::new("controller.kind is required"))?; + match kind { + "mc_aixi" => Ok(ControllerSpec::McAixi(McAixiControllerSpec { + predictor: parse_rate_backend_json( + &value["predictor"], + base_dir, + crate::api::MAX_MIXTURE_NESTING, + )?, + bit_stream_semantics: parse_bit_stream_semantics( + value.get("bit_stream_semantics"), + "controller.bit_stream_semantics", + )?, + agent_horizon: required_usize(&value["agent_horizon"], "controller.agent_horizon")?, + num_simulations: required_usize( + &value["num_simulations"], + "controller.num_simulations", + )?, + mcts_strategy: parse_mcts_strategy( + value.get("mcts_strategy"), + "controller.mcts_strategy", + )?, + exploration_exploitation_ratio: required_f64( + &value["exploration_exploitation_ratio"], + "controller.exploration_exploitation_ratio", + )?, + discount_gamma: required_f64(&value["discount_gamma"], "controller.discount_gamma")?, + })), + "aiqi_discounted" => Ok(ControllerSpec::AiqiDiscounted( + AiqiDiscountedControllerSpec { + predictor: parse_rate_backend_json( + &value["predictor"], + base_dir, + crate::api::MAX_MIXTURE_NESTING, + )?, + bit_stream_semantics: parse_bit_stream_semantics( + value.get("bit_stream_semantics"), + "controller.bit_stream_semantics", + )?, + discount_gamma: required_f64( + &value["discount_gamma"], + "controller.discount_gamma", + )?, + return_horizon: required_usize( + &value["return_horizon"], + "controller.return_horizon", + )?, + return_bins: required_usize(&value["return_bins"], "controller.return_bins")?, + augmentation_period: required_usize( + &value["augmentation_period"], + "controller.augmentation_period", + )?, + history_prune_keep_steps: optional_usize( + &value["history_prune_keep_steps"], + "controller.history_prune_keep_steps", + )?, + baseline_exploration: required_f64( + &value["baseline_exploration"], + "controller.baseline_exploration", + )?, + }, + )), + #[cfg(feature = "aixi")] + "aiqi_warmstart_exact_jh" => Ok(ControllerSpec::AiqiWarmstartExactJh( + WarmStartExactJhControllerSpec { + predictor: parse_rate_backend_json( + &value["predictor"], + base_dir, + crate::api::MAX_MIXTURE_NESTING, + )?, + bit_stream_semantics: parse_bit_stream_semantics( + value.get("bit_stream_semantics"), + "controller.bit_stream_semantics", + )?, + return_horizon: required_usize( + &value["return_horizon"], + "controller.return_horizon", + )?, + return_bins: required_usize(&value["return_bins"], "controller.return_bins")?, + label_phase_period: required_usize( + &value["label_phase_period"], + "controller.label_phase_period", + )?, + teacher_dataset_asset: required_string( + &value["teacher_dataset_asset"], + "controller.teacher_dataset_asset", + )?, + planner_simulations_per_step: required_usize( + &value["planner_simulations_per_step"], + "controller.planner_simulations_per_step", + )?, + }, + )), + #[cfg(not(feature = "aixi"))] + "aiqi_warmstart_exact_jh" => Err(SpecError::new( + "aiqi_warmstart_exact_jh controller requires infotheory built with feature 'aixi'", + )), + other => Err(SpecError::new(format!("unknown controller kind '{other}'"))), + } +} + +fn parse_bit_stream_semantics( + value: Option<&serde_json::Value>, + label: &str, +) -> SpecResult { + let Some(value) = value else { + // Default for absent bit_stream_semantics is BinaryTokens (AIXI planner + // paths explicitly set their own default via aixi::model when needed). + // This reference must remain feature-agnostic for parser hygiene. + return Ok(BitStreamSemantics::BinaryTokens); + }; + let Some(object) = value.as_object() else { + return Err(SpecError::new(format!( + "{label} must be an object with a 'kind' field" + ))); + }; + let kind = object + .get("kind") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| SpecError::new(format!("{label}.kind is required")))?; + match kind { + "byte_packed" => { + let order = match object.get("order").and_then(serde_json::Value::as_str) { + Some("msb_first") | None => BitOrder::MsbFirst, + Some("lsb_first") => BitOrder::LsbFirst, + Some(other) => { + return Err(SpecError::new(format!("unknown {label}.order '{other}'"))); + } + }; + Ok(BitStreamSemantics::BytePacked { order }) + } + "binary_tokens" => Ok(BitStreamSemantics::BinaryTokens), + other => Err(SpecError::new(format!( + "unknown bit stream semantics kind '{other}'" + ))), + } +} + +fn parse_mcts_strategy(value: Option<&serde_json::Value>, label: &str) -> SpecResult { + // Absent field defaults to the canonical sequential planner so older + // documents that predate `mcts_strategy` continue to parse cleanly. + let Some(value) = value else { + return Ok(MctsStrategy::RhoUct); + }; + // The canonical, serializer-emitted form is always an object with a + // `kind` field. Strings are not accepted: there is exactly one way to + // spell each strategy in the schema. + let Some(object) = value.as_object() else { + return Err(SpecError::new(format!( + "{label} must be an object with a 'kind' field" + ))); + }; + let kind = object + .get("kind") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| SpecError::new(format!("{label}.kind is required")))?; + match kind { + "rho_uct" => Ok(MctsStrategy::RhoUct), + "parallel_uct" => { + let workers_raw = required_usize(&value["workers"], &format!("{label}.workers"))?; + let workers = NonZeroUsize::new(workers_raw) + .ok_or_else(|| SpecError::new(format!("{label}.workers must be >= 1")))?; + let bu_uct_m_max = match object.get("bu_uct_m_max") { + Some(raw) if raw.is_null() => None, + Some(raw) => Some(required_f64(raw, &format!("{label}.bu_uct_m_max"))?), + None => None, + }; + Ok(MctsStrategy::ParallelUct { + workers, + bu_uct_m_max, + }) + } + other => Err(SpecError::new(format!( + "unknown MCTS strategy kind '{other}'" + ))), + } +} + +fn parse_runtime_spec(value: &serde_json::Value) -> SpecResult { + Ok(PlannerRuntimeSpec { + random_seed: value["random_seed"].as_u64(), + learn_cycles: optional_usize(&value["learn_cycles"], "runtime.learn_cycles")?, + eval_cycles: optional_usize(&value["eval_cycles"], "runtime.eval_cycles")?, + terminate_lifetime: default_usize( + &value["terminate_lifetime"], + 20, + "runtime.terminate_lifetime", + )?, + log_every: default_usize(&value["log_every"], 1, "runtime.log_every")?, + perf: value["perf"].as_bool().unwrap_or(false), + vm_perf_only: value["vm_perf_only"].as_bool().unwrap_or(false), + explore_epsilon: value["explore_epsilon"].as_f64().unwrap_or(0.0), + explore_gamma: value["explore_gamma"].as_f64().unwrap_or(1.0), + }) +} + +#[cfg(feature = "tuner")] +fn parse_tune_bounds_spec(value: &serde_json::Value) -> SpecResult { + ensure_known_fields( + value, + &[ + "allowed_backends", + "forbidden_backends", + "parameter_ranges", + "max_experts", + "max_mixture_nesting_depth", + "min_experts", + "allow_duplicate_experts", + "required_experts", + "forbidden_expert_pairs", + ], + "bounds", + )?; + Ok(TuneBoundsSpec { + allowed_backends: optional_tune_string_list( + value.get("allowed_backends"), + "bounds.allowed_backends", + )?, + forbidden_backends: optional_tune_string_list( + value.get("forbidden_backends"), + "bounds.forbidden_backends", + )?, + parameter_ranges: parse_tune_parameter_ranges(value.get("parameter_ranges"))?, + max_experts: required_usize(&value["max_experts"], "bounds.max_experts")?, + max_mixture_nesting_depth: required_usize( + &value["max_mixture_nesting_depth"], + "bounds.max_mixture_nesting_depth", + )?, + min_experts: optional_usize(&value["min_experts"], "bounds.min_experts")?, + allow_duplicate_experts: value["allow_duplicate_experts"].as_bool(), + required_experts: optional_tune_string_list( + value.get("required_experts"), + "bounds.required_experts", + )?, + forbidden_expert_pairs: optional_tune_pair_list( + value.get("forbidden_expert_pairs"), + "bounds.forbidden_expert_pairs", + )?, + }) +} + +#[cfg(feature = "tuner")] +fn optional_tune_string_list( + value: Option<&serde_json::Value>, + label: &str, +) -> SpecResult> { + let Some(raw) = value else { + return Ok(Vec::new()); + }; + if !raw.is_array() { + return Err(SpecError::new(format!("{label} must be an array"))); + } + string_list(raw).map_err(|err| SpecError::new(format!("{label}: {err}"))) +} + +#[cfg(feature = "tuner")] +fn optional_tune_pair_list( + value: Option<&serde_json::Value>, + label: &str, +) -> SpecResult> { + let Some(raw) = value else { + return Ok(Vec::new()); + }; + if !raw.is_array() { + return Err(SpecError::new(format!("{label} must be an array"))); + } + pair_list(raw).map_err(|err| SpecError::new(format!("{label}: {err}"))) +} + +#[cfg(feature = "tuner")] +fn parse_tune_controller_spec(value: &serde_json::Value) -> SpecResult { + let kind = value["kind"] + .as_str() + .ok_or_else(|| SpecError::new("controller.kind is required"))?; + match kind { + "annealed_hill_climbing" => { + ensure_known_fields( + value, + &["kind", "max_mutation_radius"], + "controller.annealed_hill_climbing", + )?; + Ok(TuneControllerSpec::AnnealedHillClimbing( + AnnealedHillClimbingTuneControllerSpec { + max_mutation_radius: required_usize( + &value["max_mutation_radius"], + "controller.max_mutation_radius", + )?, + }, + )) + } + "mc_aixi_fac_ctw" => { + ensure_known_fields( + value, + &["kind", "interface", "planner_simulations_per_step"], + "controller.mc_aixi_fac_ctw", + )?; + Ok(TuneControllerSpec::McAixiFacCtw( + McAixiFacCtwTuneControllerSpec { + interface: parse_tune_interface_spec(&value["interface"])?, + planner_simulations_per_step: required_usize( + &value["planner_simulations_per_step"], + "controller.planner_simulations_per_step", + )?, + }, + )) + } + "aiqi_discounted" => { + ensure_known_fields( + value, + &[ + "kind", + "interface", + "planner_simulations_per_step", + "return_horizon", + "return_bins", + "discount_factor", + "min_improvement", + "max_improvement", + ], + "controller.aiqi_discounted", + )?; + Ok(TuneControllerSpec::AiqiDiscounted( + AiqiDiscountedTuneControllerSpec { + interface: parse_tune_interface_spec(&value["interface"])?, + planner_simulations_per_step: required_usize( + &value["planner_simulations_per_step"], + "controller.planner_simulations_per_step", + )?, + return_horizon: required_usize( + &value["return_horizon"], + "controller.return_horizon", + )?, + return_bins: required_usize(&value["return_bins"], "controller.return_bins")?, + discount_factor: required_f64( + &value["discount_factor"], + "controller.discount_factor", + )?, + min_improvement: required_f64( + &value["min_improvement"], + "controller.min_improvement", + )?, + max_improvement: required_f64( + &value["max_improvement"], + "controller.max_improvement", + )?, + }, + )) + } + #[cfg(feature = "aixi")] + "aiqi_warmstart_exact_jh" => { + ensure_known_fields( + value, + &[ + "kind", + "interface", + "planner_simulations_per_step", + "return_horizon", + "warmstart_teacher_dataset_asset", + "label_phase_period", + ], + "controller.aiqi_warmstart_exact_jh", + )?; + Ok(TuneControllerSpec::AiqiWarmstartExactJh( + WarmStartExactJhTuneControllerSpec { + interface: parse_tune_interface_spec(&value["interface"])?, + planner_simulations_per_step: required_usize( + &value["planner_simulations_per_step"], + "controller.planner_simulations_per_step", + )?, + return_horizon: required_usize( + &value["return_horizon"], + "controller.return_horizon", + )?, + warmstart_teacher_dataset_asset: required_string( + &value["warmstart_teacher_dataset_asset"], + "controller.warmstart_teacher_dataset_asset", + )?, + label_phase_period: required_usize( + &value["label_phase_period"], + "controller.label_phase_period", + )?, + }, + )) + } + #[cfg(not(feature = "aixi"))] + "aiqi_warmstart_exact_jh" => Err(SpecError::new( + "aiqi_warmstart_exact_jh controller requires infotheory built with feature 'aixi'", + )), + other => Err(SpecError::new(format!( + "unknown tune controller kind '{other}'" + ))), + } +} + +#[cfg(feature = "tuner")] +fn parse_tune_parameter_ranges( + value: Option<&serde_json::Value>, +) -> SpecResult> { + let Some(raw) = value else { + return Ok(Vec::new()); + }; + let Some(items) = raw.as_array() else { + return Err(SpecError::new("bounds.parameter_ranges must be an array")); + }; + items + .iter() + .enumerate() + .map(|(index, item)| { + ensure_known_fields( + item, + &["parameter", "min", "max"], + &format!("bounds.parameter_ranges[{index}]"), + )?; + Ok(TuneParameterRangeSpec { + parameter: required_string( + &item["parameter"], + "bounds.parameter_ranges[].parameter", + )?, + min: required_f64(&item["min"], "bounds.parameter_ranges[].min")?, + max: required_f64(&item["max"], "bounds.parameter_ranges[].max")?, + }) + }) + .collect() +} + +#[cfg(feature = "vm")] +fn parse_vm_reward_policy(value: &serde_json::Value) -> SpecResult { + match value["kind"].as_str().unwrap_or("from_guest") { + "from_guest" => Ok(VmRewardPolicySpec::FromGuest), + "pattern" => Ok(VmRewardPolicySpec::Pattern { + pattern: required_string(&value["pattern"], "environment.reward_policy.pattern")?, + base_reward: value["base_reward"].as_i64().unwrap_or(0), + bonus_reward: value["bonus_reward"].as_i64().unwrap_or(10), + }), + other => Err(SpecError::new(format!( + "unknown vm reward policy '{other}'" + ))), + } +} + +#[cfg(feature = "vm")] +fn parse_optional_vm_reward_shaping( + value: &serde_json::Value, +) -> SpecResult> { + if value.is_null() { + return Ok(None); + } + let spec = match value["kind"].as_str().unwrap_or("trace_entropy") { + "entropy_reduction" => VmRewardShapingSpec::EntropyReduction { + baseline_asset: required_string( + &value["baseline_asset"], + "environment.reward_shaping.baseline_asset", + )?, + scale: value["scale"].as_f64().unwrap_or(1.0), + crash_bonus: value["crash_bonus"].as_i64(), + timeout_bonus: value["timeout_bonus"].as_i64(), + }, + "trace_entropy" => VmRewardShapingSpec::TraceEntropy { + scale: value["scale"].as_f64().unwrap_or(1.0), + normalize: value["normalize"].as_bool().unwrap_or(false), + }, + other => { + return Err(SpecError::new(format!( + "unknown vm reward shaping '{other}'" + ))); + } + }; + Ok(Some(spec)) +} + +#[cfg(feature = "vm")] +fn parse_vm_action_source(value: &serde_json::Value) -> SpecResult { + match value["kind"].as_str().unwrap_or("literal") { + "literal" => { + let encoding = super::canonicalize_vm_payload_encoding( + value["encoding"].as_str().unwrap_or("utf8"), + "environment.action_source.encoding", + )?; + let actions = value["actions"] + .as_array() + .ok_or_else(|| SpecError::new("environment.action_source.actions is required"))?; + let mut names = Vec::with_capacity(actions.len()); + let mut payloads = Vec::with_capacity(actions.len()); + for action in actions { + names.push(optional_string(&action["name"])); + payloads.push(required_string( + &action["payload"], + "environment.action_source.actions[].payload", + )?); + } + Ok(VmRuntimeActionSourceSpec::Literal { + names, + payloads, + encoding, + }) + } + "fuzz" => Ok(VmRuntimeActionSourceSpec::Fuzz { + seeds: string_list(&value["seeds"])?, + encoding: super::canonicalize_vm_payload_encoding( + value["encoding"].as_str().unwrap_or("utf8"), + "environment.action_source.encoding", + )?, + mutators: string_list(&value["mutators"])? + .into_iter() + .map(|name| super::canonicalize_vm_fuzz_mutator_name(&name)) + .collect::>>()?, + min_len: default_usize(&value["min_len"], 1, "environment.action_source.min_len")?, + max_len: default_usize(&value["max_len"], 4096, "environment.action_source.max_len")?, + dictionary: string_list(&value["dictionary"])?, + rng_seed: value["rng_seed"].as_u64().unwrap_or(0), + }), + other => Err(SpecError::new(format!( + "unknown vm action source '{other}'" + ))), + } +} + +#[cfg(feature = "vm")] +fn parse_optional_vm_action_filter( + value: &serde_json::Value, +) -> SpecResult> { + if value.is_null() { + return Ok(None); + } + Ok(Some(VmActionFilterSpec { + min_entropy: value["min_entropy"].as_f64(), + max_entropy: value["max_entropy"].as_f64(), + min_intrinsic_dependence: value["min_intrinsic_dependence"].as_f64(), + min_novelty: value["min_novelty"].as_f64(), + novelty_prior_asset: optional_string(&value["novelty_prior_asset"]), + reject_reward: value["reject_reward"].as_i64(), + })) +} + +#[cfg(feature = "vm")] +fn parse_optional_vm_trace(value: &serde_json::Value) -> SpecResult> { + if value.is_null() { + return Ok(None); + } + Ok(Some(VmTraceSpec { + shared_region_name: optional_string(&value["shared_region_name"]), + max_bytes: default_usize( + &value["max_bytes"], + 1_000_000, + "environment.trace.max_bytes", + )?, + reset_on_episode: value["reset_on_episode"].as_bool().unwrap_or(false), + })) +} + +#[cfg(any(feature = "tuner", feature = "vm"))] +fn string_list(value: &serde_json::Value) -> SpecResult> { + let Some(items) = value.as_array() else { + return Ok(Vec::new()); + }; + items + .iter() + .map(|item| { + item.as_str() + .map(|text| text.to_string()) + .ok_or_else(|| SpecError::new("expected string list item")) + }) + .collect() +} + +#[cfg(feature = "tuner")] +fn pair_list(value: &serde_json::Value) -> SpecResult> { + let Some(items) = value.as_array() else { + return Ok(Vec::new()); + }; + let mut pairs = Vec::with_capacity(items.len()); + for item in items { + let Some(pair) = item.as_array() else { + return Err(SpecError::new( + "forbidden_expert_pairs entries must be arrays", + )); + }; + if pair.len() != 2 { + return Err(SpecError::new( + "forbidden_expert_pairs entries must have length 2", + )); + } + pairs.push(( + required_string(&pair[0], "forbidden_expert_pairs[][0]")?, + required_string(&pair[1], "forbidden_expert_pairs[][1]")?, + )); + } + Ok(pairs) +} + +#[cfg(any(feature = "tuner", feature = "vm"))] +fn optional_string(value: &serde_json::Value) -> Option { + value.as_str().map(|text| text.to_string()) +} + +fn required_string(value: &serde_json::Value, label: &str) -> SpecResult { + value + .as_str() + .map(|text| text.to_string()) + .ok_or_else(|| SpecError::new(format!("{label} is required"))) +} + +fn required_f64(value: &serde_json::Value, label: &str) -> SpecResult { + value + .as_f64() + .ok_or_else(|| SpecError::new(format!("{label} is required"))) +} + +fn required_u64(value: &serde_json::Value, label: &str) -> SpecResult { + value + .as_u64() + .ok_or_else(|| SpecError::new(format!("{label} is required"))) +} + +fn required_usize(value: &serde_json::Value, label: &str) -> SpecResult { + usize::try_from(required_u64(value, label)?) + .map_err(|_| SpecError::new(format!("{label} exceeds usize::MAX"))) +} + +fn optional_usize(value: &serde_json::Value, label: &str) -> SpecResult> { + if value.is_null() { + Ok(None) + } else { + required_usize(value, label).map(Some) + } +} + +fn default_usize(value: &serde_json::Value, default: usize, label: &str) -> SpecResult { + if value.is_null() { + Ok(default) + } else { + required_usize(value, label) + } +} + +#[cfg(feature = "vm")] +fn default_u8(value: &serde_json::Value, default: u8, label: &str) -> SpecResult { + if value.is_null() { + Ok(default) + } else { + u8::try_from(required_u64(value, label)?) + .map_err(|_| SpecError::new(format!("{label} exceeds u8::MAX"))) + } +} + +fn parse_builtin_environment(name: &str) -> SpecResult { + match name { + "tuner_bridge" => Err(SpecError::new( + "builtin environment 'tuner_bridge' is an internal tuner planner bridge and is not accepted in canonical planner-run JSON", + )), + "coin_flip" => Ok(super::BuiltinEnvironmentSpec::CoinFlip), + "biased_rock_paper_scissor" => Ok(super::BuiltinEnvironmentSpec::BiasedRockPaperScissor), + "kuhn_poker" => Ok(super::BuiltinEnvironmentSpec::KuhnPoker), + "extended_tiger" => Ok(super::BuiltinEnvironmentSpec::ExtendedTiger), + "ctw_test" => Err(SpecError::new( + "builtin environment 'ctw_test' is no longer supported", + )), + "tic_tac_toe" => Ok(super::BuiltinEnvironmentSpec::TicTacToe), + "blackjack" => Ok(super::BuiltinEnvironmentSpec::Blackjack), + "platformer" => Ok(super::BuiltinEnvironmentSpec::Platformer), + other => Err(SpecError::new(format!( + "unknown builtin environment '{other}'" + ))), + } +} + +#[cfg(feature = "vm")] +fn parse_shared_memory_policy(name: &str) -> SpecResult { + match name { + "snapshot" => Ok(super::SharedMemoryPolicySpec::Snapshot), + "preserve" => Ok(super::SharedMemoryPolicySpec::Preserve), + other => Err(SpecError::new(format!( + "unknown shared memory policy '{other}'" + ))), + } +} + +fn parse_observation_key_mode(name: &str) -> SpecResult { + match name { + "first" => Ok(crate::aixi::common::ObservationKeyMode::First), + "last" => Ok(crate::aixi::common::ObservationKeyMode::Last), + "stream_hash" => Ok(crate::aixi::common::ObservationKeyMode::StreamHash), + "full_stream" => Ok(crate::aixi::common::ObservationKeyMode::FullStream), + other => Err(SpecError::new(format!( + "unknown observation key mode '{other}'" + ))), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_spec_document_json_value_rejects_bad_schema_and_unknown_kind() { + let err = match parse_spec_document_json_value( + &serde_json::json!({ + "schema_version": 0, + "kind": "planner_run", + }), + Path::new("."), + ) { + Ok(_) => panic!("unsupported schema version must fail"), + Err(err) => err, + }; + assert!( + err.to_string() + .contains("unsupported spec document schema_version") + ); + + let err = match parse_spec_document_json_value( + &serde_json::json!({ + "schema_version": SPEC_DOCUMENT_SCHEMA_VERSION, + "kind": "unknown", + }), + Path::new("."), + ) { + Ok(_) => panic!("unknown document kind must fail"), + Err(err) => err, + }; + assert!( + err.to_string() + .contains("unknown spec document kind 'unknown'") + ); + + #[cfg(not(feature = "tuner"))] + { + let result = parse_spec_document_json_value( + &serde_json::json!({ + "schema_version": SPEC_DOCUMENT_SCHEMA_VERSION, + "kind": "tune", + }), + Path::new("."), + ); + match result { + Ok(_) => panic!("tune must require tuner feature"), + Err(err) => assert!( + err.to_string() + .contains("tune documents require infotheory built with feature 'tuner'") + ), + } + } + } + + #[test] + fn parse_mcts_strategy_enforces_canonical_object_shape() { + assert_eq!( + parse_mcts_strategy(None, "controller.mcts_strategy") + .expect("missing strategy should default"), + MctsStrategy::RhoUct + ); + + let err = parse_mcts_strategy( + Some(&serde_json::json!("rho_uct")), + "controller.mcts_strategy", + ) + .expect_err("string shorthand must fail"); + assert!( + err.to_string() + .contains("controller.mcts_strategy must be an object with a 'kind' field") + ); + + let err = parse_mcts_strategy(Some(&serde_json::json!({})), "controller.mcts_strategy") + .expect_err("missing kind must fail"); + assert!( + err.to_string() + .contains("controller.mcts_strategy.kind is required") + ); + + let err = parse_mcts_strategy( + Some(&serde_json::json!({ + "kind": "parallel_uct", + "workers": 0, + })), + "controller.mcts_strategy", + ) + .expect_err("zero workers must fail"); + assert!( + err.to_string() + .contains("controller.mcts_strategy.workers must be >= 1") + ); + } + + #[test] + fn parse_runtime_spec_applies_document_defaults() { + let runtime = parse_runtime_spec(&serde_json::json!({})).expect("runtime defaults"); + assert_eq!(runtime.random_seed, None); + assert_eq!(runtime.learn_cycles, None); + assert_eq!(runtime.eval_cycles, None); + assert_eq!(runtime.terminate_lifetime, 20); + assert_eq!(runtime.log_every, 1); + assert!(!runtime.perf); + assert!(!runtime.vm_perf_only); + assert_eq!(runtime.explore_epsilon, 0.0); + assert_eq!(runtime.explore_gamma, 1.0); + } + + #[cfg(feature = "tuner")] + #[test] + fn parse_tune_bounds_and_list_helpers_cover_optional_shape_contracts() { + let parsed = parse_tune_bounds_spec(&serde_json::json!({ + "allowed_backends": ["ctw"], + "forbidden_backends": ["zpaq"], + "parameter_ranges": [{ + "parameter": "mixture.alpha", + "min": 0.1, + "max": 0.5, + }], + "max_experts": 4, + "max_mixture_nesting_depth": 2, + "min_experts": 1, + "allow_duplicate_experts": false, + "required_experts": ["ctw"], + "forbidden_expert_pairs": [["ctw", "zpaq"]], + })) + .expect("valid tune bounds"); + assert_eq!(parsed.allowed_backends, vec!["ctw"]); + assert_eq!( + parsed.forbidden_expert_pairs, + vec![("ctw".into(), "zpaq".into())] + ); + + let err = + string_list(&serde_json::json!(["ctw", 7])).expect_err("mixed string list must fail"); + assert!(err.to_string().contains("expected string list item")); + + let err = + pair_list(&serde_json::json!(["ctw"])).expect_err("non-array pair item must fail"); + assert!( + err.to_string() + .contains("forbidden_expert_pairs entries must be arrays") + ); + + let err = pair_list(&serde_json::json!([["ctw"]])).expect_err("short pair item must fail"); + assert!( + err.to_string() + .contains("forbidden_expert_pairs entries must have length 2") + ); + + let err = parse_tune_bounds_spec(&serde_json::json!({ + "allowed_backends": "ctw", + "forbidden_backends": ["zpaq"], + "parameter_ranges": [], + "max_experts": 4, + "max_mixture_nesting_depth": 2, + "required_experts": [], + "forbidden_expert_pairs": [], + })) + .expect_err("non-array allowed_backends must fail"); + assert!( + err.to_string() + .contains("bounds.allowed_backends must be an array") + ); + + let err = parse_tune_bounds_spec(&serde_json::json!({ + "allowed_backends": ["ctw"], + "forbidden_backends": [], + "parameter_ranges": [], + "max_experts": 4, + "max_mixture_nesting_depth": 2, + "required_experts": "ctw", + "forbidden_expert_pairs": [], + })) + .expect_err("non-array required_experts must fail"); + assert!( + err.to_string() + .contains("bounds.required_experts must be an array") + ); + + let err = parse_tune_bounds_spec(&serde_json::json!({ + "allowed_backends": ["ctw"], + "forbidden_backends": [], + "parameter_ranges": [], + "max_experts": 4, + "max_mixture_nesting_depth": 2, + "required_experts": [], + "forbidden_expert_pairs": "ctw,zpaq", + })) + .expect_err("non-array forbidden_expert_pairs must fail"); + assert!(err.to_string().contains("bounds.forbidden_expert_pairs")); + + let parsed_without_ranges = parse_tune_bounds_spec(&serde_json::json!({ + "allowed_backends": ["ctw"], + "forbidden_backends": ["zpaq"], + "max_experts": 4, + "max_mixture_nesting_depth": 2, + "required_experts": ["ctw"], + "forbidden_expert_pairs": [["ctw", "zpaq"]], + })) + .expect("missing parameter_ranges should parse as empty optional list"); + assert!(parsed_without_ranges.parameter_ranges.is_empty()); + + let err = parse_tune_bounds_spec(&serde_json::json!({ + "allowed_backends": ["ctw"], + "forbidden_backends": ["zpaq"], + "parameter_ranges": {"bad": "shape"}, + "max_experts": 4, + "max_mixture_nesting_depth": 2, + "required_experts": ["ctw"], + "forbidden_expert_pairs": [["ctw", "zpaq"]], + })) + .expect_err("non-array parameter_ranges must fail"); + assert!( + err.to_string() + .contains("bounds.parameter_ranges must be an array") + ); + } + + #[cfg(feature = "tuner")] + #[test] + fn parse_tune_controller_variants_cover_semantic_contracts() { + let interface = serde_json::json!({ + "observation_bits": 1, + "observation_stream_len": 1, + "observation_key_mode": "full_stream", + "reward_bits": 1, + "agent_actions": 2, + }); + + let fac = parse_tune_controller_spec(&serde_json::json!({ + "kind": "mc_aixi_fac_ctw", + "interface": interface.clone(), + "planner_simulations_per_step": 16, + })) + .expect("mc_aixi_fac_ctw controller should parse"); + assert!(matches!(fac, TuneControllerSpec::McAixiFacCtw(_))); + + let discounted = parse_tune_controller_spec(&serde_json::json!({ + "kind": "aiqi_discounted", + "interface": interface.clone(), + "planner_simulations_per_step": 32, + "return_horizon": 4, + "return_bins": 8, + "discount_factor": 0.95, + "min_improvement": -1.0, + "max_improvement": 1.0, + })) + .expect("aiqi_discounted controller should parse"); + assert!(matches!(discounted, TuneControllerSpec::AiqiDiscounted(_))); + + #[cfg(feature = "aixi")] + { + let warmstart = parse_tune_controller_spec(&serde_json::json!({ + "kind": "aiqi_warmstart_exact_jh", + "interface": interface, + "planner_simulations_per_step": 1, + "return_horizon": 3, + "warmstart_teacher_dataset_asset": "teacher", + "label_phase_period": 2, + })) + .expect("aiqi_warmstart_exact_jh controller should parse"); + assert!(matches!( + warmstart, + TuneControllerSpec::AiqiWarmstartExactJh(_) + )); + } + + let err = parse_tune_controller_spec(&serde_json::json!({ + "kind": "definitely_unknown_tune_controller" + })) + .expect_err("unknown tune controller kind must fail"); + assert!(err.to_string().contains("unknown tune controller kind")); + } + + #[cfg(feature = "tuner")] + #[test] + fn list_helpers_preserve_legacy_non_tune_defaults() { + let empty_strings = + string_list(&serde_json::json!("ctw")).expect("non-array string list should default"); + assert!(empty_strings.is_empty()); + + let empty_pairs = + pair_list(&serde_json::json!("ctw,zpaq")).expect("non-array pair list should default"); + assert!(empty_pairs.is_empty()); + } + + #[cfg(feature = "tuner")] + #[test] + fn parse_tune_reports_precise_baseline_canonical_mismatch_path() { + let tune = serde_json::json!({ + "schema_version": SPEC_DOCUMENT_SCHEMA_VERSION, + "kind": "tune", + "assets": [{ "id": "dataset", "path": "dataset.bin" }], + "input_asset": "dataset", + "baseline_candidate": { + "kind": "rate-ac", + "rate_backend": { + "kind": "ctw", + "depth": 16, + "extra_alias_field": 7 + }, + "framing": "framed" + }, + "controller": { + "kind": "annealed_hill_climbing", + "max_mutation_radius": 1 + }, + "bounds": { + "allowed_backends": ["ctw"], + "forbidden_backends": [], + "parameter_ranges": [], + "max_experts": 2, + "max_mixture_nesting_depth": 1, + "required_experts": [], + "forbidden_expert_pairs": [] + }, + "eval_time_limit_seconds": 1.0, + "time_budget_seconds": 1.0, + "min_throughput_bytes_per_second": 1.0, + "max_memory_bytes": 1024, + "output_config_path": "out.json", + "seed": 1 + }); + let err = match parse_spec_document_json_value(&tune, Path::new(".")) { + Ok(_) => panic!("non-canonical baseline candidate must fail"), + Err(err) => err, + }; + let message = err.to_string(); + assert!( + message.contains("must be canonical compression backend JSON"), + "unexpected error: {message}" + ); + assert!( + message + .contains("first mismatch at 'baseline_candidate.rate_backend.extra_alias_field'"), + "mismatch path should be explicit: {message}" + ); + } + + #[test] + fn canonical_name_parsers_reject_removed_or_unknown_aliases() { + let err = + parse_builtin_environment("ctw_test").expect_err("removed builtin alias must fail"); + assert!(err.to_string().contains("no longer supported")); + + assert_eq!( + parse_observation_key_mode("first").expect("first"), + crate::aixi::common::ObservationKeyMode::First + ); + assert_eq!( + parse_observation_key_mode("full_stream").expect("full_stream"), + crate::aixi::common::ObservationKeyMode::FullStream + ); + + let err = parse_observation_key_mode("streamhash") + .expect_err("non-canonical observation alias must fail"); + assert!(err.to_string().contains("unknown observation key mode")); + } +} diff --git a/crates/infotheory/src/spec/document/pipeline.rs b/crates/infotheory/src/spec/document/pipeline.rs new file mode 100644 index 00000000..cf16d2bb --- /dev/null +++ b/crates/infotheory/src/spec/document/pipeline.rs @@ -0,0 +1,1499 @@ +//! Canonical validation and compile pipeline for spec documents. + +use super::{ + AssetBinding, CompiledPlannerController, CompiledPlannerRunSpec, ControllerSpec, + EnvironmentSpec, PlannerInterfaceSpec, PlannerRunSpec, PlannerRuntimeSpec, + ResolvedAssetBinding, SpecEnvironment, SpecError, SpecResult, ValidatedPlannerRunSpec, +}; +#[cfg(feature = "tuner")] +use super::{ + CompiledTuneController, CompiledTuneSpec, TUNE_CANONICALIZATION_CLASSIFICATION_VERSION, + TuneBoundsSpec, TuneControllerSpec, TunePlannerInterfaceSpec, TuneSpec, ValidatedTuneSpec, +}; +use crate::aixi::common::{ + MctsStrategy, bits_for_cardinality, byte_packed_percept_bits, resolve_random_seed, + validate_aiqi_byte_packed_alignment, validate_mc_aixi_byte_packed_alignment, + warn_parallel_uct_workers_one_once, +}; +#[cfg(feature = "aixi")] +use crate::aixi::warmstart::{ + WarmStartExactJhError, max_reward_from_exact_return_bins, reward_bounds_from_exact_return_bins, +}; +use crate::spec::core::AssetRef; +use std::collections::HashMap; +#[cfg(feature = "aixi")] +use std::num::NonZeroUsize; +use std::path::Path; +use std::sync::Arc; + +#[cfg(feature = "vm")] +use super::{VmEnvironmentSpec, VmRewardShapingSpec, VmRuntimeActionSourceSpec}; + +fn resolve_asset_bindings( + bindings: &[AssetBinding], + base_dir: &Path, +) -> Arc<[ResolvedAssetBinding]> { + bindings + .iter() + .map(|binding| ResolvedAssetBinding { + id: binding.id.clone(), + asset: AssetRef::Filesystem(super::super::resolve_spec_path(base_dir, &binding.path)), + }) + .collect::>() + .into() +} + +fn compile_planner_controller( + spec: &ControllerSpec, + env: &SpecEnvironment, +) -> SpecResult { + match spec { + ControllerSpec::McAixi(inner) => Ok(CompiledPlannerController::McAixi { + predictor: inner.predictor.validate_in(env)?.compile()?, + bit_stream_semantics: inner.bit_stream_semantics, + agent_horizon: inner.agent_horizon, + num_simulations: inner.num_simulations, + mcts_strategy: inner.mcts_strategy, + exploration_exploitation_ratio: inner.exploration_exploitation_ratio, + discount_gamma: inner.discount_gamma, + }), + ControllerSpec::AiqiDiscounted(inner) => Ok(CompiledPlannerController::AiqiDiscounted { + predictor: inner.predictor.validate_in(env)?.compile()?, + bit_stream_semantics: inner.bit_stream_semantics, + discount_gamma: inner.discount_gamma, + return_horizon: inner.return_horizon, + return_bins: inner.return_bins, + augmentation_period: inner.augmentation_period, + history_prune_keep_steps: inner.history_prune_keep_steps, + baseline_exploration: inner.baseline_exploration, + }), + #[cfg(feature = "aixi")] + ControllerSpec::AiqiWarmstartExactJh(inner) => { + Ok(CompiledPlannerController::AiqiWarmstartExactJh { + predictor: inner.predictor.validate_in(env)?.compile()?, + bit_stream_semantics: inner.bit_stream_semantics, + return_horizon: inner.return_horizon, + return_bins: inner.return_bins, + label_phase_period: inner.label_phase_period, + teacher_dataset_asset: inner.teacher_dataset_asset.clone(), + planner_simulations_per_step: inner.planner_simulations_per_step, + }) + } + } +} + +fn validate_mc_aixi_mcts_strategy(strategy: MctsStrategy) -> SpecResult<()> { + match strategy { + MctsStrategy::RhoUct => Ok(()), + MctsStrategy::ParallelUct { + workers, + bu_uct_m_max, + } => { + // `workers` is type-enforced non-zero by `NonZeroUsize`; only the + // `workers == 1` warning and the BU-UCT threshold range remain + // checkable at this layer. + if workers.get() == 1 { + warn_parallel_uct_workers_one_once(); + } + if bu_uct_m_max.is_some_and(|m_max| !(0.0 < m_max && m_max < 1.0)) { + return Err(SpecError::new( + "controller.mcts_strategy.bu_uct_m_max must be in (0, 1)", + )); + } + Ok(()) + } + } +} + +#[cfg(feature = "aixi")] +fn validate_warmstart_direct_evaluator_marker( + planner_simulations_per_step: usize, +) -> SpecResult<()> { + if planner_simulations_per_step != 1 { + return Err(SpecError::new( + "planner_simulations_per_step must be exactly 1 for warm-start exact-J_H direct evaluation", + )); + } + Ok(()) +} + +#[cfg(feature = "aixi")] +fn validate_warmstart_exact_reward_channel( + return_horizon: usize, + return_bins: usize, + reward_bits: usize, +) -> SpecResult<()> { + let (return_horizon, return_bins) = + nonzero_warmstart_exact_return_shape(return_horizon, return_bins)?; + if let Err(err) = reward_bounds_from_exact_return_bins(return_horizon, return_bins, reward_bits) + { + return match err { + WarmStartExactJhError::RewardEncoding(err) => { + let max_reward = max_reward_from_exact_return_bins(return_horizon, return_bins) + .map_err(|err| SpecError::new(err.to_string()))?; + Err(SpecError::new(format!( + "return_bins imply max_reward={max_reward} for return_horizon={}, \ + but that reward range is not representable by reward_bits={reward_bits}: {err}", + return_horizon.get() + ))) + } + err => Err(SpecError::new(err.to_string())), + }; + } + Ok(()) +} + +#[cfg(feature = "aixi")] +fn validate_warmstart_exact_return_bins( + return_horizon: usize, + return_bins: usize, +) -> SpecResult<()> { + let (return_horizon, return_bins) = + nonzero_warmstart_exact_return_shape(return_horizon, return_bins)?; + max_reward_from_exact_return_bins(return_horizon, return_bins) + .map(|_| ()) + .map_err(|err| SpecError::new(err.to_string())) +} + +#[cfg(feature = "aixi")] +fn nonzero_warmstart_exact_return_shape( + return_horizon: usize, + return_bins: usize, +) -> SpecResult<(NonZeroUsize, NonZeroUsize)> { + let return_horizon = NonZeroUsize::new(return_horizon) + .ok_or_else(|| SpecError::new("return_horizon must be >= 1"))?; + let return_bins = + NonZeroUsize::new(return_bins).ok_or_else(|| SpecError::new("return_bins must be >= 1"))?; + Ok((return_horizon, return_bins)) +} + +#[cfg(feature = "tuner")] +fn compile_tune_controller(spec: &TuneControllerSpec) -> CompiledTuneController { + match spec { + TuneControllerSpec::AnnealedHillClimbing(inner) => { + CompiledTuneController::AnnealedHillClimbing(inner.clone()) + } + TuneControllerSpec::McAixiFacCtw(inner) => { + CompiledTuneController::McAixiFacCtw(inner.clone()) + } + TuneControllerSpec::AiqiDiscounted(inner) => { + CompiledTuneController::AiqiDiscounted(inner.clone()) + } + #[cfg(feature = "aixi")] + TuneControllerSpec::AiqiWarmstartExactJh(inner) => { + CompiledTuneController::AiqiWarmstartExactJh(inner.clone()) + } + } +} + +pub(super) fn compile_planner_run_spec( + spec: &PlannerRunSpec, + base_dir: &Path, +) -> SpecResult { + let validated = spec.validate_in(&SpecEnvironment::new(base_dir))?; + compile_validated_planner_run_spec(&validated) +} + +pub(super) fn compile_validated_planner_run_spec( + validated: &ValidatedPlannerRunSpec, +) -> SpecResult { + let env = SpecEnvironment::new(&validated.base_dir); + Ok(CompiledPlannerRunSpec { + canonical_spec: validated.canonical_spec.clone(), + canonical_bytes: validated.canonical_bytes().clone(), + resolved_assets: resolve_asset_bindings( + &validated.canonical_spec().assets, + &validated.base_dir, + ), + interface: validated.canonical_spec().interface.clone(), + runtime: validated.canonical_spec().runtime.clone(), + controller: compile_planner_controller(&validated.canonical_spec().controller, &env)?, + action_bits: bits_for_cardinality(validated.canonical_spec().interface.agent_actions.get()), + }) +} + +#[cfg(feature = "tuner")] +pub(super) fn compile_tune_spec(spec: &TuneSpec, base_dir: &Path) -> SpecResult { + let validated = spec.validate_in(&SpecEnvironment::new(base_dir))?; + compile_validated_tune_spec(&validated) +} + +#[cfg(feature = "tuner")] +pub(super) fn compile_validated_tune_spec( + validated: &ValidatedTuneSpec, +) -> SpecResult { + let env = SpecEnvironment::new(&validated.base_dir); + Ok(CompiledTuneSpec { + canonical_spec: validated.canonical_spec.clone(), + canonical_bytes: validated.canonical_bytes().clone(), + base_dir: validated.base_dir.clone(), + resolved_assets: resolve_asset_bindings( + &validated.canonical_spec().assets, + &validated.base_dir, + ), + baseline_candidate: validated + .canonical_spec() + .baseline_candidate + .validate_in(&env)? + .compile()?, + controller: compile_tune_controller(&validated.canonical_spec().controller), + candidate_canonicalization_version: TUNE_CANONICALIZATION_CLASSIFICATION_VERSION, + }) +} + +pub(super) fn canonicalize_planner_run( + spec: &PlannerRunSpec, + env: &SpecEnvironment, +) -> SpecResult { + validate_asset_bindings(&spec.assets)?; + let environment = canonicalize_environment_spec(&spec.environment, &spec.assets, env)?; + let interface = canonicalize_interface_spec(&spec.interface)?; + let controller = + canonicalize_controller_spec(&spec.controller, &spec.assets, env, Some(&interface))?; + let runtime = canonicalize_runtime_spec(&spec.runtime)?; + Ok(PlannerRunSpec { + assets: canonicalize_assets(&spec.assets), + environment, + interface, + controller, + runtime, + }) +} + +#[cfg(feature = "tuner")] +pub(super) fn canonicalize_tune_spec( + spec: &TuneSpec, + env: &SpecEnvironment, +) -> SpecResult { + validate_asset_bindings(&spec.assets)?; + ensure_asset_exists(&spec.assets, &spec.input_asset)?; + let baseline = spec + .baseline_candidate + .validate_in(env)? + .canonical_spec() + .clone(); + let controller = canonicalize_tune_controller(&spec.controller, &spec.assets, env)?; + validate_tune_bounds(&spec.bounds)?; + Ok(TuneSpec { + assets: canonicalize_assets(&spec.assets), + input_asset: spec.input_asset.trim().to_string(), + baseline_candidate: baseline, + controller, + bounds: canonicalize_tune_bounds(&spec.bounds), + eval_time_limit_seconds: finite_positive( + spec.eval_time_limit_seconds, + "eval_time_limit_seconds", + )?, + time_budget_seconds: finite_positive(spec.time_budget_seconds, "time_budget_seconds")?, + min_throughput_bytes_per_second: finite_positive( + spec.min_throughput_bytes_per_second, + "min_throughput_bytes_per_second", + )?, + max_memory_bytes: nonzero_u64(spec.max_memory_bytes, "max_memory_bytes")?, + output_config_path: spec.output_config_path.trim().to_string(), + seed: spec.seed, + report_path: clean_optional_string(spec.report_path.as_deref()), + }) +} + +#[cfg(feature = "tuner")] +fn canonicalize_tune_controller( + controller: &TuneControllerSpec, + assets: &[AssetBinding], + _env: &SpecEnvironment, +) -> SpecResult { + match controller { + TuneControllerSpec::AnnealedHillClimbing(inner) => { + if inner.max_mutation_radius == 0 { + return Err(SpecError::new("max_mutation_radius must be >= 1")); + } + Ok(TuneControllerSpec::AnnealedHillClimbing(inner.clone())) + } + TuneControllerSpec::McAixiFacCtw(inner) => { + canonicalize_tune_interface_spec(&inner.interface)?; + if inner.planner_simulations_per_step == 0 { + return Err(SpecError::new("planner_simulations_per_step must be >= 1")); + } + Ok(TuneControllerSpec::McAixiFacCtw(inner.clone())) + } + TuneControllerSpec::AiqiDiscounted(inner) => { + canonicalize_tune_interface_spec(&inner.interface)?; + if inner.planner_simulations_per_step == 0 { + return Err(SpecError::new("planner_simulations_per_step must be >= 1")); + } + if inner.return_horizon == 0 { + return Err(SpecError::new("return_horizon must be >= 1")); + } + if inner.return_bins == 0 { + return Err(SpecError::new("return_bins must be >= 1")); + } + if !(0.0..1.0).contains(&inner.discount_factor) { + return Err(SpecError::new("discount_factor must be in [0, 1)")); + } + if !inner.min_improvement.is_finite() { + return Err(SpecError::new("min_improvement must be finite")); + } + if !inner.max_improvement.is_finite() { + return Err(SpecError::new("max_improvement must be finite")); + } + if inner.max_improvement <= inner.min_improvement { + return Err(SpecError::new( + "max_improvement must be greater than min_improvement", + )); + } + Ok(TuneControllerSpec::AiqiDiscounted(inner.clone())) + } + #[cfg(feature = "aixi")] + TuneControllerSpec::AiqiWarmstartExactJh(inner) => { + canonicalize_tune_interface_spec(&inner.interface)?; + if inner.planner_simulations_per_step == 0 { + return Err(SpecError::new("planner_simulations_per_step must be >= 1")); + } + validate_warmstart_direct_evaluator_marker(inner.planner_simulations_per_step)?; + if inner.return_horizon == 0 { + return Err(SpecError::new("return_horizon must be >= 1")); + } + if inner.label_phase_period < inner.return_horizon { + return Err(SpecError::new( + "label_phase_period must be >= return_horizon", + )); + } + ensure_asset_exists(assets, &inner.warmstart_teacher_dataset_asset)?; + Ok(TuneControllerSpec::AiqiWarmstartExactJh(inner.clone())) + } + } +} + +#[cfg(feature = "tuner")] +fn canonicalize_tune_interface_spec( + spec: &TunePlannerInterfaceSpec, +) -> SpecResult { + if spec.observation_stream_len == 0 { + return Err(SpecError::new("observation_stream_len must be >= 1")); + } + if spec.reward_bits == 0 { + return Err(SpecError::new("reward_bits must be >= 1")); + } + Ok(spec.clone()) +} + +fn canonicalize_assets(bindings: &[AssetBinding]) -> Vec { + let mut out = bindings.to_vec(); + out.sort_by(|a, b| a.id.cmp(&b.id).then_with(|| a.path.cmp(&b.path))); + out +} + +fn validate_asset_bindings(bindings: &[AssetBinding]) -> SpecResult<()> { + let mut seen = HashMap::<&str, &str>::new(); + for binding in bindings { + let id = binding.id.trim(); + let path = binding.path.trim(); + if id.is_empty() { + return Err(SpecError::new("asset id cannot be empty")); + } + if path.is_empty() { + return Err(SpecError::new(format!( + "asset '{}' path cannot be empty", + binding.id + ))); + } + if let Some(previous) = seen.insert(id, path) + && previous != path + { + return Err(SpecError::new(format!( + "asset '{}' is bound to more than one path", + binding.id + ))); + } + } + Ok(()) +} + +#[cfg(feature = "aixi")] +fn ensure_asset_exists(bindings: &[AssetBinding], id: &str) -> SpecResult<()> { + if bindings.iter().any(|binding| binding.id == id) { + Ok(()) + } else { + Err(SpecError::new(format!("unknown asset id '{id}'"))) + } +} + +fn canonicalize_interface_spec(spec: &PlannerInterfaceSpec) -> SpecResult { + if spec.observation_stream_len == 0 { + return Err(SpecError::new("observation_stream_len must be >= 1")); + } + if spec.reward_bits == 0 { + return Err(SpecError::new("reward_bits must be >= 1")); + } + Ok(spec.clone()) +} + +fn canonicalize_controller_spec( + spec: &ControllerSpec, + #[cfg_attr(not(feature = "aixi"), allow(unused_variables))] assets: &[AssetBinding], + env: &SpecEnvironment, + interface: Option<&PlannerInterfaceSpec>, +) -> SpecResult { + match spec { + ControllerSpec::McAixi(inner) => { + if inner.agent_horizon == 0 { + return Err(SpecError::new("agent_horizon must be >= 1")); + } + if inner.num_simulations == 0 { + return Err(SpecError::new("num_simulations must be >= 1")); + } + validate_mc_aixi_mcts_strategy(inner.mcts_strategy)?; + if inner.exploration_exploitation_ratio <= 0.0 { + return Err(SpecError::new("exploration_exploitation_ratio must be > 0")); + } + if !(0.0..=1.0).contains(&inner.discount_gamma) { + return Err(SpecError::new("discount_gamma must be in [0, 1]")); + } + if matches!( + inner.bit_stream_semantics, + crate::api::BitStreamSemantics::BytePacked { .. } + ) && let Some(interface) = interface + { + let action_bits = interface.agent_actions.action_bits(); + let percept_bits = byte_packed_percept_bits( + interface.observation_bits, + interface.observation_stream_len, + interface.reward_bits, + ); + validate_mc_aixi_byte_packed_alignment(action_bits, percept_bits) + .map_err(SpecError::new)?; + } + let validated_predictor = inner.predictor.validate_in(env)?; + let predictor = validated_predictor.canonical_spec().clone(); + if validated_predictor.capabilities().contains_zpaq { + return Err(SpecError::new( + "MC-AIXI strict generic rate_backend support requires reversible action conditioning; configured rate_backend contains zpaq which does not provide the reversible action conditioning required by \"A Monte-Carlo AIXI Approximation\"", + )); + } + Ok(ControllerSpec::McAixi(super::McAixiControllerSpec { + predictor, + bit_stream_semantics: inner.bit_stream_semantics, + agent_horizon: inner.agent_horizon, + num_simulations: inner.num_simulations, + mcts_strategy: inner.mcts_strategy, + exploration_exploitation_ratio: inner.exploration_exploitation_ratio, + discount_gamma: inner.discount_gamma, + })) + } + ControllerSpec::AiqiDiscounted(inner) => { + if inner.return_horizon == 0 { + return Err(SpecError::new("return_horizon must be >= 1")); + } + if inner.return_bins == 0 { + return Err(SpecError::new("return_bins must be >= 1")); + } + if inner.augmentation_period < inner.return_horizon { + return Err(SpecError::new( + "augmentation_period must be >= return_horizon", + )); + } + if !(0.0 < inner.discount_gamma && inner.discount_gamma < 1.0) { + return Err(SpecError::new("discount_gamma must be in (0, 1)")); + } + if !(0.0 < inner.baseline_exploration && inner.baseline_exploration <= 1.0) { + return Err(SpecError::new("baseline_exploration must be in (0, 1]")); + } + if matches!( + inner.bit_stream_semantics, + crate::api::BitStreamSemantics::BytePacked { .. } + ) && let Some(interface) = interface + { + let action_bits = interface.agent_actions.action_bits(); + let percept_bits = byte_packed_percept_bits( + interface.observation_bits, + interface.observation_stream_len, + interface.reward_bits, + ); + let return_bits = crate::aixi::common::bits_for_cardinality(inner.return_bins); + validate_aiqi_byte_packed_alignment(action_bits, percept_bits, return_bits) + .map_err(SpecError::new)?; + } + let validated_predictor = inner.predictor.validate_in(env)?; + let predictor = validated_predictor.canonical_spec().clone(); + if !validated_predictor + .capabilities() + .supports_frozen_conditioning + { + return Err(SpecError::new( + "AIQI strict mode requires frozen context updates; configured rate_backend contains zpaq which does not provide strict frozen conditioning", + )); + } + Ok(ControllerSpec::AiqiDiscounted( + super::AiqiDiscountedControllerSpec { + predictor, + bit_stream_semantics: inner.bit_stream_semantics, + discount_gamma: inner.discount_gamma, + return_horizon: inner.return_horizon, + return_bins: inner.return_bins, + augmentation_period: inner.augmentation_period, + history_prune_keep_steps: inner.history_prune_keep_steps, + baseline_exploration: inner.baseline_exploration, + }, + )) + } + #[cfg(feature = "aixi")] + ControllerSpec::AiqiWarmstartExactJh(inner) => { + if inner.planner_simulations_per_step == 0 { + return Err(SpecError::new("planner_simulations_per_step must be >= 1")); + } + validate_warmstart_direct_evaluator_marker(inner.planner_simulations_per_step)?; + if inner.return_horizon == 0 { + return Err(SpecError::new("return_horizon must be >= 1")); + } + if inner.return_bins == 0 { + return Err(SpecError::new("return_bins must be >= 1")); + } + validate_warmstart_exact_return_bins(inner.return_horizon, inner.return_bins)?; + if let Some(interface) = interface { + validate_warmstart_exact_reward_channel( + inner.return_horizon, + inner.return_bins, + interface.reward_bits, + )?; + } + if inner.label_phase_period < inner.return_horizon { + return Err(SpecError::new( + "label_phase_period must be >= return_horizon", + )); + } + if matches!( + inner.bit_stream_semantics, + crate::api::BitStreamSemantics::BytePacked { .. } + ) && let Some(interface) = interface + { + let action_bits = interface.agent_actions.action_bits(); + let percept_bits = byte_packed_percept_bits( + interface.observation_bits, + interface.observation_stream_len, + interface.reward_bits, + ); + let return_bits = crate::aixi::common::bits_for_cardinality(inner.return_bins); + validate_aiqi_byte_packed_alignment(action_bits, percept_bits, return_bits) + .map_err(SpecError::new)?; + } + let validated_predictor = inner.predictor.validate_in(env)?; + let predictor = validated_predictor.canonical_spec().clone(); + let teacher_dataset_asset = inner.teacher_dataset_asset.trim(); + if teacher_dataset_asset.is_empty() { + return Err(SpecError::new("teacher_dataset_asset cannot be empty")); + } + ensure_asset_exists(assets, teacher_dataset_asset)?; + Ok(ControllerSpec::AiqiWarmstartExactJh( + super::WarmStartExactJhControllerSpec { + predictor, + bit_stream_semantics: inner.bit_stream_semantics, + return_horizon: inner.return_horizon, + return_bins: inner.return_bins, + label_phase_period: inner.label_phase_period, + teacher_dataset_asset: teacher_dataset_asset.to_string(), + planner_simulations_per_step: inner.planner_simulations_per_step, + }, + )) + } + } +} + +#[cfg(feature = "vm")] +fn canonicalize_vm_action_source( + source: &VmRuntimeActionSourceSpec, +) -> SpecResult { + match source { + VmRuntimeActionSourceSpec::Literal { + names, + payloads, + encoding, + } => Ok(VmRuntimeActionSourceSpec::Literal { + names: names.clone(), + payloads: payloads.clone(), + encoding: *encoding, + }), + VmRuntimeActionSourceSpec::Fuzz { + seeds, + encoding, + mutators, + min_len, + max_len, + dictionary, + rng_seed, + } => { + if seeds.is_empty() { + return Err(SpecError::new( + "environment.action_source.seeds must include at least one seed in fuzz mode", + )); + } + if mutators.is_empty() { + return Err(SpecError::new( + "environment.action_source.mutators must include at least one mutator in fuzz mode", + )); + } + if min_len > max_len { + return Err(SpecError::new( + "environment.action_source.min_len cannot exceed max_len", + )); + } + Ok(VmRuntimeActionSourceSpec::Fuzz { + seeds: seeds.clone(), + encoding: *encoding, + mutators: mutators.clone(), + min_len: *min_len, + max_len: *max_len, + dictionary: dictionary.clone(), + rng_seed: *rng_seed, + }) + } + } +} + +#[cfg(feature = "vm")] +fn canonicalize_vm_environment_spec( + vm: &VmEnvironmentSpec, + assets: &[AssetBinding], + env: &SpecEnvironment, +) -> SpecResult { + ensure_asset_exists(assets, &vm.firecracker_config_asset)?; + vm.stats_backend.validate_in(env)?; + if let Some(shape) = &vm.reward_shaping + && let VmRewardShapingSpec::EntropyReduction { baseline_asset, .. } = shape + { + ensure_asset_exists(assets, baseline_asset)?; + } + if let Some(filter) = &vm.action_filter + && let Some(asset) = &filter.novelty_prior_asset + { + ensure_asset_exists(assets, asset)?; + } + if vm.episode_steps == 0 { + return Err(SpecError::new("environment.episode_steps must be >= 1")); + } + let mut canonical = vm.clone(); + canonical.action_source = canonicalize_vm_action_source(&vm.action_source)?; + Ok(canonical) +} + +pub(super) fn canonicalize_environment_spec( + spec: &EnvironmentSpec, + _assets: &[AssetBinding], + _env: &SpecEnvironment, +) -> SpecResult { + match spec { + EnvironmentSpec::Builtin { builtin } => Ok(EnvironmentSpec::Builtin { builtin: *builtin }), + #[cfg(feature = "vm")] + EnvironmentSpec::NyxVm(vm) => Ok(EnvironmentSpec::NyxVm(canonicalize_vm_environment_spec( + vm, _assets, _env, + )?)), + } +} + +fn canonicalize_runtime_spec(spec: &PlannerRuntimeSpec) -> SpecResult { + if spec.terminate_lifetime == 0 { + return Err(SpecError::new("terminate_lifetime must be >= 1")); + } + if spec.log_every == 0 { + return Err(SpecError::new("log_every must be >= 1")); + } + if spec.explore_epsilon < 0.0 { + return Err(SpecError::new("explore_epsilon must be >= 0")); + } + if spec.explore_gamma <= 0.0 { + return Err(SpecError::new("explore_gamma must be > 0")); + } + let mut canonical = spec.clone(); + canonical.random_seed = Some(resolve_random_seed(spec.random_seed)); + Ok(canonical) +} + +#[cfg(feature = "tuner")] +fn validate_tune_bounds(bounds: &TuneBoundsSpec) -> SpecResult<()> { + if bounds.max_experts == 0 { + return Err(SpecError::new("max_experts must be >= 1")); + } + if bounds.max_mixture_nesting_depth == 0 { + return Err(SpecError::new("max_mixture_nesting_depth must be >= 1")); + } + if let Some(min_experts) = bounds.min_experts + && min_experts > bounds.max_experts + { + return Err(SpecError::new("min_experts cannot exceed max_experts")); + } + for range in &bounds.parameter_ranges { + if range.parameter.trim().is_empty() { + return Err(SpecError::new( + "bounds.parameter_ranges[].parameter cannot be empty", + )); + } + if !range.min.is_finite() || !range.max.is_finite() { + return Err(SpecError::new( + "bounds.parameter_ranges must use finite min/max values", + )); + } + if range.min > range.max { + return Err(SpecError::new( + "bounds.parameter_ranges min cannot exceed max", + )); + } + } + for name in bounds + .allowed_backends + .iter() + .chain(bounds.forbidden_backends.iter()) + .chain(bounds.required_experts.iter()) + { + if name.trim().is_empty() { + return Err(SpecError::new( + "bounds backend/expert names cannot be empty", + )); + } + } + for name in &bounds.allowed_backends { + if bounds + .forbidden_backends + .iter() + .any(|forbidden| forbidden == name) + { + return Err(SpecError::new( + "allowed_backends and forbidden_backends cannot overlap", + )); + } + } + Ok(()) +} + +#[cfg(feature = "tuner")] +fn canonicalize_tune_bounds(bounds: &TuneBoundsSpec) -> TuneBoundsSpec { + let mut allowed = bounds.allowed_backends.clone(); + allowed.sort(); + allowed.dedup(); + let mut forbidden_backends = bounds.forbidden_backends.clone(); + forbidden_backends.sort(); + forbidden_backends.dedup(); + let mut required = bounds.required_experts.clone(); + required.sort(); + required.dedup(); + let mut parameter_ranges = bounds.parameter_ranges.clone(); + parameter_ranges.sort_by(|a, b| a.parameter.cmp(&b.parameter)); + parameter_ranges + .dedup_by(|a, b| a.parameter == b.parameter && a.min == b.min && a.max == b.max); + let mut forbidden_pairs = bounds + .forbidden_expert_pairs + .iter() + .map(|(a, b)| { + if a <= b { + (a.clone(), b.clone()) + } else { + (b.clone(), a.clone()) + } + }) + .collect::>(); + forbidden_pairs.sort(); + forbidden_pairs.dedup(); + TuneBoundsSpec { + allowed_backends: allowed, + forbidden_backends, + parameter_ranges, + max_experts: bounds.max_experts, + max_mixture_nesting_depth: bounds.max_mixture_nesting_depth, + min_experts: bounds.min_experts, + allow_duplicate_experts: bounds.allow_duplicate_experts, + required_experts: required, + forbidden_expert_pairs: forbidden_pairs, + } +} + +#[cfg(feature = "tuner")] +fn finite_positive(value: f64, label: &str) -> SpecResult { + if value.is_finite() && value > 0.0 { + Ok(value) + } else { + Err(SpecError::new(format!("{label} must be > 0"))) + } +} + +#[cfg(feature = "tuner")] +fn nonzero_u64(value: u64, label: &str) -> SpecResult { + if value > 0 { + Ok(value) + } else { + Err(SpecError::new(format!("{label} must be > 0"))) + } +} + +#[cfg(feature = "tuner")] +fn clean_optional_string(value: Option<&str>) -> Option { + value.and_then(|text| { + let trimmed = text.trim(); + (!trimmed.is_empty()).then(|| trimmed.to_string()) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + #[cfg(feature = "backend-ctw")] + use crate::aixi::common::MctsStrategy; + use crate::aixi::common::{ActionAlphabet, ObservationKeyMode}; + #[cfg(all(feature = "backend-ctw", feature = "tuner"))] + use crate::api::CompressionBackend; + #[cfg(feature = "backend-ctw")] + use crate::api::RateBackend; + #[cfg(feature = "backend-ctw")] + use std::num::NonZeroUsize; + + fn action_alphabet(n: usize) -> ActionAlphabet { + ActionAlphabet::try_from_usize(n).expect("test action alphabet must be non-zero") + } + + fn sample_interface() -> PlannerInterfaceSpec { + PlannerInterfaceSpec { + observation_bits: 8, + observation_stream_len: 1, + observation_key_mode: ObservationKeyMode::FullStream, + reward_bits: 8, + agent_actions: action_alphabet(2), + } + } + + #[cfg(all(feature = "backend-ctw", feature = "tuner"))] + fn sample_tune_interface() -> TunePlannerInterfaceSpec { + TunePlannerInterfaceSpec { + observation_bits: 8, + observation_stream_len: 1, + observation_key_mode: ObservationKeyMode::FullStream, + reward_bits: 8, + agent_actions: action_alphabet(2), + } + } + + #[test] + fn asset_binding_validation_and_sorting_are_stable() { + let sorted = canonicalize_assets(&[ + AssetBinding { + id: "b".to_string(), + path: "b.bin".to_string(), + }, + AssetBinding { + id: "a".to_string(), + path: "a.bin".to_string(), + }, + ]); + assert_eq!(sorted[0].id, "a"); + assert_eq!(sorted[1].id, "b"); + + validate_asset_bindings(&[ + AssetBinding { + id: "dataset".to_string(), + path: "one.bin".to_string(), + }, + AssetBinding { + id: "dataset".to_string(), + path: "one.bin".to_string(), + }, + ]) + .expect("duplicate identical bindings are benign"); + + let err = validate_asset_bindings(&[AssetBinding { + id: " ".to_string(), + path: "x".to_string(), + }]) + .expect_err("blank asset id must fail"); + assert!(err.to_string().contains("asset id cannot be empty")); + + let err = validate_asset_bindings(&[AssetBinding { + id: "dataset".to_string(), + path: " ".to_string(), + }]) + .expect_err("blank asset path must fail"); + assert!( + err.to_string() + .contains("asset 'dataset' path cannot be empty") + ); + + let err = validate_asset_bindings(&[ + AssetBinding { + id: "dataset".to_string(), + path: "one.bin".to_string(), + }, + AssetBinding { + id: "dataset".to_string(), + path: "two.bin".to_string(), + }, + ]) + .expect_err("conflicting asset bindings must fail"); + assert!( + err.to_string() + .contains("asset 'dataset' is bound to more than one path") + ); + + #[cfg(feature = "aixi")] + { + ensure_asset_exists( + &[AssetBinding { + id: "dataset".to_string(), + path: "one.bin".to_string(), + }], + "dataset", + ) + .expect("known asset id"); + let err = ensure_asset_exists(&[], "missing").expect_err("missing asset must fail"); + assert!(err.to_string().contains("unknown asset id 'missing'")); + } + } + + #[test] + fn interface_runtime_and_scalar_validators_enforce_contracts() { + canonicalize_interface_spec(&sample_interface()).expect("valid interface"); + + let mut bad_interface = sample_interface(); + bad_interface.observation_stream_len = 0; + let err = canonicalize_interface_spec(&bad_interface) + .expect_err("zero observation stream length must fail"); + assert!( + err.to_string() + .contains("observation_stream_len must be >= 1") + ); + + bad_interface = sample_interface(); + bad_interface.reward_bits = 0; + let err = + canonicalize_interface_spec(&bad_interface).expect_err("zero reward bits must fail"); + assert!(err.to_string().contains("reward_bits must be >= 1")); + + let runtime = canonicalize_runtime_spec(&PlannerRuntimeSpec { + random_seed: None, + learn_cycles: None, + eval_cycles: None, + terminate_lifetime: 4, + log_every: 2, + perf: false, + vm_perf_only: false, + explore_epsilon: 0.0, + explore_gamma: 1.0, + }) + .expect("valid runtime"); + assert_eq!(runtime.random_seed, Some(resolve_random_seed(None))); + + let err = canonicalize_runtime_spec(&PlannerRuntimeSpec { + terminate_lifetime: 0, + ..runtime.clone() + }) + .expect_err("zero terminate_lifetime must fail"); + assert!(err.to_string().contains("terminate_lifetime must be >= 1")); + + let err = canonicalize_runtime_spec(&PlannerRuntimeSpec { + log_every: 0, + ..runtime.clone() + }) + .expect_err("zero log_every must fail"); + assert!(err.to_string().contains("log_every must be >= 1")); + + let err = canonicalize_runtime_spec(&PlannerRuntimeSpec { + explore_epsilon: -0.1, + ..runtime.clone() + }) + .expect_err("negative explore_epsilon must fail"); + assert!(err.to_string().contains("explore_epsilon must be >= 0")); + + let err = canonicalize_runtime_spec(&PlannerRuntimeSpec { + explore_gamma: 0.0, + ..runtime + }) + .expect_err("non-positive explore_gamma must fail"); + assert!(err.to_string().contains("explore_gamma must be > 0")); + + #[cfg(feature = "tuner")] + { + assert_eq!(finite_positive(0.5, "x").expect("positive finite"), 0.5); + assert!(finite_positive(f64::INFINITY, "x").is_err()); + assert_eq!(nonzero_u64(7, "y").expect("nonzero"), 7); + assert!(nonzero_u64(0, "y").is_err()); + assert_eq!( + clean_optional_string(Some(" trimmed ")), + Some("trimmed".to_string()) + ); + assert_eq!(clean_optional_string(Some(" ")), None); + assert_eq!(clean_optional_string(None), None); + } + } + + #[cfg(feature = "backend-ctw")] + #[test] + fn planner_controller_validation_covers_mc_aixi_and_aiqi_contracts() { + let env = SpecEnvironment::default(); + let workers = NonZeroUsize::new(2).expect("non-zero workers"); + + canonicalize_controller_spec( + &ControllerSpec::McAixi(super::super::McAixiControllerSpec { + predictor: RateBackend::Ctw { depth: 4 }, + bit_stream_semantics: crate::api::BitStreamSemantics::BinaryTokens, + agent_horizon: 2, + num_simulations: 8, + mcts_strategy: MctsStrategy::ParallelUct { + workers, + bu_uct_m_max: Some(0.5), + }, + exploration_exploitation_ratio: 1.0, + discount_gamma: 0.8, + }), + &[], + &env, + None, + ) + .expect("valid MC-AIXI controller"); + + let err = match canonicalize_controller_spec( + &ControllerSpec::McAixi(super::super::McAixiControllerSpec { + predictor: RateBackend::Ctw { depth: 4 }, + bit_stream_semantics: crate::api::BitStreamSemantics::BinaryTokens, + agent_horizon: 0, + num_simulations: 8, + mcts_strategy: MctsStrategy::RhoUct, + exploration_exploitation_ratio: 1.0, + discount_gamma: 0.8, + }), + &[], + &env, + None, + ) { + Ok(_) => panic!("zero horizon must fail"), + Err(err) => err, + }; + assert!(err.to_string().contains("agent_horizon must be >= 1")); + + let err = match canonicalize_controller_spec( + &ControllerSpec::AiqiDiscounted(super::super::AiqiDiscountedControllerSpec { + predictor: RateBackend::Ctw { depth: 4 }, + bit_stream_semantics: crate::api::BitStreamSemantics::BinaryTokens, + discount_gamma: 1.0, + return_horizon: 2, + return_bins: 8, + augmentation_period: 2, + history_prune_keep_steps: None, + baseline_exploration: 0.1, + }), + &[], + &env, + None, + ) { + Ok(_) => panic!("discount_gamma=1 must fail"), + Err(err) => err, + }; + assert!(err.to_string().contains("discount_gamma must be in (0, 1)")); + } + + #[cfg(all(feature = "backend-ctw", feature = "aixi"))] + #[test] + fn planner_controller_validation_covers_warmstart_contract() { + let env = SpecEnvironment::default(); + let warmstart_assets = vec![AssetBinding { + id: "teacher-ds".to_string(), + path: "teacher.json".to_string(), + }]; + + let warmstart = canonicalize_controller_spec( + &ControllerSpec::AiqiWarmstartExactJh(super::super::WarmStartExactJhControllerSpec { + predictor: RateBackend::Ctw { depth: 4 }, + bit_stream_semantics: crate::api::BitStreamSemantics::BinaryTokens, + return_horizon: 2, + return_bins: 5, + label_phase_period: 3, + teacher_dataset_asset: " teacher-ds ".to_string(), + planner_simulations_per_step: 1, + }), + &warmstart_assets, + &env, + Some(&sample_interface()), + ) + .expect("valid warmstart controller"); + match warmstart { + ControllerSpec::AiqiWarmstartExactJh(inner) => { + assert_eq!(inner.teacher_dataset_asset, "teacher-ds"); + } + _ => panic!("expected warmstart controller"), + } + + let err = match canonicalize_controller_spec( + &ControllerSpec::AiqiWarmstartExactJh(super::super::WarmStartExactJhControllerSpec { + predictor: RateBackend::Ctw { depth: 4 }, + bit_stream_semantics: crate::api::BitStreamSemantics::BinaryTokens, + return_horizon: 4, + return_bins: 8, + label_phase_period: 4, + teacher_dataset_asset: "teacher-ds".to_string(), + planner_simulations_per_step: 1, + }), + &warmstart_assets, + &env, + Some(&sample_interface()), + ) { + Ok(_) => panic!("warmstart slack return bins must fail"), + Err(err) => err, + }; + assert!( + err.to_string() + .contains("return_bins must be exactly H * max_reward + 1"), + "{err}" + ); + + let err = match canonicalize_controller_spec( + &ControllerSpec::AiqiWarmstartExactJh(super::super::WarmStartExactJhControllerSpec { + predictor: RateBackend::Ctw { depth: 4 }, + bit_stream_semantics: crate::api::BitStreamSemantics::BinaryTokens, + return_horizon: 2, + return_bins: 513, + label_phase_period: 3, + teacher_dataset_asset: "teacher-ds".to_string(), + planner_simulations_per_step: 1, + }), + &warmstart_assets, + &env, + Some(&PlannerInterfaceSpec { + reward_bits: 8, + ..sample_interface() + }), + ) { + Ok(_) => panic!("warmstart reward_bits too narrow must fail at spec time"), + Err(err) => err, + }; + assert!( + err.to_string() + .contains("not representable by reward_bits=8"), + "{err}" + ); + + let err = match canonicalize_controller_spec( + &ControllerSpec::AiqiWarmstartExactJh(super::super::WarmStartExactJhControllerSpec { + predictor: RateBackend::Ctw { depth: 4 }, + bit_stream_semantics: crate::api::BitStreamSemantics::BinaryTokens, + return_horizon: 2, + return_bins: 5, + label_phase_period: 3, + teacher_dataset_asset: "missing-teacher".to_string(), + planner_simulations_per_step: 1, + }), + &warmstart_assets, + &env, + Some(&sample_interface()), + ) { + Ok(_) => panic!("warmstart missing teacher asset must fail"), + Err(err) => err, + }; + assert!( + err.to_string() + .contains("unknown asset id 'missing-teacher'"), + "{err}" + ); + + let err = match canonicalize_controller_spec( + &ControllerSpec::AiqiWarmstartExactJh(super::super::WarmStartExactJhControllerSpec { + predictor: RateBackend::Ctw { depth: 4 }, + bit_stream_semantics: crate::api::BitStreamSemantics::BinaryTokens, + return_horizon: 2, + return_bins: 5, + label_phase_period: 3, + teacher_dataset_asset: " ".to_string(), + planner_simulations_per_step: 1, + }), + &warmstart_assets, + &env, + Some(&sample_interface()), + ) { + Ok(_) => panic!("warmstart blank teacher asset must fail"), + Err(err) => err, + }; + assert!( + err.to_string() + .contains("teacher_dataset_asset cannot be empty"), + "{err}" + ); + + let err = match canonicalize_controller_spec( + &ControllerSpec::AiqiWarmstartExactJh(super::super::WarmStartExactJhControllerSpec { + predictor: RateBackend::Ctw { depth: 4 }, + bit_stream_semantics: crate::api::BitStreamSemantics::BinaryTokens, + return_horizon: 2, + return_bins: 5, + label_phase_period: 3, + teacher_dataset_asset: "teacher-ds".to_string(), + planner_simulations_per_step: 2, + }), + &warmstart_assets, + &env, + Some(&sample_interface()), + ) { + Ok(_) => panic!("warmstart non-direct planner marker must fail"), + Err(err) => err, + }; + assert!( + err.to_string() + .contains("planner_simulations_per_step must be exactly 1"), + "{err}" + ); + } + + #[cfg(all(feature = "backend-ctw", feature = "tuner"))] + #[test] + fn tune_controller_and_bounds_validation_cover_semantic_errors() { + let assets = vec![AssetBinding { + id: "teacher".to_string(), + path: "teacher.bin".to_string(), + }]; + let env = SpecEnvironment::default(); + + let annealed = canonicalize_tune_controller( + &TuneControllerSpec::AnnealedHillClimbing( + super::super::AnnealedHillClimbingTuneControllerSpec { + max_mutation_radius: 2, + }, + ), + &assets, + &env, + ) + .expect("valid annealed controller"); + assert!(matches!( + annealed, + TuneControllerSpec::AnnealedHillClimbing(_) + )); + + canonicalize_tune_controller( + &TuneControllerSpec::AiqiDiscounted(super::super::AiqiDiscountedTuneControllerSpec { + interface: sample_tune_interface(), + planner_simulations_per_step: 2, + return_horizon: 2, + return_bins: 3, + discount_factor: 0.5, + min_improvement: -1.0, + max_improvement: 1.0, + }), + &assets, + &env, + ) + .expect("non-power-of-two bins are valid for discounted AIQI"); + + let err = canonicalize_tune_controller( + &TuneControllerSpec::AiqiDiscounted(super::super::AiqiDiscountedTuneControllerSpec { + interface: sample_tune_interface(), + planner_simulations_per_step: 2, + return_horizon: 2, + return_bins: 4, + discount_factor: 0.5, + min_improvement: f64::NAN, + max_improvement: 1.0, + }), + &assets, + &env, + ) + .expect_err("NaN min_improvement must fail"); + assert!(err.to_string().contains("min_improvement must be finite")); + + let err = canonicalize_tune_controller( + &TuneControllerSpec::AiqiDiscounted(super::super::AiqiDiscountedTuneControllerSpec { + interface: sample_tune_interface(), + planner_simulations_per_step: 2, + return_horizon: 2, + return_bins: 4, + discount_factor: 0.5, + min_improvement: f64::INFINITY, + max_improvement: 1.0, + }), + &assets, + &env, + ) + .expect_err("infinite min_improvement must fail"); + assert!(err.to_string().contains("min_improvement must be finite")); + + let err = canonicalize_tune_controller( + &TuneControllerSpec::AiqiDiscounted(super::super::AiqiDiscountedTuneControllerSpec { + interface: sample_tune_interface(), + planner_simulations_per_step: 2, + return_horizon: 2, + return_bins: 4, + discount_factor: 0.5, + min_improvement: -1.0, + max_improvement: f64::NAN, + }), + &assets, + &env, + ) + .expect_err("NaN max_improvement must fail"); + assert!(err.to_string().contains("max_improvement must be finite")); + + let err = canonicalize_tune_controller( + &TuneControllerSpec::AiqiDiscounted(super::super::AiqiDiscountedTuneControllerSpec { + interface: sample_tune_interface(), + planner_simulations_per_step: 2, + return_horizon: 2, + return_bins: 4, + discount_factor: 0.5, + min_improvement: -1.0, + max_improvement: f64::INFINITY, + }), + &assets, + &env, + ) + .expect_err("infinite max_improvement must fail"); + assert!(err.to_string().contains("max_improvement must be finite")); + + validate_tune_bounds(&TuneBoundsSpec { + allowed_backends: vec!["ctw".to_string()], + forbidden_backends: vec!["zpaq".to_string()], + parameter_ranges: vec![super::super::TuneParameterRangeSpec { + parameter: "alpha".to_string(), + min: 0.1, + max: 0.2, + }], + max_experts: 4, + max_mixture_nesting_depth: 2, + min_experts: Some(1), + allow_duplicate_experts: Some(false), + required_experts: vec!["ctw".to_string()], + forbidden_expert_pairs: vec![("zpaq".to_string(), "ctw".to_string())], + }) + .expect("valid bounds"); + + let canonical = canonicalize_tune_bounds(&TuneBoundsSpec { + allowed_backends: vec!["ctw".to_string(), "ctw".to_string(), "rosa".to_string()], + forbidden_backends: vec!["zpaq".to_string(), "zpaq".to_string()], + parameter_ranges: vec![ + super::super::TuneParameterRangeSpec { + parameter: "beta".to_string(), + min: 0.2, + max: 0.4, + }, + super::super::TuneParameterRangeSpec { + parameter: "alpha".to_string(), + min: 0.1, + max: 0.3, + }, + ], + max_experts: 4, + max_mixture_nesting_depth: 2, + min_experts: Some(1), + allow_duplicate_experts: Some(false), + required_experts: vec!["rosa".to_string(), "rosa".to_string()], + forbidden_expert_pairs: vec![ + ("zpaq".to_string(), "ctw".to_string()), + ("ctw".to_string(), "zpaq".to_string()), + ], + }); + assert_eq!(canonical.allowed_backends, vec!["ctw", "rosa"]); + assert_eq!(canonical.forbidden_backends, vec!["zpaq"]); + assert_eq!(canonical.required_experts, vec!["rosa"]); + assert_eq!( + canonical.forbidden_expert_pairs, + vec![("ctw".to_string(), "zpaq".to_string())] + ); + + let err = validate_tune_bounds(&TuneBoundsSpec { + allowed_backends: vec!["ctw".to_string()], + forbidden_backends: vec!["ctw".to_string()], + parameter_ranges: vec![], + max_experts: 4, + max_mixture_nesting_depth: 2, + min_experts: Some(1), + allow_duplicate_experts: None, + required_experts: vec![], + forbidden_expert_pairs: vec![], + }) + .expect_err("overlapping allow/forbid bounds must fail"); + assert!( + err.to_string() + .contains("allowed_backends and forbidden_backends cannot overlap") + ); + + let compiled = compile_tune_controller(&TuneControllerSpec::AnnealedHillClimbing( + super::super::AnnealedHillClimbingTuneControllerSpec { + max_mutation_radius: 2, + }, + )); + assert!(matches!( + compiled, + CompiledTuneController::AnnealedHillClimbing(_) + )); + + let baseline = CompressionBackend::Rate { + rate_backend: RateBackend::Ctw { depth: 4 }, + coder: crate::coders::CoderType::AC, + framing: crate::compression::FramingMode::Framed, + }; + let tune = TuneSpec { + assets: assets.clone(), + input_asset: "teacher".to_string(), + baseline_candidate: baseline, + controller: TuneControllerSpec::AnnealedHillClimbing( + super::super::AnnealedHillClimbingTuneControllerSpec { + max_mutation_radius: 2, + }, + ), + bounds: TuneBoundsSpec { + allowed_backends: vec!["ctw".to_string()], + forbidden_backends: vec![], + parameter_ranges: vec![], + max_experts: 4, + max_mixture_nesting_depth: 2, + min_experts: Some(1), + allow_duplicate_experts: Some(false), + required_experts: vec![], + forbidden_expert_pairs: vec![], + }, + eval_time_limit_seconds: 1.0, + time_budget_seconds: 5.0, + min_throughput_bytes_per_second: 1.0, + max_memory_bytes: 1024, + output_config_path: " out.json ".to_string(), + seed: 9, + report_path: Some(" report.json ".to_string()), + }; + let compiled = compile_tune_spec(&tune, Path::new(".")).expect("compile tune spec"); + assert_eq!(compiled.canonical_spec().output_config_path, "out.json"); + } + + #[cfg(all(feature = "backend-ctw", feature = "tuner"))] + #[test] + fn tune_controller_validation_covers_warmstart_contract() { + let assets = vec![AssetBinding { + id: "teacher".to_string(), + path: "teacher.bin".to_string(), + }]; + let env = SpecEnvironment::default(); + + let err = canonicalize_tune_controller( + &TuneControllerSpec::AiqiWarmstartExactJh( + super::super::WarmStartExactJhTuneControllerSpec { + interface: sample_tune_interface(), + planner_simulations_per_step: 1, + return_horizon: 2, + warmstart_teacher_dataset_asset: "missing".to_string(), + label_phase_period: 3, + }, + ), + &assets, + &env, + ) + .expect_err("missing teacher asset must fail"); + assert!(err.to_string().contains("unknown asset id 'missing'")); + + let err = canonicalize_tune_controller( + &TuneControllerSpec::AiqiWarmstartExactJh( + super::super::WarmStartExactJhTuneControllerSpec { + interface: sample_tune_interface(), + planner_simulations_per_step: 2, + return_horizon: 2, + warmstart_teacher_dataset_asset: "teacher".to_string(), + label_phase_period: 3, + }, + ), + &assets, + &env, + ) + .expect_err("warmstart tune non-direct planner marker must fail"); + assert!( + err.to_string() + .contains("planner_simulations_per_step must be exactly 1"), + "{err}" + ); + } +} diff --git a/crates/infotheory/src/spec/document/serializer.rs b/crates/infotheory/src/spec/document/serializer.rs new file mode 100644 index 00000000..45e69778 --- /dev/null +++ b/crates/infotheory/src/spec/document/serializer.rs @@ -0,0 +1,379 @@ +//! Canonical JSON serialization for spec documents. + +use super::{ + AssetBinding, ControllerSpec, EnvironmentSpec, PlannerInterfaceSpec, PlannerRunSpec, + PlannerRuntimeSpec, SPEC_DOCUMENT_SCHEMA_VERSION, SpecDocument, SpecResult, + compression_backend_to_json_value, rate_backend_to_json_value, +}; +#[cfg(feature = "tuner")] +use super::{ + TuneBoundsSpec, TuneControllerSpec, TuneParameterRangeSpec, TunePlannerInterfaceSpec, TuneSpec, +}; +use crate::aixi::common::MctsStrategy; +use crate::api::{BitOrder, BitStreamSemantics}; + +#[cfg(feature = "vm")] +use super::{ + VmActionFilterSpec, VmRewardPolicySpec, VmRewardShapingSpec, VmRuntimeActionSourceSpec, + VmTraceSpec, +}; + +pub(super) fn spec_document_to_json_value(doc: &SpecDocument) -> SpecResult { + match doc { + SpecDocument::PlannerRun(spec) => planner_run_to_json_value(spec), + #[cfg(feature = "tuner")] + SpecDocument::Tune(spec) => tune_spec_to_json_value(spec), + SpecDocument::RateBackend(backend) => Ok(serde_json::json!({ + "schema_version": SPEC_DOCUMENT_SCHEMA_VERSION, + "kind": "rate_backend", + "backend": rate_backend_to_json_value(backend)?, + })), + SpecDocument::CompressionBackend(backend) => Ok(serde_json::json!({ + "schema_version": SPEC_DOCUMENT_SCHEMA_VERSION, + "kind": "compression_backend", + "backend": compression_backend_to_json_value(backend)?, + })), + } +} + +pub(super) fn planner_run_to_json_value(spec: &PlannerRunSpec) -> SpecResult { + Ok(serde_json::json!({ + "schema_version": SPEC_DOCUMENT_SCHEMA_VERSION, + "kind": "planner_run", + "assets": spec.assets.iter().map(asset_binding_to_json_value).collect::>(), + "environment": environment_spec_to_json_value(&spec.environment)?, + "interface": interface_spec_to_json_value(&spec.interface), + "controller": controller_spec_to_json_value(&spec.controller)?, + "runtime": runtime_spec_to_json_value(&spec.runtime), + })) +} + +#[cfg(feature = "tuner")] +pub(super) fn tune_spec_to_json_value(spec: &TuneSpec) -> SpecResult { + Ok(serde_json::json!({ + "schema_version": SPEC_DOCUMENT_SCHEMA_VERSION, + "kind": "tune", + "assets": spec.assets.iter().map(asset_binding_to_json_value).collect::>(), + "input_asset": spec.input_asset, + "baseline_candidate": compression_backend_to_json_value(&spec.baseline_candidate)?, + "controller": tune_controller_to_json_value(&spec.controller), + "bounds": tune_bounds_to_json_value(&spec.bounds), + "eval_time_limit_seconds": spec.eval_time_limit_seconds, + "time_budget_seconds": spec.time_budget_seconds, + "min_throughput_bytes_per_second": spec.min_throughput_bytes_per_second, + "max_memory_bytes": spec.max_memory_bytes, + "output_config_path": spec.output_config_path, + "seed": spec.seed, + "report_path": spec.report_path, + })) +} + +fn asset_binding_to_json_value(binding: &AssetBinding) -> serde_json::Value { + serde_json::json!({ + "id": binding.id, + "path": binding.path, + }) +} + +fn environment_spec_to_json_value(spec: &EnvironmentSpec) -> SpecResult { + match spec { + EnvironmentSpec::Builtin { builtin } => Ok(serde_json::json!({ + "kind": "builtin", + "name": super::builtin_environment_name(*builtin), + })), + #[cfg(feature = "vm")] + EnvironmentSpec::NyxVm(vm) => Ok(serde_json::json!({ + "kind": "nyx_vm", + "firecracker_config_asset": vm.firecracker_config_asset, + "instance_id": vm.instance_id, + "shared_region_name": vm.shared_region_name, + "shared_region_size": vm.shared_region_size, + "shared_memory_policy": super::shared_memory_policy_name(vm.shared_memory_policy), + "step_timeout_ms": vm.step_timeout_ms, + "boot_timeout_ms": vm.boot_timeout_ms, + "episode_steps": vm.episode_steps, + "step_cost": vm.step_cost, + "observation_policy": super::vm_observation_policy_name(vm.observation_policy), + "observation_bits": vm.observation_bits, + "observation_stream_len": vm.observation_stream_len, + "observation_stream_mode": super::vm_observation_stream_mode_name(vm.observation_stream_mode), + "observation_pad_byte": vm.observation_pad_byte, + "reward_bits": vm.reward_bits, + "reward_policy": vm_reward_policy_to_json_value(&vm.reward_policy), + "reward_shaping": vm.reward_shaping.as_ref().map(vm_reward_shaping_to_json_value), + "action_source": vm_action_source_to_json_value(&vm.action_source), + "action_filter": vm.action_filter.as_ref().map(vm_action_filter_to_json_value), + "protocol": { + "action_prefix": vm.action_prefix, + "action_suffix": vm.action_suffix, + "obs_prefix": vm.obs_prefix, + "rew_prefix": vm.rew_prefix, + "done_prefix": vm.done_prefix, + "data_prefix": vm.data_prefix, + "wire_encoding": super::vm_payload_encoding_name(vm.wire_encoding), + }, + "stats_backend": rate_backend_to_json_value(&vm.stats_backend)?, + "trace": vm.trace.as_ref().map(vm_trace_to_json_value), + "debug_mode": vm.debug_mode, + "crash_log": vm.crash_log, + })), + } +} + +fn interface_spec_to_json_value(spec: &PlannerInterfaceSpec) -> serde_json::Value { + serde_json::json!({ + "observation_bits": spec.observation_bits, + "observation_stream_len": spec.observation_stream_len, + "observation_key_mode": super::observation_key_mode_name(spec.observation_key_mode), + "reward_bits": spec.reward_bits, + "agent_actions": spec.agent_actions.get(), + }) +} + +#[cfg(feature = "tuner")] +fn tune_interface_spec_to_json_value(spec: &TunePlannerInterfaceSpec) -> serde_json::Value { + serde_json::json!({ + "observation_bits": spec.observation_bits, + "observation_stream_len": spec.observation_stream_len, + "observation_key_mode": super::observation_key_mode_name(spec.observation_key_mode), + "reward_bits": spec.reward_bits, + "agent_actions": spec.agent_actions.get(), + }) +} + +fn controller_spec_to_json_value(spec: &ControllerSpec) -> SpecResult { + match spec { + ControllerSpec::McAixi(inner) => Ok(serde_json::json!({ + "kind": "mc_aixi", + "predictor": rate_backend_to_json_value(&inner.predictor)?, + "bit_stream_semantics": bit_stream_semantics_to_json_value(inner.bit_stream_semantics), + "agent_horizon": inner.agent_horizon, + "num_simulations": inner.num_simulations, + "mcts_strategy": mcts_strategy_to_json_value(inner.mcts_strategy), + "exploration_exploitation_ratio": inner.exploration_exploitation_ratio, + "discount_gamma": inner.discount_gamma, + })), + ControllerSpec::AiqiDiscounted(inner) => Ok(serde_json::json!({ + "kind": "aiqi_discounted", + "predictor": rate_backend_to_json_value(&inner.predictor)?, + "bit_stream_semantics": bit_stream_semantics_to_json_value(inner.bit_stream_semantics), + "discount_gamma": inner.discount_gamma, + "return_horizon": inner.return_horizon, + "return_bins": inner.return_bins, + "augmentation_period": inner.augmentation_period, + "history_prune_keep_steps": inner.history_prune_keep_steps, + "baseline_exploration": inner.baseline_exploration, + })), + #[cfg(feature = "aixi")] + ControllerSpec::AiqiWarmstartExactJh(inner) => Ok(serde_json::json!({ + "kind": "aiqi_warmstart_exact_jh", + "predictor": rate_backend_to_json_value(&inner.predictor)?, + "bit_stream_semantics": bit_stream_semantics_to_json_value(inner.bit_stream_semantics), + "return_horizon": inner.return_horizon, + "return_bins": inner.return_bins, + "label_phase_period": inner.label_phase_period, + "teacher_dataset_asset": inner.teacher_dataset_asset, + "planner_simulations_per_step": inner.planner_simulations_per_step, + })), + } +} + +fn bit_stream_semantics_to_json_value(semantics: BitStreamSemantics) -> serde_json::Value { + match semantics { + BitStreamSemantics::BytePacked { order } => serde_json::json!({ + "kind": "byte_packed", + "order": match order { + BitOrder::MsbFirst => "msb_first", + BitOrder::LsbFirst => "lsb_first", + }, + }), + BitStreamSemantics::BinaryTokens => serde_json::json!({ + "kind": "binary_tokens", + }), + } +} + +fn mcts_strategy_to_json_value(strategy: MctsStrategy) -> serde_json::Value { + match strategy { + MctsStrategy::RhoUct => serde_json::json!({ + "kind": "rho_uct", + }), + MctsStrategy::ParallelUct { + workers, + bu_uct_m_max, + } => serde_json::json!({ + "kind": "parallel_uct", + "workers": workers.get(), + "bu_uct_m_max": bu_uct_m_max, + }), + } +} + +fn runtime_spec_to_json_value(spec: &PlannerRuntimeSpec) -> serde_json::Value { + serde_json::json!({ + "random_seed": spec.random_seed, + "learn_cycles": spec.learn_cycles, + "eval_cycles": spec.eval_cycles, + "terminate_lifetime": spec.terminate_lifetime, + "log_every": spec.log_every, + "perf": spec.perf, + "vm_perf_only": spec.vm_perf_only, + "explore_epsilon": spec.explore_epsilon, + "explore_gamma": spec.explore_gamma, + }) +} + +#[cfg(feature = "tuner")] +fn tune_bounds_to_json_value(bounds: &TuneBoundsSpec) -> serde_json::Value { + serde_json::json!({ + "allowed_backends": bounds.allowed_backends, + "forbidden_backends": bounds.forbidden_backends, + "parameter_ranges": bounds.parameter_ranges.iter().map(tune_parameter_range_to_json_value).collect::>(), + "max_experts": bounds.max_experts, + "max_mixture_nesting_depth": bounds.max_mixture_nesting_depth, + "min_experts": bounds.min_experts, + "allow_duplicate_experts": bounds.allow_duplicate_experts, + "required_experts": bounds.required_experts, + "forbidden_expert_pairs": bounds.forbidden_expert_pairs.iter().map(|(a, b)| vec![a, b]).collect::>(), + }) +} + +#[cfg(feature = "tuner")] +fn tune_controller_to_json_value(spec: &TuneControllerSpec) -> serde_json::Value { + match spec { + TuneControllerSpec::AnnealedHillClimbing(inner) => serde_json::json!({ + "kind": "annealed_hill_climbing", + "max_mutation_radius": inner.max_mutation_radius, + }), + TuneControllerSpec::McAixiFacCtw(inner) => serde_json::json!({ + "kind": "mc_aixi_fac_ctw", + "interface": tune_interface_spec_to_json_value(&inner.interface), + "planner_simulations_per_step": inner.planner_simulations_per_step, + }), + TuneControllerSpec::AiqiDiscounted(inner) => serde_json::json!({ + "kind": "aiqi_discounted", + "interface": tune_interface_spec_to_json_value(&inner.interface), + "planner_simulations_per_step": inner.planner_simulations_per_step, + "return_horizon": inner.return_horizon, + "return_bins": inner.return_bins, + "discount_factor": inner.discount_factor, + "min_improvement": inner.min_improvement, + "max_improvement": inner.max_improvement, + }), + #[cfg(feature = "aixi")] + TuneControllerSpec::AiqiWarmstartExactJh(inner) => serde_json::json!({ + "kind": "aiqi_warmstart_exact_jh", + "interface": tune_interface_spec_to_json_value(&inner.interface), + "planner_simulations_per_step": inner.planner_simulations_per_step, + "return_horizon": inner.return_horizon, + "warmstart_teacher_dataset_asset": inner.warmstart_teacher_dataset_asset, + "label_phase_period": inner.label_phase_period, + }), + } +} + +#[cfg(feature = "tuner")] +fn tune_parameter_range_to_json_value(range: &TuneParameterRangeSpec) -> serde_json::Value { + serde_json::json!({ + "parameter": range.parameter, + "min": range.min, + "max": range.max, + }) +} + +#[cfg(feature = "vm")] +fn vm_reward_policy_to_json_value(policy: &VmRewardPolicySpec) -> serde_json::Value { + match policy { + VmRewardPolicySpec::FromGuest => serde_json::json!({ "kind": "from_guest" }), + VmRewardPolicySpec::Pattern { + pattern, + base_reward, + bonus_reward, + } => serde_json::json!({ + "kind": "pattern", + "pattern": pattern, + "base_reward": base_reward, + "bonus_reward": bonus_reward, + }), + } +} + +#[cfg(feature = "vm")] +fn vm_reward_shaping_to_json_value(spec: &VmRewardShapingSpec) -> serde_json::Value { + match spec { + VmRewardShapingSpec::EntropyReduction { + baseline_asset, + scale, + crash_bonus, + timeout_bonus, + } => serde_json::json!({ + "kind": "entropy_reduction", + "baseline_asset": baseline_asset, + "scale": scale, + "crash_bonus": crash_bonus, + "timeout_bonus": timeout_bonus, + }), + VmRewardShapingSpec::TraceEntropy { scale, normalize } => serde_json::json!({ + "kind": "trace_entropy", + "scale": scale, + "normalize": normalize, + }), + } +} + +#[cfg(feature = "vm")] +fn vm_action_source_to_json_value(spec: &VmRuntimeActionSourceSpec) -> serde_json::Value { + match spec { + VmRuntimeActionSourceSpec::Literal { + names, + payloads, + encoding, + } => serde_json::json!({ + "kind": "literal", + "encoding": super::vm_payload_encoding_name(*encoding), + "actions": payloads.iter().enumerate().map(|(idx, payload)| serde_json::json!({ + "name": names.get(idx).cloned().flatten(), + "payload": payload, + })).collect::>(), + }), + VmRuntimeActionSourceSpec::Fuzz { + seeds, + encoding, + mutators, + min_len, + max_len, + dictionary, + rng_seed, + } => serde_json::json!({ + "kind": "fuzz", + "encoding": super::vm_payload_encoding_name(*encoding), + "seeds": seeds, + "mutators": mutators.iter().map(|mutator| super::vm_fuzz_mutator_name(*mutator)).collect::>(), + "min_len": min_len, + "max_len": max_len, + "dictionary": dictionary, + "rng_seed": rng_seed, + }), + } +} + +#[cfg(feature = "vm")] +fn vm_action_filter_to_json_value(spec: &VmActionFilterSpec) -> serde_json::Value { + serde_json::json!({ + "min_entropy": spec.min_entropy, + "max_entropy": spec.max_entropy, + "min_intrinsic_dependence": spec.min_intrinsic_dependence, + "min_novelty": spec.min_novelty, + "novelty_prior_asset": spec.novelty_prior_asset, + "reject_reward": spec.reject_reward, + }) +} + +#[cfg(feature = "vm")] +fn vm_trace_to_json_value(spec: &VmTraceSpec) -> serde_json::Value { + serde_json::json!({ + "shared_region_name": spec.shared_region_name, + "max_bytes": spec.max_bytes, + "reset_on_episode": spec.reset_on_episode, + }) +} diff --git a/crates/infotheory/src/spec/document/tests.rs b/crates/infotheory/src/spec/document/tests.rs new file mode 100644 index 00000000..1a2f3c08 --- /dev/null +++ b/crates/infotheory/src/spec/document/tests.rs @@ -0,0 +1,2027 @@ +//! Tests for canonical top-level specification documents. + +use super::*; +#[cfg(any(feature = "backend-ctw", feature = "all-backends"))] +use crate::aixi::common::MctsStrategy; +use crate::aixi::common::{ActionAlphabet, ObservationKeyMode}; +#[cfg(feature = "backend-ctw")] +use crate::aixi::common::{ + DEFAULT_RANDOM_SEED, parallel_uct_workers_one_warning_count_for_tests, + reset_parallel_uct_workers_one_warning_for_tests, +}; +#[cfg_attr(not(feature = "tuner"), allow(unused_imports))] +#[cfg(feature = "backend-ctw")] +use crate::api::CompressionBackend; +use crate::api::RateBackend; +#[cfg(feature = "all-backends")] +use crate::api::{ + CalibratedSpec, CalibrationContextKind, MixtureExpertSpec, MixtureKind, MixtureScheduleMode, + MixtureSpec, ParticleSpec, +}; +#[cfg(any(feature = "backend-mamba", feature = "backend-rwkv"))] +use crate::backends::llm_policy::{ + LlmPolicy, OptimizerHyperParams, OptimizerKind, PolicyAction, PolicyRule, PositionExpr, + RepeatRule, RepeatSegment, ScheduleRule, TrainAction, TrainScopeSet, +}; +#[cfg(feature = "backend-ctw")] +use std::num::NonZeroUsize; +#[cfg(feature = "all-backends")] +use std::sync::Arc; + +fn default_bit_stream_semantics() -> crate::api::BitStreamSemantics { + crate::api::BitStreamSemantics::BinaryTokens +} + +#[cfg(feature = "backend-ctw")] +fn nz(n: usize) -> NonZeroUsize { + NonZeroUsize::new(n).expect("test fixture worker count must be non-zero") +} + +fn action_alphabet(n: usize) -> ActionAlphabet { + ActionAlphabet::try_from_usize(n).expect("test fixture action alphabet must be non-zero") +} + +#[cfg(all(feature = "backend-ctw", feature = "tuner"))] +fn sample_tune_spec() -> TuneSpec { + TuneSpec { + assets: vec![AssetBinding { + id: "dataset".to_string(), + path: "input.bin".to_string(), + }], + input_asset: "dataset".to_string(), + baseline_candidate: CompressionBackend::Rate { + rate_backend: RateBackend::Ctw { depth: 8 }, + coder: crate::coders::CoderType::AC, + framing: crate::compression::FramingMode::Framed, + }, + controller: TuneControllerSpec::AnnealedHillClimbing( + AnnealedHillClimbingTuneControllerSpec { + max_mutation_radius: 2, + }, + ), + bounds: TuneBoundsSpec { + allowed_backends: vec!["ctw".to_string()], + forbidden_backends: vec!["zpaq".to_string()], + parameter_ranges: vec![TuneParameterRangeSpec { + parameter: "mixture.alpha".to_string(), + min: 0.1, + max: 0.5, + }], + max_experts: 4, + max_mixture_nesting_depth: 2, + min_experts: Some(1), + allow_duplicate_experts: Some(false), + required_experts: vec!["ctw".to_string()], + forbidden_expert_pairs: vec![], + }, + eval_time_limit_seconds: 1.0, + time_budget_seconds: 10.0, + min_throughput_bytes_per_second: 1024.0, + max_memory_bytes: 1 << 20, + output_config_path: "best.json".to_string(), + seed: 7, + report_path: Some("report.json".to_string()), + } +} + +#[cfg(feature = "backend-ctw")] +fn sample_planner_run() -> PlannerRunSpec { + PlannerRunSpec { + assets: Vec::new(), + environment: EnvironmentSpec::Builtin { + builtin: BuiltinEnvironmentSpec::CoinFlip, + }, + interface: PlannerInterfaceSpec { + observation_bits: 1, + observation_stream_len: 1, + observation_key_mode: ObservationKeyMode::FullStream, + reward_bits: 1, + agent_actions: action_alphabet(2), + }, + controller: ControllerSpec::AiqiDiscounted(AiqiDiscountedControllerSpec { + predictor: RateBackend::Ctw { depth: 8 }, + bit_stream_semantics: default_bit_stream_semantics(), + discount_gamma: 0.99, + return_horizon: 2, + return_bins: 8, + augmentation_period: 2, + history_prune_keep_steps: None, + baseline_exploration: 0.01, + }), + runtime: PlannerRuntimeSpec { + random_seed: Some(7), + learn_cycles: Some(4), + eval_cycles: Some(2), + terminate_lifetime: 4, + log_every: 1, + perf: false, + vm_perf_only: false, + explore_epsilon: 0.0, + explore_gamma: 1.0, + }, + } +} + +#[cfg(feature = "backend-ctw")] +fn sample_mc_aixi_planner_run(mcts_strategy: MctsStrategy) -> PlannerRunSpec { + let mut spec = sample_planner_run(); + spec.controller = ControllerSpec::McAixi(McAixiControllerSpec { + predictor: RateBackend::Ctw { depth: 8 }, + bit_stream_semantics: default_bit_stream_semantics(), + agent_horizon: 2, + num_simulations: 4, + mcts_strategy, + exploration_exploitation_ratio: 1.0, + discount_gamma: 0.95, + }); + spec +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn planner_run_parser_accepts_canonical_builtin_names() { + let names = [ + ("coin_flip", BuiltinEnvironmentSpec::CoinFlip), + ( + "biased_rock_paper_scissor", + BuiltinEnvironmentSpec::BiasedRockPaperScissor, + ), + ("kuhn_poker", BuiltinEnvironmentSpec::KuhnPoker), + ("extended_tiger", BuiltinEnvironmentSpec::ExtendedTiger), + ("tic_tac_toe", BuiltinEnvironmentSpec::TicTacToe), + ("blackjack", BuiltinEnvironmentSpec::Blackjack), + ("platformer", BuiltinEnvironmentSpec::Platformer), + ]; + + for (name, expected_builtin) in names { + let mut value = sample_planner_run() + .to_canonical_json_value() + .expect("planner run json"); + value["environment"]["name"] = serde_json::Value::String(name.to_string()); + + let parsed = SpecDocument::parse_json_value(&value, Path::new(".")).expect("parse"); + let SpecDocument::PlannerRun(planner_run) = parsed else { + panic!("expected planner run document for builtin '{name}'"); + }; + match planner_run.environment { + EnvironmentSpec::Builtin { builtin } => assert_eq!(builtin, expected_builtin), + #[cfg(feature = "vm")] + EnvironmentSpec::NyxVm(_) => { + panic!("expected builtin environment for alias '{name}'"); + } + } + } +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn planner_run_parser_rejects_noncanonical_builtin_names() { + for name in [ + "coin-flip", + "biased_coinflip", + "biased_rps", + "kuhn-poker", + "extended-poker", + "extended_poker", + "extended-tiger", + "tic-tac-toe", + "tictactoe", + ] { + let mut value = sample_planner_run() + .to_canonical_json_value() + .expect("planner run json"); + value["environment"]["name"] = serde_json::Value::String(name.to_string()); + + let err = match SpecDocument::parse_json_value(&value, Path::new(".")) { + Ok(_) => panic!("noncanonical builtin name '{name}' must be rejected"), + Err(err) => err, + }; + assert!( + err.to_string().contains("unknown builtin environment"), + "unexpected parser error for '{name}': {err}" + ); + } +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn planner_run_parser_rejects_internal_tuner_bridge_environment() { + let mut value = sample_planner_run() + .to_canonical_json_value() + .expect("planner run json"); + value["environment"]["name"] = serde_json::Value::String("tuner_bridge".to_string()); + + let err = match SpecDocument::parse_json_value(&value, Path::new(".")) { + Ok(_) => panic!("internal tuner_bridge environment must be rejected in planner JSON"), + Err(err) => err, + }; + assert!( + err.to_string().contains("internal tuner planner bridge"), + "{err}" + ); +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn planner_run_parser_rejects_zero_action_alphabet() { + let mut value = sample_planner_run() + .to_canonical_json_value() + .expect("planner run json"); + value["interface"]["agent_actions"] = serde_json::json!(0); + + let err = match SpecDocument::parse_json_value(&value, Path::new(".")) { + Ok(_) => panic!("agent_actions=0 must be rejected at parse time"), + Err(err) => err, + }; + assert!( + err.to_string() + .contains("interface.agent_actions must be >= 1"), + "unexpected parser error: {err}" + ); +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn planner_run_binary_roundtrip_covers_canonical_builtins() { + let builtins = [ + BuiltinEnvironmentSpec::CoinFlip, + BuiltinEnvironmentSpec::BiasedRockPaperScissor, + BuiltinEnvironmentSpec::KuhnPoker, + BuiltinEnvironmentSpec::ExtendedTiger, + BuiltinEnvironmentSpec::TicTacToe, + BuiltinEnvironmentSpec::Blackjack, + BuiltinEnvironmentSpec::Platformer, + ]; + + for builtin in builtins { + let mut spec = sample_planner_run(); + spec.environment = EnvironmentSpec::Builtin { builtin }; + let bytes = SpecDocument::PlannerRun(spec).to_binary(); + let parsed = SpecDocument::from_binary(&bytes, Path::new(".")).expect("binary parse"); + let SpecDocument::PlannerRun(parsed_run) = parsed else { + panic!("expected planner run document for builtin {builtin:?}"); + }; + match parsed_run.environment { + EnvironmentSpec::Builtin { + builtin: parsed_builtin, + } => assert_eq!(parsed_builtin, builtin), + #[cfg(feature = "vm")] + EnvironmentSpec::NyxVm(_) => { + panic!("expected builtin environment for builtin {builtin:?}"); + } + } + } +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn planner_run_binary_rejects_internal_tuner_bridge_environment() { + let mut spec = sample_planner_run(); + spec.environment = EnvironmentSpec::Builtin { + builtin: BuiltinEnvironmentSpec::TunerBridge, + }; + let bytes = SpecDocument::PlannerRun(spec).to_binary(); + let err = match SpecDocument::from_binary(&bytes, Path::new(".")) { + Ok(_) => panic!("internal tuner_bridge must be rejected in public binary planner docs"), + Err(err) => err, + }; + assert!( + err.to_string().contains("internal tuner planner bridge"), + "{err}" + ); +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn planner_run_json_roundtrip_is_stable() { + let spec = sample_planner_run(); + let expected = spec.to_canonical_json().expect("json"); + let value = spec.to_canonical_json_value().expect("json value"); + let reparsed = SpecDocument::parse_json_value(&value, Path::new(".")).expect("parse"); + match reparsed { + SpecDocument::PlannerRun(parsed) => { + assert_eq!(parsed.to_canonical_json().expect("parsed json"), expected) + } + _ => panic!("expected planner run document"), + } +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn planner_run_binary_roundtrip_is_stable() { + let spec = sample_planner_run(); + let expected = spec.to_canonical_json().expect("json"); + let bytes = SpecDocument::PlannerRun(spec.clone()).to_binary(); + let reparsed = SpecDocument::from_binary(&bytes, Path::new(".")).expect("binary"); + match reparsed { + SpecDocument::PlannerRun(parsed) => { + assert_eq!(parsed.to_canonical_json().expect("parsed json"), expected) + } + _ => panic!("expected planner run document"), + } +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn mc_aixi_missing_mcts_strategy_canonicalizes_to_explicit_rho_uct() { + let spec = sample_mc_aixi_planner_run(MctsStrategy::RhoUct); + let mut value = SpecDocument::PlannerRun(spec) + .to_canonical_json_value() + .expect("canonical json value"); + value["controller"] + .as_object_mut() + .expect("controller object") + .remove("mcts_strategy"); + + let parsed = SpecDocument::parse_json_value(&value, Path::new(".")).expect("parse"); + let SpecDocument::PlannerRun(parsed_run) = parsed else { + panic!("expected planner run document"); + }; + let ControllerSpec::McAixi(inner) = &parsed_run.controller else { + panic!("expected MC-AIXI controller"); + }; + assert_eq!(inner.mcts_strategy, MctsStrategy::RhoUct); + + let canonical = parsed_run + .to_canonical_json_value() + .expect("canonical json value"); + assert_eq!( + canonical["controller"]["mcts_strategy"], + serde_json::json!({ "kind": "rho_uct" }) + ); +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn mc_aixi_missing_bit_stream_semantics_defaults_to_binary_tokens() { + let spec = sample_mc_aixi_planner_run(MctsStrategy::RhoUct); + let mut value = SpecDocument::PlannerRun(spec) + .to_canonical_json_value() + .expect("canonical json value"); + value["controller"] + .as_object_mut() + .expect("controller object") + .remove("bit_stream_semantics"); + + let parsed = SpecDocument::parse_json_value(&value, Path::new(".")).expect("parse"); + let SpecDocument::PlannerRun(parsed_run) = parsed else { + panic!("expected planner run document"); + }; + let ControllerSpec::McAixi(inner) = &parsed_run.controller else { + panic!("expected MC-AIXI controller"); + }; + assert_eq!(inner.bit_stream_semantics, default_bit_stream_semantics()); + + let canonical = parsed_run + .to_canonical_json_value() + .expect("canonical json value"); + assert_eq!( + canonical["controller"]["bit_stream_semantics"], + serde_json::json!({ "kind": "binary_tokens" }) + ); +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn mc_aixi_parallel_uct_binary_roundtrip_preserves_strategy() { + let spec = sample_mc_aixi_planner_run(MctsStrategy::ParallelUct { + workers: nz(16), + bu_uct_m_max: Some(0.8), + }); + let expected = SpecDocument::PlannerRun(spec.clone()) + .to_canonical_json_value() + .expect("canonical json value"); + let bytes = SpecDocument::PlannerRun(spec).to_binary(); + let reparsed = SpecDocument::from_binary(&bytes, Path::new(".")).expect("binary parse"); + let SpecDocument::PlannerRun(parsed_run) = reparsed else { + panic!("expected planner run document"); + }; + assert_eq!( + parsed_run + .to_canonical_json_value() + .expect("canonical json value"), + expected + ); + let ControllerSpec::McAixi(inner) = parsed_run.controller else { + panic!("expected MC-AIXI controller"); + }; + assert_eq!( + inner.mcts_strategy, + MctsStrategy::ParallelUct { + workers: nz(16), + bu_uct_m_max: Some(0.8), + } + ); +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn mc_aixi_parallel_uct_json_roundtrip_preserves_strategy() { + let spec = sample_mc_aixi_planner_run(MctsStrategy::ParallelUct { + workers: nz(16), + bu_uct_m_max: None, + }); + let expected = SpecDocument::PlannerRun(spec.clone()) + .to_canonical_json_value() + .expect("canonical json value"); + let parsed = + SpecDocument::parse_json_value(&expected, Path::new(".")).expect("canonical json parse"); + let SpecDocument::PlannerRun(parsed_run) = parsed else { + panic!("expected planner run document"); + }; + assert_eq!( + parsed_run + .to_canonical_json_value() + .expect("canonical json value"), + expected + ); + let ControllerSpec::McAixi(inner) = parsed_run.controller else { + panic!("expected MC-AIXI controller"); + }; + assert_eq!( + inner.mcts_strategy, + MctsStrategy::ParallelUct { + workers: nz(16), + bu_uct_m_max: None, + } + ); +} + +#[cfg(all(feature = "backend-ctw", feature = "aixi"))] +fn sample_warmstart_exact_jh_planner_run() -> PlannerRunSpec { + PlannerRunSpec { + assets: vec![AssetBinding { + id: "teacher".to_string(), + path: "teacher.json".to_string(), + }], + environment: EnvironmentSpec::Builtin { + builtin: BuiltinEnvironmentSpec::CoinFlip, + }, + interface: PlannerInterfaceSpec { + observation_bits: 2, + observation_stream_len: 1, + observation_key_mode: ObservationKeyMode::FullStream, + reward_bits: 2, + agent_actions: action_alphabet(2), + }, + controller: ControllerSpec::AiqiWarmstartExactJh(WarmStartExactJhControllerSpec { + predictor: RateBackend::Ctw { depth: 8 }, + bit_stream_semantics: default_bit_stream_semantics(), + return_horizon: 2, + return_bins: 5, + label_phase_period: 2, + teacher_dataset_asset: "teacher".to_string(), + planner_simulations_per_step: 1, + }), + runtime: PlannerRuntimeSpec { + random_seed: Some(11), + learn_cycles: Some(4), + eval_cycles: Some(2), + terminate_lifetime: 6, + log_every: 1, + perf: false, + vm_perf_only: false, + explore_epsilon: 0.0, + explore_gamma: 1.0, + }, + } +} + +#[cfg(all(feature = "backend-ctw", feature = "aixi"))] +#[test] +fn warmstart_exact_jh_json_binary_and_compile_roundtrip() { + let spec = sample_warmstart_exact_jh_planner_run(); + let document = SpecDocument::PlannerRun(spec.clone()); + let expected_value = document + .to_canonical_json_value() + .expect("canonical json value"); + let reparsed = + SpecDocument::from_binary(&document.to_binary(), Path::new(".")).expect("binary"); + assert_eq!( + reparsed + .to_canonical_json_value() + .expect("parsed json value"), + expected_value + ); + + let SpecDocument::PlannerRun(parsed_run) = + SpecDocument::parse_json_value(&expected_value, Path::new(".")).expect("json parse") + else { + panic!("expected planner run document"); + }; + assert!(matches!( + parsed_run.controller, + ControllerSpec::AiqiWarmstartExactJh(_) + )); + let compiled = parsed_run.compile().expect("warmstart compile"); + assert_eq!(compiled.controller().kind_str(), "aiqi_warmstart_exact_jh"); +} + +#[cfg(all(feature = "backend-ctw", feature = "aixi"))] +#[test] +fn warmstart_exact_jh_parser_rejects_zero_return_horizon_and_bins() { + for (field, message) in [ + ("return_horizon", "return_horizon must be >= 1"), + ("return_bins", "return_bins must be >= 1"), + ] { + let mut spec = sample_warmstart_exact_jh_planner_run(); + let ControllerSpec::AiqiWarmstartExactJh(inner) = &mut spec.controller else { + panic!("expected warmstart controller fixture"); + }; + if field == "return_horizon" { + inner.return_horizon = 0; + } else { + inner.return_bins = 0; + } + let err = match spec.compile() { + Ok(_) => panic!("zero {field} must fail"), + Err(err) => err, + }; + assert!(err.to_string().contains(message), "{err}"); + } +} + +#[cfg(all(feature = "backend-ctw", feature = "aixi"))] +#[test] +fn warmstart_exact_jh_rejects_non_direct_planner_simulations() { + let mut spec = sample_warmstart_exact_jh_planner_run(); + let ControllerSpec::AiqiWarmstartExactJh(inner) = &mut spec.controller else { + panic!("expected warmstart controller fixture"); + }; + inner.planner_simulations_per_step = 2; + let err = match spec.compile() { + Ok(_) => panic!("non-direct planner simulation budget must fail"), + Err(err) => err, + }; + assert!( + err.to_string() + .contains("planner_simulations_per_step must be exactly 1"), + "{err}" + ); +} + +#[cfg(all(feature = "backend-ctw", feature = "aixi"))] +#[test] +fn warmstart_exact_jh_rejects_slack_return_bins() { + let mut spec = sample_warmstart_exact_jh_planner_run(); + let ControllerSpec::AiqiWarmstartExactJh(inner) = &mut spec.controller else { + panic!("expected warmstart controller fixture"); + }; + inner.return_bins = 6; + let err = match spec.compile() { + Ok(_) => panic!("slack return bins must fail"), + Err(err) => err, + }; + assert!( + err.to_string() + .contains("return_bins must be exactly H * max_reward + 1"), + "{err}" + ); +} + +#[cfg(all(feature = "backend-ctw", feature = "aixi"))] +#[test] +fn warmstart_exact_jh_rejects_narrow_reward_bits() { + let mut spec = sample_warmstart_exact_jh_planner_run(); + spec.interface.reward_bits = 1; + let err = match spec.compile() { + Ok(_) => panic!("narrow reward_bits must fail"), + Err(err) => err, + }; + assert!(err.to_string().contains("max_reward=2"), "{err}"); +} + +#[cfg(all(feature = "backend-ctw", feature = "aixi"))] +#[test] +fn warmstart_exact_jh_rejects_missing_teacher_asset() { + let mut spec = sample_warmstart_exact_jh_planner_run(); + let ControllerSpec::AiqiWarmstartExactJh(inner) = &mut spec.controller else { + panic!("expected warmstart controller fixture"); + }; + inner.teacher_dataset_asset = "missing".to_string(); + let err = match spec.compile() { + Ok(_) => panic!("missing teacher asset must fail"), + Err(err) => err, + }; + assert!( + err.to_string().contains("unknown asset id 'missing'"), + "{err}" + ); +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn mc_aixi_parallel_uct_parser_rejects_zero_workers_in_canonical_json() { + // `workers == 0` is type-prevented in `MctsStrategy::ParallelUct` itself + // (`NonZeroUsize`), so the only surface where it can still be expressed + // is the document layer. Verify the canonical-JSON parser rejects it + // with a stable, label-prefixed error message. + let mut spec = sample_mc_aixi_planner_run(MctsStrategy::ParallelUct { + workers: nz(1), + bu_uct_m_max: None, + }); + // Take a valid canonical JSON value, then mutate `workers` to 0. + let mut value = SpecDocument::PlannerRun(spec.clone()) + .to_canonical_json_value() + .expect("canonical json value"); + value["controller"]["mcts_strategy"]["workers"] = serde_json::json!(0); + let err = match SpecDocument::parse_json_value(&value, Path::new(".")) { + Ok(_) => panic!("workers=0 must be rejected at parse time"), + Err(err) => err, + }; + assert!( + err.to_string() + .contains("controller.mcts_strategy.workers must be >= 1"), + "{err}" + ); + // Sanity check: an unrelated mutation (validating `bu_uct_m_max`) still + // routes through the spec-pipeline validation layer. + spec.controller = ControllerSpec::McAixi(McAixiControllerSpec { + predictor: RateBackend::Ctw { depth: 8 }, + bit_stream_semantics: default_bit_stream_semantics(), + agent_horizon: 2, + num_simulations: 4, + mcts_strategy: MctsStrategy::ParallelUct { + workers: nz(4), + bu_uct_m_max: Some(1.0), + }, + exploration_exploitation_ratio: 1.0, + discount_gamma: 0.95, + }); + let err = match spec.compile() { + Ok(_) => panic!("invalid bu_uct_m_max must fail"), + Err(err) => err, + }; + assert!( + err.to_string() + .contains("controller.mcts_strategy.bu_uct_m_max must be in (0, 1)"), + "{err}" + ); +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn mc_aixi_parallel_uct_rejects_invalid_bu_threshold() { + for invalid in [0.0, 1.0, -0.25, 1.25] { + let spec = sample_mc_aixi_planner_run(MctsStrategy::ParallelUct { + workers: nz(4), + bu_uct_m_max: Some(invalid), + }); + let err = match spec.compile() { + Ok(_) => panic!("invalid bu_uct_m_max={invalid} must fail"), + Err(err) => err, + }; + assert!( + err.to_string() + .contains("controller.mcts_strategy.bu_uct_m_max must be in (0, 1)"), + "{err}" + ); + } +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn mc_aixi_parallel_uct_canonical_json_rejects_string_shorthand_for_rho_uct() { + // The canonical schema requires the object form for every strategy. The + // serializer always emits `{ "kind": "rho_uct" }` (or the parallel_uct + // object), so the parser must symmetrically refuse string shorthands. + let mut value = SpecDocument::PlannerRun(sample_mc_aixi_planner_run(MctsStrategy::RhoUct)) + .to_canonical_json_value() + .expect("canonical json value"); + value["controller"]["mcts_strategy"] = serde_json::json!("rho_uct"); + let err = match SpecDocument::parse_json_value(&value, Path::new(".")) { + Ok(_) => panic!("string shorthand must be rejected"), + Err(err) => err, + }; + assert!( + err.to_string() + .contains("controller.mcts_strategy must be an object with a 'kind' field"), + "{err}" + ); +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn mc_aixi_parallel_uct_workers_one_warns_once_across_repeated_initialization() { + reset_parallel_uct_workers_one_warning_for_tests(); + let spec = sample_mc_aixi_planner_run(MctsStrategy::ParallelUct { + workers: nz(1), + bu_uct_m_max: None, + }); + + spec.compile().expect("workers=1 should compile"); + assert_eq!( + parallel_uct_workers_one_warning_count_for_tests(), + 1, + "workers=1 should emit exactly one warning during first initialization" + ); + + spec.compile() + .expect("workers=1 should keep compiling on subsequent initialization"); + assert_eq!( + parallel_uct_workers_one_warning_count_for_tests(), + 1, + "workers=1 warning must remain one-time across repeated initialization" + ); +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn planner_run_omitted_runtime_seed_canonicalizes_to_default_seed() { + let mut spec = sample_planner_run(); + spec.runtime.random_seed = None; + + let compiled = spec.compile().expect("compile"); + assert_eq!( + compiled.runtime().random_seed, + Some(DEFAULT_RANDOM_SEED), + "runtime.random_seed should canonicalize to deterministic default", + ); + assert_eq!( + compiled.canonical_spec().runtime.random_seed, + Some(DEFAULT_RANDOM_SEED), + "canonical spec should preserve the resolved default seed", + ); + + let canonical_value = compiled + .canonical_spec() + .to_canonical_json_value() + .expect("canonical json value"); + assert_eq!( + canonical_value["runtime"]["random_seed"], + serde_json::Value::from(DEFAULT_RANDOM_SEED), + "canonical JSON should expose resolved runtime.random_seed", + ); +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn planner_run_resolved_seed_survives_binary_roundtrip() { + let mut spec = sample_planner_run(); + spec.runtime.random_seed = None; + + let compiled = spec.compile().expect("compile"); + let bytes = SpecDocument::PlannerRun(compiled.canonical_spec().clone()).to_binary(); + let parsed = SpecDocument::from_binary(&bytes, Path::new(".")).expect("from binary"); + let SpecDocument::PlannerRun(roundtripped) = parsed else { + panic!("expected planner_run document"); + }; + let roundtripped_compiled = roundtripped.compile().expect("recompile"); + assert_eq!( + roundtripped_compiled.runtime().random_seed, + Some(DEFAULT_RANDOM_SEED), + ); +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn staged_document_and_compiled_planner_accessors_preserve_metadata() { + let base_dir = Path::new("/tmp/infotheory-stage-planner"); + let spec = sample_planner_run(); + let value = SpecDocument::PlannerRun(spec.clone()) + .to_canonical_json_value() + .expect("planner json value"); + let parsed = + SpecDocument::parse_json_value_staged(&value, base_dir).expect("staged planner parse"); + + assert_eq!(parsed.base_dir(), base_dir); + assert!(matches!(parsed.document(), SpecDocument::PlannerRun(_))); + assert!(matches!( + parsed.clone().into_document(), + SpecDocument::PlannerRun(_) + )); + + let validated = parsed.validate().expect("staged planner validate"); + assert!(!validated.canonical_bytes().is_empty()); + let compiled_doc = validated.compile().expect("staged planner compile"); + assert_eq!( + compiled_doc.canonical_bytes().as_slice(), + validated.canonical_bytes().as_slice() + ); + + match compiled_doc { + CompiledSpecDocument::PlannerRun(compiled) => { + assert_eq!( + compiled + .canonical_spec() + .to_canonical_json() + .expect("canonical json"), + spec.compile() + .expect("direct planner compile") + .canonical_spec() + .to_canonical_json() + .expect("direct canonical json") + ); + assert_eq!(compiled.resolved_assets().len(), 0); + assert_eq!(compiled.interface().agent_actions.get(), 2); + assert_eq!(compiled.runtime().random_seed, Some(7)); + assert_eq!(compiled.resolved_random_seed(), 7); + assert_eq!(compiled.action_bits(), 1); + assert_eq!(compiled.controller().kind_str(), "aiqi_discounted"); + assert_eq!(compiled.controller().backend_label(), "ctw(depth=8)"); + } + _ => panic!("expected compiled planner document"), + } +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn staged_pipeline_matches_direct_planner_compile() { + let spec = sample_planner_run(); + let value = SpecDocument::PlannerRun(spec.clone()) + .to_canonical_json_value() + .expect("planner run json value"); + let parsed = + SpecDocument::parse_json_value_staged(&value, Path::new(".")).expect("staged parse"); + let validated = parsed.validate().expect("staged validate"); + let compiled = validated.compile().expect("staged compile"); + let direct = spec.compile().expect("direct compile"); + + match compiled { + CompiledSpecDocument::PlannerRun(staged) => { + assert_eq!( + staged.canonical_bytes().as_slice(), + direct.canonical_bytes().as_slice() + ); + } + _ => panic!("expected compiled planner-run document"), + } +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn staged_pipeline_supports_standalone_backend_documents() { + let document = SpecDocument::RateBackend(RateBackend::Ctw { depth: 8 }); + let expected_json = document.to_canonical_json().expect("document json"); + let value = document + .to_canonical_json_value() + .expect("document json value"); + + let parsed = + SpecDocument::parse_json_value_staged(&value, Path::new(".")).expect("staged parse"); + let validated = parsed.validate().expect("staged validate"); + assert!(!validated.canonical_bytes().is_empty()); + let compiled = validated.compile().expect("staged compile"); + match compiled { + CompiledSpecDocument::RateBackend(compiled_backend) => { + let reparsed = SpecDocument::RateBackend(compiled_backend.canonical_spec().clone()) + .to_canonical_json() + .expect("compiled canonical json"); + assert_eq!(reparsed, expected_json); + } + _ => panic!("expected compiled rate-backend document"), + } + + let bytes = document.to_binary(); + let binary_parsed = + SpecDocument::from_binary_staged(&bytes, Path::new(".")).expect("staged binary parse"); + let binary_compiled = binary_parsed + .validate() + .expect("staged binary validate") + .compile() + .expect("staged binary compile"); + match binary_compiled { + CompiledSpecDocument::RateBackend(compiled_backend) => { + assert!(matches!( + compiled_backend.canonical_spec(), + RateBackend::Ctw { depth: 8 } + )); + } + _ => panic!("expected compiled rate-backend document from binary"), + } +} + +#[cfg(all(feature = "backend-ctw", feature = "tuner"))] +#[test] +fn tune_validation_and_compilation_accessors_surface_baseline_metadata() { + let spec = sample_tune_spec(); + let validated = spec.validate().expect("validated tune spec"); + assert_eq!(validated.canonical_spec().input_asset, "dataset"); + assert!(!validated.canonical_bytes().is_empty()); + + let compiled = validated.compile().expect("compiled tune spec"); + assert_eq!(compiled.canonical_spec().input_asset, "dataset"); + assert_eq!(compiled.resolved_assets().len(), 1); + assert_eq!(compiled.resolved_assets()[0].id, "dataset"); + let crate::spec::AssetRef::Filesystem(path) = &compiled.resolved_assets()[0].asset; + assert!(path.ends_with("input.bin")); + assert!(matches!( + compiled.controller(), + CompiledTuneController::AnnealedHillClimbing(_) + )); + assert_eq!(compiled.candidate_canonicalization_version(), "bounds-v1"); + assert_eq!( + compiled.baseline_candidate_model_bytes(), + compiled.baseline_candidate().canonical_bytes().len() + ); +} + +#[cfg(all(feature = "backend-ctw", feature = "tuner"))] +#[test] +fn standalone_backend_documents_roundtrip_without_embedded_json_fragments() { + let rate = SpecDocument::RateBackend(RateBackend::Ctw { depth: 8 }); + let compression = SpecDocument::CompressionBackend(CompressionBackend::Rate { + rate_backend: RateBackend::Ctw { depth: 8 }, + coder: crate::coders::CoderType::AC, + framing: crate::compression::FramingMode::Framed, + }); + + for doc in [rate, compression] { + let expected = doc.to_canonical_json().expect("json"); + let bytes = doc.to_binary(); + assert!( + !bytes + .windows(br#""kind""#.len()) + .any(|window| window == br#""kind""#), + "binary document should not embed canonical JSON object keys: {bytes:?}" + ); + let reparsed = SpecDocument::from_binary(&bytes, Path::new(".")).expect("binary"); + assert_eq!(reparsed.to_canonical_json().expect("parsed json"), expected); + } +} + +#[cfg(feature = "all-backends")] +#[test] +fn standalone_rate_backend_documents_cover_all_binary_backend_tags() { + let rate_docs = vec![ + SpecDocument::RateBackend(RateBackend::RosaPlus { max_order: 32 }), + SpecDocument::RateBackend(RateBackend::Match { + hash_bits: 18, + min_len: 2, + max_len: 32, + base_mix: 0.05, + confidence_scale: 1.0, + }), + SpecDocument::RateBackend(RateBackend::SparseMatch { + hash_bits: 18, + min_len: 2, + max_len: 32, + gap_min: 1, + gap_max: 4, + base_mix: 0.05, + confidence_scale: 1.0, + }), + SpecDocument::RateBackend(RateBackend::Ppmd { + order: 6, + memory_mb: 8, + }), + SpecDocument::RateBackend(RateBackend::Sequitur { context_bytes: 64 }), + SpecDocument::RateBackend(RateBackend::Ctw { depth: 12 }), + SpecDocument::RateBackend(RateBackend::FacCtw { + base_depth: 10, + num_percept_bits: 8, + encoding_bits: 1, + msb_first: None, + }), + SpecDocument::RateBackend(RateBackend::FacCtw { + base_depth: 10, + num_percept_bits: 8, + encoding_bits: 8, + msb_first: Some(false), + }), + SpecDocument::RateBackend(RateBackend::Zpaq { + method: crate::api::ZpaqMethodSpec::literal("1"), + }), + SpecDocument::RateBackend(RateBackend::Mixture { + spec: Arc::new(MixtureSpec::new( + MixtureKind::Bayes, + vec![MixtureExpertSpec::new(RateBackend::Ctw { depth: 6 })], + )), + }), + SpecDocument::RateBackend(RateBackend::Mixture { + spec: Arc::new( + MixtureSpec::new( + MixtureKind::FadingBayes, + vec![MixtureExpertSpec::new(RateBackend::Match { + hash_bits: 16, + min_len: 2, + max_len: 16, + base_mix: 0.05, + confidence_scale: 1.0, + })], + ) + .with_decay(0.97), + ), + }), + SpecDocument::RateBackend(RateBackend::Mixture { + spec: Arc::new( + MixtureSpec::new( + MixtureKind::Switching, + vec![MixtureExpertSpec::new(RateBackend::Ctw { depth: 4 })], + ) + .with_schedule(MixtureScheduleMode::Theorem), + ), + }), + SpecDocument::RateBackend(RateBackend::Mixture { + spec: Arc::new( + MixtureSpec::new( + MixtureKind::Convex, + vec![MixtureExpertSpec::new(RateBackend::Ppmd { + order: 5, + memory_mb: 4, + })], + ) + .with_alpha(1.25) + .with_schedule(MixtureScheduleMode::Theorem), + ), + }), + SpecDocument::RateBackend(RateBackend::Mixture { + spec: Arc::new(MixtureSpec::new( + MixtureKind::Mdl, + vec![MixtureExpertSpec::new(RateBackend::Sequitur { + context_bytes: 48, + })], + )), + }), + SpecDocument::RateBackend(RateBackend::Mixture { + spec: Arc::new(MixtureSpec::new( + MixtureKind::Neural, + vec![MixtureExpertSpec::new(RateBackend::FacCtw { + base_depth: 8, + num_percept_bits: 8, + encoding_bits: 1, + msb_first: None, + })], + )), + }), + SpecDocument::RateBackend(RateBackend::Particle { + spec: Arc::new(ParticleSpec::default()), + }), + SpecDocument::RateBackend(RateBackend::Calibrated { + spec: Arc::new(CalibratedSpec { + context: CalibrationContextKind::Text, + bins: 17, + learning_rate: 0.05, + bias_clip: 3.0, + base: RateBackend::Ctw { depth: 8 }, + }), + }), + ]; + + for doc in rate_docs { + let expected = doc.to_canonical_json().expect("json"); + let reparsed = SpecDocument::from_binary(&doc.to_binary(), Path::new(".")).expect("binary"); + assert_eq!(reparsed.to_canonical_json().expect("parsed json"), expected); + } +} + +#[cfg(feature = "all-backends")] +#[test] +fn standalone_compression_backend_documents_cover_binary_coder_variants() { + let docs = vec![ + SpecDocument::CompressionBackend(CompressionBackend::zpaq("5")), + SpecDocument::CompressionBackend(CompressionBackend::Rate { + rate_backend: RateBackend::Ctw { depth: 8 }, + coder: crate::coders::CoderType::AC, + framing: crate::compression::FramingMode::Raw, + }), + SpecDocument::CompressionBackend(CompressionBackend::Rate { + rate_backend: RateBackend::Mixture { + spec: Arc::new(MixtureSpec::new( + MixtureKind::Bayes, + vec![MixtureExpertSpec::new(RateBackend::FacCtw { + base_depth: 8, + num_percept_bits: 8, + encoding_bits: 1, + msb_first: None, + })], + )), + }, + coder: crate::coders::CoderType::RANS, + framing: crate::compression::FramingMode::Framed, + }), + ]; + + for doc in docs { + let expected = doc.to_canonical_json().expect("json"); + let reparsed = SpecDocument::from_binary(&doc.to_binary(), Path::new(".")).expect("binary"); + assert_eq!(reparsed.to_canonical_json().expect("parsed json"), expected); + } +} + +#[cfg(any(feature = "backend-mamba", feature = "backend-rwkv"))] +fn sample_llm_policy() -> LlmPolicy { + LlmPolicy { + load_from: Some("checkpoint;v1.safetensors".into()), + schedule: vec![ + ScheduleRule::Interval(PolicyRule { + start: PositionExpr::Bytes(0), + end: PositionExpr::Percent(0.5), + action: PolicyAction::Infer, + }), + ScheduleRule::Repeat(RepeatRule { + start: PositionExpr::Bytes(10), + end: PositionExpr::Bytes(200), + period: PositionExpr::Bytes(6), + pattern: vec![ + RepeatSegment { + span: PositionExpr::Bytes(2), + action: PolicyAction::Train(TrainAction { + scope: TrainScopeSet { + all: false, + names: vec!["head".to_string(), "bias".to_string()], + }, + optimizer: OptimizerKind::Adam, + hyper: OptimizerHyperParams { + lr: 0.01, + stride: 2, + bptt: 4, + clip: 1.5, + momentum: 0.9, + }, + }), + }, + RepeatSegment { + span: PositionExpr::Percent(0.5), + action: PolicyAction::Train(TrainAction { + scope: TrainScopeSet::all(), + optimizer: OptimizerKind::Sgd, + hyper: OptimizerHyperParams { + lr: 0.005, + stride: 1, + bptt: 1, + clip: 0.0, + momentum: 0.2, + }, + }), + }, + ], + }), + ], + } +} + +#[cfg(feature = "backend-rwkv")] +#[test] +fn standalone_rwkv_method_documents_roundtrip_file_and_online_policies() { + let file_doc = SpecDocument::RateBackend(RateBackend::Rwkv7Method { + method: crate::rwkvzip::MethodSpec::File { + path: "models/rwkv;demo.safetensors".into(), + policy: Some(sample_llm_policy()), + }, + }); + let online_doc = SpecDocument::CompressionBackend(CompressionBackend::Rwkv7 { + method: crate::rwkvzip::MethodSpec::Online { + cfg: crate::rwkvzip::OnlineConfig { + hidden: 64, + layers: 1, + intermediate: 64, + decay_rank: 8, + a_rank: 8, + v_rank: 8, + g_rank: 8, + seed: 17, + train_mode: crate::rwkvzip::OnlineTrainMode::Adam, + lr: 0.01, + stride: 3, + }, + policy: Some(sample_llm_policy()), + }, + coder: crate::coders::CoderType::RANS, + }); + + for doc in [file_doc, online_doc] { + let expected = doc.to_canonical_json().expect("json"); + let reparsed = SpecDocument::from_binary(&doc.to_binary(), Path::new(".")).expect("binary"); + assert_eq!(reparsed.to_canonical_json().expect("parsed json"), expected); + } +} + +#[cfg(feature = "backend-mamba")] +#[test] +fn standalone_mamba_method_documents_roundtrip_file_and_online_policies() { + let file_doc = SpecDocument::RateBackend(RateBackend::MambaMethod { + method: crate::mambazip::MethodSpec::File { + path: "models/mamba;demo.safetensors".into(), + policy: Some(sample_llm_policy()), + }, + }); + let online_doc = SpecDocument::RateBackend(RateBackend::MambaMethod { + method: crate::mambazip::MethodSpec::Online { + cfg: crate::mambazip::OnlineConfig { + hidden: 64, + layers: 2, + intermediate: 96, + state: 8, + conv: 4, + dt_rank: 8, + seed: 23, + train_mode: crate::mambazip::OnlineTrainMode::Sgd, + lr: 0.02, + stride: 2, + }, + policy: Some(sample_llm_policy()), + }, + }); + + for doc in [file_doc, online_doc] { + let expected = doc.to_canonical_json().expect("json"); + let reparsed = SpecDocument::from_binary(&doc.to_binary(), Path::new(".")).expect("binary"); + assert_eq!(reparsed.to_canonical_json().expect("parsed json"), expected); + } +} + +#[cfg(all(feature = "all-backends", feature = "tuner"))] +#[test] +fn planner_and_tune_documents_roundtrip_all_controller_variants() { + let planner_interface = PlannerInterfaceSpec { + observation_bits: 2, + observation_stream_len: 2, + observation_key_mode: ObservationKeyMode::StreamHash, + reward_bits: 2, + agent_actions: action_alphabet(3), + }; + let tune_interface = TunePlannerInterfaceSpec { + observation_bits: 2, + observation_stream_len: 2, + observation_key_mode: ObservationKeyMode::StreamHash, + reward_bits: 2, + agent_actions: action_alphabet(3), + }; + + let planner_docs = vec![ + SpecDocument::PlannerRun(PlannerRunSpec { + assets: vec![], + environment: EnvironmentSpec::Builtin { + builtin: BuiltinEnvironmentSpec::CoinFlip, + }, + interface: planner_interface.clone(), + controller: ControllerSpec::McAixi(McAixiControllerSpec { + predictor: RateBackend::FacCtw { + base_depth: 8, + num_percept_bits: 8, + encoding_bits: 1, + msb_first: None, + }, + bit_stream_semantics: default_bit_stream_semantics(), + agent_horizon: 4, + num_simulations: 12, + mcts_strategy: MctsStrategy::ParallelUct { + workers: nz(3), + bu_uct_m_max: Some(0.5), + }, + exploration_exploitation_ratio: 1.1, + discount_gamma: 0.95, + }), + runtime: PlannerRuntimeSpec { + random_seed: None, + learn_cycles: Some(5), + eval_cycles: Some(3), + terminate_lifetime: 8, + log_every: 2, + perf: true, + vm_perf_only: false, + explore_epsilon: 0.2, + explore_gamma: 0.9, + }, + }), + SpecDocument::PlannerRun(PlannerRunSpec { + assets: vec![], + environment: EnvironmentSpec::Builtin { + builtin: BuiltinEnvironmentSpec::Blackjack, + }, + interface: planner_interface.clone(), + controller: ControllerSpec::AiqiWarmstartExactJh(WarmStartExactJhControllerSpec { + predictor: RateBackend::Mixture { + spec: Arc::new( + MixtureSpec::new( + MixtureKind::Switching, + vec![ + MixtureExpertSpec::new(RateBackend::Ctw { depth: 6 }), + MixtureExpertSpec::new(RateBackend::FacCtw { + base_depth: 6, + num_percept_bits: 8, + encoding_bits: 1, + msb_first: None, + }), + ], + ) + .with_schedule(MixtureScheduleMode::Theorem), + ), + }, + bit_stream_semantics: default_bit_stream_semantics(), + return_horizon: 4, + return_bins: 17, + label_phase_period: 6, + teacher_dataset_asset: "teacher".to_string(), + planner_simulations_per_step: 1, + }), + runtime: PlannerRuntimeSpec { + random_seed: Some(19), + learn_cycles: None, + eval_cycles: Some(4), + terminate_lifetime: 7, + log_every: 1, + perf: false, + vm_perf_only: false, + explore_epsilon: 0.0, + explore_gamma: 1.0, + }, + }), + ]; + + for doc in planner_docs { + let expected = doc.to_canonical_json().expect("json"); + let reparsed = SpecDocument::from_binary(&doc.to_binary(), Path::new(".")).expect("binary"); + assert_eq!(reparsed.to_canonical_json().expect("parsed json"), expected); + } + + let bounds = TuneBoundsSpec { + allowed_backends: vec!["ctw".to_string(), "fac-ctw".to_string()], + forbidden_backends: vec!["zpaq".to_string()], + parameter_ranges: vec![TuneParameterRangeSpec { + parameter: "mixture.alpha".to_string(), + min: 0.01, + max: 0.5, + }], + max_experts: 4, + max_mixture_nesting_depth: 2, + min_experts: Some(1), + allow_duplicate_experts: Some(false), + required_experts: vec!["ctw".to_string()], + forbidden_expert_pairs: vec![("ppmd".to_string(), "sequitur".to_string())], + }; + + let tune_docs = vec![ + SpecDocument::Tune(TuneSpec { + assets: vec![], + input_asset: "dataset".to_string(), + baseline_candidate: CompressionBackend::Rate { + rate_backend: RateBackend::Ctw { depth: 8 }, + coder: crate::coders::CoderType::AC, + framing: crate::compression::FramingMode::Framed, + }, + controller: TuneControllerSpec::McAixiFacCtw(McAixiFacCtwTuneControllerSpec { + interface: tune_interface.clone(), + planner_simulations_per_step: 10, + }), + bounds: bounds.clone(), + eval_time_limit_seconds: 1.5, + time_budget_seconds: 20.0, + min_throughput_bytes_per_second: 2048.0, + max_memory_bytes: 1 << 20, + output_config_path: "mcaixi.json".to_string(), + seed: 11, + report_path: None, + }), + SpecDocument::Tune(TuneSpec { + assets: vec![], + input_asset: "dataset".to_string(), + baseline_candidate: CompressionBackend::Rate { + rate_backend: RateBackend::FacCtw { + base_depth: 8, + num_percept_bits: 8, + encoding_bits: 1, + msb_first: None, + }, + coder: crate::coders::CoderType::RANS, + framing: crate::compression::FramingMode::Raw, + }, + controller: TuneControllerSpec::AiqiDiscounted(AiqiDiscountedTuneControllerSpec { + interface: tune_interface.clone(), + planner_simulations_per_step: 12, + return_horizon: 5, + return_bins: 16, + discount_factor: 0.97, + min_improvement: -1.0, + max_improvement: 1.0, + }), + bounds: bounds.clone(), + eval_time_limit_seconds: 2.0, + time_budget_seconds: 30.0, + min_throughput_bytes_per_second: 4096.0, + max_memory_bytes: 1 << 21, + output_config_path: "aiqi.json".to_string(), + seed: 13, + report_path: Some("aiqi-report.json".to_string()), + }), + SpecDocument::Tune(TuneSpec { + assets: vec![AssetBinding { + id: "teacher".to_string(), + path: "teacher.bin".to_string(), + }], + input_asset: "dataset".to_string(), + baseline_candidate: CompressionBackend::Rate { + rate_backend: RateBackend::Match { + hash_bits: 16, + min_len: 2, + max_len: 16, + base_mix: 0.05, + confidence_scale: 1.0, + }, + coder: crate::coders::CoderType::AC, + framing: crate::compression::FramingMode::Framed, + }, + controller: TuneControllerSpec::AiqiWarmstartExactJh( + WarmStartExactJhTuneControllerSpec { + interface: tune_interface, + planner_simulations_per_step: 1, + return_horizon: 4, + warmstart_teacher_dataset_asset: "teacher".to_string(), + label_phase_period: 5, + }, + ), + bounds, + eval_time_limit_seconds: 3.0, + time_budget_seconds: 40.0, + min_throughput_bytes_per_second: 1024.0, + max_memory_bytes: 1 << 22, + output_config_path: "warmstart.json".to_string(), + seed: 17, + report_path: Some("warmstart-report.json".to_string()), + }), + ]; + + for doc in tune_docs { + let expected = doc.to_canonical_json().expect("json"); + let reparsed = SpecDocument::from_binary(&doc.to_binary(), Path::new(".")).expect("binary"); + assert_eq!(reparsed.to_canonical_json().expect("parsed json"), expected); + } +} + +#[cfg(feature = "all-backends")] +#[test] +fn binary_spec_document_corruption_reports_precise_envelope_errors() { + let rate_doc = SpecDocument::RateBackend(RateBackend::Ctw { depth: 8 }); + + let mut bad_magic = rate_doc.to_binary(); + bad_magic[0] ^= 0x01; + let err = match SpecDocument::from_binary(&bad_magic, Path::new(".")) { + Ok(_) => panic!("corrupted magic must be rejected"), + Err(err) => err, + }; + assert!(err.to_string().contains("invalid spec document magic")); + + let mut bad_version = rate_doc.to_binary(); + bad_version[4] = 99; + let err = match SpecDocument::from_binary(&bad_version, Path::new(".")) { + Ok(_) => panic!("unknown version must be rejected"), + Err(err) => err, + }; + assert!( + err.to_string() + .contains("unsupported spec document binary version") + ); + + let mut bad_doc_tag = rate_doc.to_binary(); + bad_doc_tag[5] = 99; + let err = match SpecDocument::from_binary(&bad_doc_tag, Path::new(".")) { + Ok(_) => panic!("unknown top-level tag must be rejected"), + Err(err) => err, + }; + assert!(err.to_string().contains("unknown spec document tag")); + + let mut bad_rate_tag = rate_doc.to_binary(); + bad_rate_tag[6] = 99; + let err = match SpecDocument::from_binary(&bad_rate_tag, Path::new(".")) { + Ok(_) => panic!("unknown rate backend tag must be rejected"), + Err(err) => err, + }; + assert!(err.to_string().contains("unknown rate backend tag")); + + let mut bad_compression_coder = SpecDocument::CompressionBackend(CompressionBackend::Rate { + rate_backend: RateBackend::Ctw { depth: 8 }, + coder: crate::coders::CoderType::AC, + framing: crate::compression::FramingMode::Raw, + }) + .to_binary(); + bad_compression_coder[7] = 99; + let err = match SpecDocument::from_binary(&bad_compression_coder, Path::new(".")) { + Ok(_) => panic!("unknown coder tag must be rejected"), + Err(err) => err, + }; + assert!(err.to_string().contains("unknown coder tag")); + + let mut bad_compression_framing = SpecDocument::CompressionBackend(CompressionBackend::Rate { + rate_backend: RateBackend::Ctw { depth: 8 }, + coder: crate::coders::CoderType::AC, + framing: crate::compression::FramingMode::Raw, + }) + .to_binary(); + bad_compression_framing[8] = 99; + let err = match SpecDocument::from_binary(&bad_compression_framing, Path::new(".")) { + Ok(_) => panic!("unknown framing tag must be rejected"), + Err(err) => err, + }; + assert!(err.to_string().contains("unknown framing tag")); + + let mut bad_mixture_kind = SpecDocument::RateBackend(RateBackend::Mixture { + spec: Arc::new(MixtureSpec::new( + MixtureKind::Bayes, + vec![MixtureExpertSpec::new(RateBackend::Ctw { depth: 4 })], + )), + }) + .to_binary(); + bad_mixture_kind[7] = 99; + let err = match SpecDocument::from_binary(&bad_mixture_kind, Path::new(".")) { + Ok(_) => panic!("unknown mixture kind must be rejected"), + Err(err) => err, + }; + assert!(err.to_string().contains("unknown mixture kind tag")); + + let mut bad_mixture_schedule = SpecDocument::RateBackend(RateBackend::Mixture { + spec: Arc::new(MixtureSpec::new( + MixtureKind::Switching, + vec![MixtureExpertSpec::new(RateBackend::Ctw { depth: 4 })], + )), + }) + .to_binary(); + bad_mixture_schedule[8] = 99; + let err = match SpecDocument::from_binary(&bad_mixture_schedule, Path::new(".")) { + Ok(_) => panic!("unknown mixture schedule must be rejected"), + Err(err) => err, + }; + assert!(err.to_string().contains("unknown mixture schedule tag")); +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn planner_run_compile_exposes_compiled_predictor_and_action_bits() { + let compiled = sample_planner_run() + .compile() + .expect("compiled planner run"); + assert_eq!(compiled.action_bits(), 1); + match compiled.controller() { + CompiledPlannerController::AiqiDiscounted { predictor, .. } => { + assert!(matches!( + predictor.canonical_spec(), + RateBackend::Ctw { depth: 8 } + )); + } + _ => panic!("expected compiled aiqi controller"), + } +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn planner_run_parser_rejects_legacy_shared_interface_reward_fields() { + let err = match SpecDocument::parse_json_value( + &serde_json::json!({ + "schema_version": 1, + "kind": "planner_run", + "assets": [], + "environment": { + "kind": "builtin", + "name": "coin_flip" + }, + "interface": { + "observation_bits": 1, + "observation_stream_len": 1, + "observation_key_mode": "full_stream", + "reward_bits": 1, + "agent_actions": 2, + "min_reward": 0 + }, + "controller": { + "kind": "aiqi_discounted", + "predictor": {"kind":"ctw","depth":8}, + "discount_gamma": 0.99, + "return_horizon": 2, + "return_bins": 8, + "augmentation_period": 2, + "baseline_exploration": 0.01 + }, + "runtime": {} + }), + std::path::Path::new("."), + ) { + Ok(_) => panic!("legacy interface reward fields must be rejected"), + Err(err) => err, + }; + assert!(err.to_string().contains("unknown interface field"), "{err}"); +} + +#[cfg(all(feature = "backend-ctw", feature = "backend-zpaq"))] +#[test] +fn planner_run_compile_rejects_mcaixi_predictors_with_zpaq_conditioning() { + let mut spec = sample_planner_run(); + spec.controller = ControllerSpec::McAixi(McAixiControllerSpec { + predictor: RateBackend::Zpaq { + method: crate::api::ZpaqMethodSpec::literal("1"), + }, + bit_stream_semantics: default_bit_stream_semantics(), + agent_horizon: 1, + num_simulations: 1, + mcts_strategy: MctsStrategy::RhoUct, + exploration_exploitation_ratio: 1.0, + discount_gamma: 1.0, + }); + let err = match spec.compile() { + Ok(_) => panic!("MC-AIXI zpaq backend must fail"), + Err(err) => err, + }; + assert!( + err.to_string().contains("reversible action conditioning"), + "{err}" + ); +} + +#[cfg(all(feature = "backend-ctw", feature = "backend-zpaq"))] +#[test] +fn planner_run_compile_rejects_aiqi_predictors_without_frozen_conditioning() { + let mut spec = sample_planner_run(); + spec.controller = ControllerSpec::AiqiDiscounted(AiqiDiscountedControllerSpec { + predictor: RateBackend::Zpaq { + method: crate::api::ZpaqMethodSpec::literal("1"), + }, + bit_stream_semantics: default_bit_stream_semantics(), + discount_gamma: 0.99, + return_horizon: 2, + return_bins: 8, + augmentation_period: 2, + history_prune_keep_steps: None, + baseline_exploration: 0.01, + }); + let err = match spec.compile() { + Ok(_) => panic!("AIQI zpaq backend must fail"), + Err(err) => err, + }; + assert!( + err.to_string().contains("strict frozen conditioning"), + "{err}" + ); +} + +#[cfg(all(feature = "backend-ctw", feature = "vm"))] +fn sample_vm_planner_run() -> PlannerRunSpec { + PlannerRunSpec { + assets: vec![AssetBinding { + id: "firecracker".to_string(), + path: "dummy-firecracker.json".to_string(), + }], + environment: EnvironmentSpec::NyxVm(VmEnvironmentSpec { + firecracker_config_asset: "firecracker".to_string(), + instance_id: "vm-test".to_string(), + shared_region_name: "shared".to_string(), + shared_region_size: 4096, + shared_memory_policy: SharedMemoryPolicySpec::Snapshot, + step_timeout_ms: 100, + boot_timeout_ms: 1_000, + episode_steps: 4, + step_cost: 0, + observation_policy: VmObservationPolicySpec::OutputHash, + observation_bits: 8, + observation_stream_len: 16, + observation_stream_mode: VmObservationStreamModeSpec::PadTruncate, + observation_pad_byte: 0, + reward_bits: 8, + reward_policy: VmRewardPolicySpec::FromGuest, + reward_shaping: None, + action_source: VmRuntimeActionSourceSpec::Fuzz { + seeds: vec!["seed".to_string()], + encoding: VmPayloadEncodingSpec::Utf8, + mutators: vec![VmFuzzMutatorSpec::FlipBit, VmFuzzMutatorSpec::SpliceSeed], + min_len: 1, + max_len: 16, + dictionary: vec!["tok".to_string()], + rng_seed: 7, + }, + action_filter: None, + action_prefix: "ACT ".to_string(), + action_suffix: "\n".to_string(), + obs_prefix: "OBS ".to_string(), + rew_prefix: "REW ".to_string(), + done_prefix: "DONE ".to_string(), + data_prefix: "DATA ".to_string(), + wire_encoding: VmPayloadEncodingSpec::Utf8, + stats_backend: RateBackend::Ctw { depth: 8 }, + trace: None, + debug_mode: false, + crash_log: None, + }), + interface: PlannerInterfaceSpec { + observation_bits: 8, + observation_stream_len: 16, + observation_key_mode: ObservationKeyMode::FullStream, + reward_bits: 8, + agent_actions: action_alphabet(1), + }, + controller: ControllerSpec::McAixi(McAixiControllerSpec { + predictor: RateBackend::Ctw { depth: 8 }, + bit_stream_semantics: default_bit_stream_semantics(), + agent_horizon: 1, + num_simulations: 1, + mcts_strategy: MctsStrategy::RhoUct, + exploration_exploitation_ratio: 1.0, + discount_gamma: 1.0, + }), + runtime: PlannerRuntimeSpec { + random_seed: Some(7), + learn_cycles: Some(1), + eval_cycles: Some(0), + terminate_lifetime: 1, + log_every: 1, + perf: false, + vm_perf_only: false, + explore_epsilon: 0.0, + explore_gamma: 1.0, + }, + } +} + +#[cfg(all(feature = "backend-ctw", feature = "vm"))] +#[test] +fn planner_run_compile_normalizes_vm_aliases_to_canonical_names() { + let compiled = sample_vm_planner_run() + .compile() + .expect("vm planner run should compile"); + let EnvironmentSpec::NyxVm(vm) = &compiled.canonical_spec().environment else { + panic!("expected vm environment"); + }; + assert_eq!(vm.observation_policy, VmObservationPolicySpec::OutputHash); + assert_eq!( + vm.observation_stream_mode, + VmObservationStreamModeSpec::PadTruncate + ); + assert_eq!(vm.wire_encoding, VmPayloadEncodingSpec::Utf8); + match &vm.action_source { + VmRuntimeActionSourceSpec::Fuzz { + encoding, mutators, .. + } => { + assert_eq!(*encoding, VmPayloadEncodingSpec::Utf8); + assert_eq!( + mutators, + &vec![VmFuzzMutatorSpec::FlipBit, VmFuzzMutatorSpec::SpliceSeed] + ); + } + other => panic!("expected fuzz action source, got {other:?}"), + } +} + +#[cfg(all(feature = "backend-ctw", feature = "vm"))] +#[test] +fn planner_run_binary_roundtrip_preserves_vm_fuzz_action_source_layout() { + let spec = sample_vm_planner_run(); + let expected_json = SpecDocument::PlannerRun(spec.clone()) + .to_canonical_json() + .expect("json"); + let bytes = SpecDocument::PlannerRun(spec.clone()).to_binary(); + let reparsed = SpecDocument::from_binary(&bytes, Path::new(".")).expect("binary"); + let reparsed_json = reparsed.to_canonical_json().expect("json"); + + assert_eq!(reparsed_json, expected_json); + + let SpecDocument::PlannerRun(reparsed_spec) = reparsed else { + panic!("expected planner_run document"); + }; + + let EnvironmentSpec::NyxVm(vm) = reparsed_spec.environment else { + panic!("expected vm environment"); + }; + match vm.action_source { + VmRuntimeActionSourceSpec::Fuzz { + seeds, + encoding, + mutators, + min_len, + max_len, + dictionary, + rng_seed, + } => { + assert_eq!(seeds, vec!["seed".to_string()]); + assert_eq!(encoding, VmPayloadEncodingSpec::Utf8); + assert_eq!( + mutators, + vec![VmFuzzMutatorSpec::FlipBit, VmFuzzMutatorSpec::SpliceSeed] + ); + assert_eq!(min_len, 1); + assert_eq!(max_len, 16); + assert_eq!(dictionary, vec!["tok".to_string()]); + assert_eq!(rng_seed, 7); + } + other => panic!("expected fuzz action source, got {other:?}"), + } +} + +#[cfg(all(feature = "backend-ctw", feature = "vm"))] +#[test] +fn planner_run_compile_rejects_unknown_vm_enum_names() { + let unknown = sample_vm_planner_run(); + let mut value = unknown.to_canonical_json_value().expect("canonical json"); + value["environment"] = serde_json::json!({ + "kind": "nyx_vm", + "firecracker_config_asset": "firecracker", + "observation_policy": "nope" + }); + let err = match SpecDocument::parse_json_value(&value, Path::new(".")) { + Ok(_) => panic!("unknown observation policy must fail"), + Err(err) => err, + }; + assert!( + err.to_string().contains("unknown VM observation_policy"), + "{err}" + ); + + let mut unknown_encoding_value = sample_vm_planner_run() + .to_canonical_json_value() + .expect("canonical json"); + unknown_encoding_value["environment"]["protocol"]["wire_encoding"] = + serde_json::json!("base64"); + let err = match SpecDocument::parse_json_value(&unknown_encoding_value, Path::new(".")) { + Ok(_) => panic!("unknown wire encoding must fail"), + Err(err) => err, + }; + assert!( + err.to_string().contains("unknown VM payload encoding"), + "{err}" + ); + + let mut unknown_mutator_value = sample_vm_planner_run() + .to_canonical_json_value() + .expect("canonical json"); + unknown_mutator_value["environment"]["action_source"]["mutators"] = + serde_json::json!(["invalid-mutator"]); + let err = match SpecDocument::parse_json_value(&unknown_mutator_value, Path::new(".")) { + Ok(_) => panic!("unknown mutator must fail"), + Err(err) => err, + }; + assert!(err.to_string().contains("unknown VM fuzz mutator"), "{err}"); +} + +#[cfg(all(feature = "backend-ctw", feature = "tuner"))] +#[test] +fn tune_document_binary_roundtrip_is_stable() { + let spec = sample_tune_spec(); + let expected = spec.to_canonical_json().expect("json"); + let bytes = SpecDocument::Tune(spec.clone()).to_binary(); + let reparsed = SpecDocument::from_binary(&bytes, Path::new(".")).expect("binary"); + match reparsed { + SpecDocument::Tune(parsed) => { + assert_eq!(parsed.to_canonical_json().expect("parsed json"), expected) + } + _ => panic!("expected tune document"), + } +} + +#[cfg(all(feature = "backend-ctw", feature = "tuner"))] +#[test] +fn tune_compile_model_bytes_ignore_outer_request_controls() { + let mut base = sample_tune_spec(); + base.bounds.forbidden_backends = vec![]; + base.bounds.parameter_ranges = vec![]; + base.bounds.required_experts = vec![]; + base.report_path = Some("report-a.json".to_string()); + base.output_config_path = "best-a.json".to_string(); + let mut other = base.clone(); + other.output_config_path = "best-b.json".to_string(); + other.report_path = Some("report-b.json".to_string()); + + let compiled_a = base.compile().expect("compiled tune a"); + let compiled_b = other.compile().expect("compiled tune b"); + assert_eq!( + compiled_a.baseline_candidate().canonical_bytes().as_slice(), + compiled_b.baseline_candidate().canonical_bytes().as_slice() + ); + assert_eq!( + compiled_a.baseline_candidate_model_bytes(), + compiled_b.baseline_candidate_model_bytes() + ); +} + +#[cfg(all(feature = "backend-ctw", not(feature = "aixi")))] +#[test] +fn planner_run_rejects_warmstart_controller_without_aixi_feature() { + let json = serde_json::json!({ + "schema_version": SPEC_DOCUMENT_SCHEMA_VERSION, + "kind": "planner_run", + "assets": [{ + "id": "teacher", + "path": "teacher.json", + }], + "environment": { + "kind": "builtin", + "name": "coin_flip", + }, + "interface": { + "observation_bits": 2, + "observation_stream_len": 1, + "observation_key_mode": "full_stream", + "reward_bits": 2, + "agent_actions": 2, + }, + "controller": { + "kind": "aiqi_warmstart_exact_jh", + "predictor": { "kind": "ctw", "depth": 8 }, + "bit_stream_semantics": { "kind": "binary_tokens" }, + "return_horizon": 2, + "return_bins": 5, + "label_phase_period": 2, + "teacher_dataset_asset": "teacher", + "planner_simulations_per_step": 1, + }, + "runtime": { + "random_seed": 11, + "learn_cycles": 4, + "eval_cycles": 2, + "terminate_lifetime": 6, + "log_every": 1, + "perf": false, + "vm_perf_only": false, + "explore_epsilon": 0.0, + "explore_gamma": 1.0, + }, + }); + let err = match SpecDocument::parse_json_value(&json, Path::new(".")) { + Ok(_) => panic!("warmstart controller must require aixi feature"), + Err(err) => err, + }; + assert!( + err.to_string().contains( + "aiqi_warmstart_exact_jh controller requires infotheory built with feature 'aixi'" + ), + "{err}" + ); +} + +#[cfg(not(feature = "backend-ctw"))] +#[test] +fn planner_run_validation_reports_missing_backend_feature() { + let spec = PlannerRunSpec { + assets: Vec::new(), + environment: EnvironmentSpec::Builtin { + builtin: BuiltinEnvironmentSpec::TicTacToe, + }, + interface: PlannerInterfaceSpec { + observation_bits: 1, + observation_stream_len: 1, + observation_key_mode: ObservationKeyMode::FullStream, + reward_bits: 1, + agent_actions: action_alphabet(2), + }, + controller: ControllerSpec::AiqiDiscounted(AiqiDiscountedControllerSpec { + predictor: RateBackend::Ctw { depth: 8 }, + bit_stream_semantics: default_bit_stream_semantics(), + discount_gamma: 0.99, + return_horizon: 2, + return_bins: 8, + augmentation_period: 2, + history_prune_keep_steps: None, + baseline_exploration: 0.01, + }), + runtime: PlannerRuntimeSpec { + random_seed: Some(7), + learn_cycles: Some(4), + eval_cycles: Some(2), + terminate_lifetime: 4, + log_every: 1, + perf: false, + vm_perf_only: false, + explore_epsilon: 0.0, + explore_gamma: 1.0, + }, + }; + let err = spec.validate().expect_err("missing feature should fail"); + assert!( + err.to_string() + .contains("requires infotheory feature 'backend-ctw'"), + "{err}" + ); +} + +#[test] +fn builtin_environment_canonical_names_round_trip() { + use BuiltinEnvironmentSpec::*; + let cases: &[(BuiltinEnvironmentSpec, &str)] = &[ + (TunerBridge, "tuner_bridge"), + (CoinFlip, "coin_flip"), + (BiasedRockPaperScissor, "biased_rock_paper_scissor"), + (KuhnPoker, "kuhn_poker"), + (ExtendedTiger, "extended_tiger"), + (TicTacToe, "tic_tac_toe"), + (Blackjack, "blackjack"), + (Platformer, "platformer"), + ]; + for (variant, expected) in cases { + assert_eq!( + variant.canonical_name(), + *expected, + "canonical_name() for {variant:?}" + ); + } +} + +#[test] +fn spec_document_kind_str_matches_serialized_kind_field() { + use crate::api::{CompressionBackend, RateBackend}; + use crate::coders::CoderType; + use crate::compression::FramingMode; + + let rate = SpecDocument::RateBackend(RateBackend::Ctw { depth: 4 }); + assert_eq!(rate.kind_str(), "rate_backend"); + + let compression = SpecDocument::CompressionBackend(CompressionBackend::Rate { + rate_backend: RateBackend::Ctw { depth: 4 }, + coder: CoderType::AC, + framing: FramingMode::Framed, + }); + assert_eq!(compression.kind_str(), "compression_backend"); +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn environment_spec_kind_str_is_stable() { + let spec = EnvironmentSpec::Builtin { + builtin: BuiltinEnvironmentSpec::CoinFlip, + }; + assert_eq!(spec.kind_str(), "builtin"); +} diff --git a/crates/infotheory/src/spec/document/types.rs b/crates/infotheory/src/spec/document/types.rs new file mode 100644 index 00000000..b81b5c8d --- /dev/null +++ b/crates/infotheory/src/spec/document/types.rs @@ -0,0 +1,895 @@ +//! Canonical top-level specification document schema types. + +use crate::aixi::common::{ActionAlphabet, MctsStrategy, ObservationKeyMode}; +use crate::api::{BitStreamSemantics, CompressionBackend, RateBackend}; +use crate::spec::core::{ + AssetRef, CanonicalBytes, CompiledCompressionBackend, CompiledRateBackend, + ValidatedCompressionBackend, ValidatedRateBackend, +}; +use std::path::PathBuf; +use std::sync::Arc; + +/// Stable identifier for an external asset binding. +pub type AssetId = String; + +/// Filesystem binding for a named external asset referenced by a spec document. +#[derive(Clone, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub struct AssetBinding { + /// Stable asset identifier used inside canonical specs. + pub id: AssetId, + /// Filesystem path used to resolve the asset at runtime. + pub path: String, +} + +/// Resolved runtime asset binding derived from a canonical asset identifier. +#[derive(Clone, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub struct ResolvedAssetBinding { + /// Stable asset identifier used inside canonical specs. + pub id: AssetId, + /// Resolved asset handle in the current compilation environment. + pub asset: AssetRef, +} + +/// Built-in non-VM environment choices available to planner runs. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub enum BuiltinEnvironmentSpec { + /// Internal planner environment bridge used by tuner controller execution. + TunerBridge, + /// GameEngine biased coin-flip environment. + CoinFlip, + /// GameEngine biased rock-paper-scissor environment. + BiasedRockPaperScissor, + /// GameEngine Kuhn poker environment. + KuhnPoker, + /// GameEngine extended tiger environment. + ExtendedTiger, + /// GameEngine tic-tac-toe environment. + TicTacToe, + /// GameEngine blackjack environment. + Blackjack, + /// GameEngine platformer environment. + Platformer, +} + +/// Shared-memory persistence policy for Nyx VM environments. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub enum SharedMemoryPolicySpec { + /// Preserve the region across resets. + Preserve, + /// Reset from the snapshot baseline each iteration. + Snapshot, +} + +/// Reward shaping configuration for VM environments. +/// +/// Algorithmic configuration for the entropy estimator (such as ROSA's +/// `max_order`) lives inside the active rate backend's variant; the shaping +/// spec only carries shaping-policy parameters. +#[derive(Clone, Debug, PartialEq)] +#[non_exhaustive] +pub enum VmRewardShapingSpec { + /// Entropy reduction relative to a baseline asset. + EntropyReduction { + /// Asset identifier providing baseline bytes. + baseline_asset: AssetId, + /// Linear scale applied to the shaping reward. + scale: f64, + /// Optional bonus applied on crash exits. + crash_bonus: Option, + /// Optional bonus applied on timeout exits. + timeout_bonus: Option, + }, + /// Trace entropy shaping using online trace bytes. + TraceEntropy { + /// Linear scale applied to the shaping reward. + scale: f64, + /// Whether to normalize by trace length. + normalize: bool, + }, +} + +/// Reward policy for canonical VM environment specs. +#[derive(Clone, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub enum VmRewardPolicySpec { + /// Parse reward directly from the guest protocol. + FromGuest, + /// Pattern-based reward shaping against guest output. + Pattern { + /// Pattern searched in guest output. + pattern: String, + /// Reward applied when the pattern does not match. + base_reward: i64, + /// Additional reward applied on match. + bonus_reward: i64, + }, +} + +/// Optional information-theoretic action filtering for VM runs. +/// +/// Algorithmic configuration for the entropy estimator (such as ROSA's +/// `max_order`) lives inside the active rate backend's variant; the filter +/// spec only carries filter-policy thresholds. +#[derive(Clone, Debug, PartialEq)] +#[non_exhaustive] +pub struct VmActionFilterSpec { + /// Minimum entropy threshold. + pub min_entropy: Option, + /// Maximum entropy threshold. + pub max_entropy: Option, + /// Minimum intrinsic dependence threshold. + pub min_intrinsic_dependence: Option, + /// Minimum novelty threshold. + pub min_novelty: Option, + /// Optional prior asset used for novelty scoring. + pub novelty_prior_asset: Option, + /// Reward assigned when an action is rejected. + pub reject_reward: Option, +} + +/// Optional VM trace collection settings. +#[derive(Clone, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub struct VmTraceSpec { + /// Shared-memory region name carrying trace bytes. + pub shared_region_name: Option, + /// Maximum trace bytes collected per step. + pub max_bytes: usize, + /// Whether the trace model resets on episode boundaries. + pub reset_on_episode: bool, +} + +/// Canonical observation derivation modes for VM environments. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub enum VmObservationPolicySpec { + /// Parse observations from the guest protocol. + FromGuest, + /// Hash guest output into the observation stream. + OutputHash, + /// Use raw guest output bytes directly. + RawOutput, + /// Read observations from shared memory. + SharedMemory, +} + +/// Canonical normalization modes for VM observation streams. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub enum VmObservationStreamModeSpec { + /// Pad short streams and truncate long streams. + PadTruncate, + /// Only pad short streams. + Pad, + /// Only truncate long streams. + Truncate, +} + +/// Canonical payload encodings for VM action and protocol payloads. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub enum VmPayloadEncodingSpec { + /// Interpret payload strings as UTF-8 text. + Utf8, + /// Interpret payload strings as hexadecimal bytes. + Hex, +} + +/// Canonical fuzz mutator choices for VM action generation. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub enum VmFuzzMutatorSpec { + /// Flip one random bit. + FlipBit, + /// Flip one random byte. + FlipByte, + /// Insert one random byte. + InsertByte, + /// Delete one random byte. + DeleteByte, + /// Splice in bytes from another seed. + SpliceSeed, + /// Reset to a seed input. + ResetSeed, + /// Apply a short random mutation sequence. + Havoc, +} + +/// Canonical runtime action source for VM environments. +#[derive(Clone, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub enum VmRuntimeActionSourceSpec { + /// Inline literal action payloads. + Literal { + /// Human-readable action names. + names: Vec>, + /// Action payloads encoded with the selected payload encoding. + payloads: Vec, + /// Payload encoding applied to all literals. + encoding: VmPayloadEncodingSpec, + }, + /// Mutation-based fuzzing configuration. + Fuzz { + /// Seed inputs encoded with the selected payload encoding. + seeds: Vec, + /// Payload encoding applied to seeds and dictionary entries. + encoding: VmPayloadEncodingSpec, + /// Enabled mutators by canonical name. + mutators: Vec, + /// Minimum generated payload length. + min_len: usize, + /// Maximum generated payload length. + max_len: usize, + /// Optional dictionary entries. + dictionary: Vec, + /// Deterministic RNG seed for mutation sampling. + rng_seed: u64, + }, +} + +/// Canonical Nyx/Firecracker environment configuration. +#[derive(Clone)] +#[non_exhaustive] +pub struct VmEnvironmentSpec { + /// Asset identifier pointing at the Firecracker JSON config. + pub firecracker_config_asset: AssetId, + /// VM instance identifier. + pub instance_id: String, + /// Shared-memory region name used for guest communication. + pub shared_region_name: String, + /// Shared-memory region size in bytes. + pub shared_region_size: usize, + /// Snapshot-vs-preserve policy for shared memory. + pub shared_memory_policy: SharedMemoryPolicySpec, + /// Per-step timeout in milliseconds. + pub step_timeout_ms: u64, + /// Initial boot timeout in milliseconds. + pub boot_timeout_ms: u64, + /// Episode length in steps. + pub episode_steps: usize, + /// Per-step cost subtracted from rewards. + pub step_cost: i64, + /// Observation derivation mode. + pub observation_policy: VmObservationPolicySpec, + /// Observation bit width. + pub observation_bits: usize, + /// Observation stream length. + pub observation_stream_len: usize, + /// Observation stream normalization mode. + pub observation_stream_mode: VmObservationStreamModeSpec, + /// Padding byte for short observation streams. + pub observation_pad_byte: u8, + /// Reward bit width. + pub reward_bits: usize, + /// Reward policy. + pub reward_policy: VmRewardPolicySpec, + /// Optional reward shaping policy. + pub reward_shaping: Option, + /// Runtime action source. + pub action_source: VmRuntimeActionSourceSpec, + /// Optional information-theoretic filter. + pub action_filter: Option, + /// Protocol action prefix. + pub action_prefix: String, + /// Protocol action suffix. + pub action_suffix: String, + /// Protocol observation prefix. + pub obs_prefix: String, + /// Protocol reward prefix. + pub rew_prefix: String, + /// Protocol done prefix. + pub done_prefix: String, + /// Protocol data prefix. + pub data_prefix: String, + /// Payload encoding label (`utf8` or `hex`). + pub wire_encoding: VmPayloadEncodingSpec, + /// Rate backend used for entropy/statistics estimation. + pub stats_backend: RateBackend, + /// Optional trace configuration. + pub trace: Option, + /// Whether to enable verbose VM diagnostics. + pub debug_mode: bool, + /// Optional crash log path for VM exits. + pub crash_log: Option, +} + +/// Canonical planner-visible environment specification. +#[derive(Clone)] +#[non_exhaustive] +pub enum EnvironmentSpec { + /// Built-in Rust environment. + Builtin { + /// Built-in environment kind. + builtin: BuiltinEnvironmentSpec, + }, + /// Nyx/Firecracker VM environment. + #[cfg(feature = "vm")] + NyxVm(VmEnvironmentSpec), +} + +/// Planner observation/reward/action interface contract. +#[derive(Clone, Debug, PartialEq)] +#[non_exhaustive] +pub struct PlannerInterfaceSpec { + /// Observation bit width. + pub observation_bits: usize, + /// Observation stream length. + pub observation_stream_len: usize, + /// Observation key projection used by MC-AIXI. + pub observation_key_mode: ObservationKeyMode, + /// Reward bit width. + pub reward_bits: usize, + /// Action alphabet cardinality. + pub agent_actions: ActionAlphabet, +} + +/// Canonical tuning interface contract for planner-visible I/O shape. +/// +/// Unlike [`PlannerInterfaceSpec`], this tune-document interface is limited to +/// candidate-facing semantics and intentionally excludes reward encoding +/// bounds that belong to executor/runtime policy. +#[cfg(feature = "tuner")] +#[derive(Clone, Debug, PartialEq)] +#[non_exhaustive] +pub struct TunePlannerInterfaceSpec { + /// Observation bit width. + pub observation_bits: usize, + /// Observation stream length. + pub observation_stream_len: usize, + /// Observation key projection used by MC-AIXI. + pub observation_key_mode: ObservationKeyMode, + /// Reward bit width. + pub reward_bits: usize, + /// Action alphabet cardinality. + pub agent_actions: ActionAlphabet, +} + +/// MC-AIXI controller configuration using a unified rate-backend predictor. +#[derive(Clone)] +#[non_exhaustive] +pub struct McAixiControllerSpec { + /// Predictive backend used by the planner model. + pub predictor: RateBackend, + /// Bit-stream semantics used to adapt the predictor to AIXI symbols. + pub bit_stream_semantics: BitStreamSemantics, + /// Planning horizon. + pub agent_horizon: usize, + /// Number of simulations per planning step. + pub num_simulations: usize, + /// Explicit MCTS strategy used by the planner. + pub mcts_strategy: MctsStrategy, + /// UCT exploration constant. + pub exploration_exploitation_ratio: f64, + /// Reward discount factor. + pub discount_gamma: f64, +} + +/// Discounted AIQI controller configuration. +#[derive(Clone)] +#[non_exhaustive] +pub struct AiqiDiscountedControllerSpec { + /// Predictive backend used by the return model. + pub predictor: RateBackend, + /// Bit-stream semantics used to adapt the predictor to AIQI symbols. + pub bit_stream_semantics: BitStreamSemantics, + /// Discount factor used for return construction. + pub discount_gamma: f64, + /// Return horizon. + pub return_horizon: usize, + /// Number of discrete return bins. + pub return_bins: usize, + /// Label augmentation period. + pub augmentation_period: usize, + /// Optional bounded-history retention hint. + pub history_prune_keep_steps: Option, + /// Baseline exploration probability. + pub baseline_exploration: f64, +} + +/// Warm-start exact-\u{1d4a5}_H controller configuration. +#[cfg(feature = "aixi")] +#[derive(Clone)] +#[non_exhaustive] +pub struct WarmStartExactJhControllerSpec { + /// Predictive backend used by the return model. + pub predictor: RateBackend, + /// Bit-stream semantics used to adapt the predictor to warm-start symbols. + pub bit_stream_semantics: BitStreamSemantics, + /// Return horizon in planner steps. + pub return_horizon: usize, + /// Exact return-label alphabet size. + /// + /// Valid canonical warm-start specs use `return_horizon * max_reward + 1`; + /// slack labels are rejected because they do not represent reachable exact + /// returns. + pub return_bins: usize, + /// Delayed-label phase period. + pub label_phase_period: usize, + /// Asset identifier for the warm-start teacher dataset. + pub teacher_dataset_asset: AssetId, + /// Canonical direct-evaluator budget marker. + /// + /// Warm-start exact-\(J_H\) performs deterministic full return-law + /// evaluation, not MCTS-style simulation. Valid canonical specs use `1`. + pub planner_simulations_per_step: usize, +} + +/// Canonical planner controller selection. +#[derive(Clone)] +#[non_exhaustive] +pub enum ControllerSpec { + /// Monte Carlo AIXI. + McAixi(McAixiControllerSpec), + /// Discounted AIQI. + AiqiDiscounted(AiqiDiscountedControllerSpec), + /// Warm-start exact-\u{1d4a5}_H AIQI-style controller. + #[cfg(feature = "aixi")] + AiqiWarmstartExactJh(WarmStartExactJhControllerSpec), +} + +/// Operational planner-run controls that do not change predictor semantics. +#[derive(Clone, Debug, PartialEq)] +#[non_exhaustive] +pub struct PlannerRuntimeSpec { + /// Seed used for planner/environment stochasticity. + /// + /// `None` canonicalizes to `Some(0)` during validation/compilation. + pub random_seed: Option, + /// Number of learning cycles. + pub learn_cycles: Option, + /// Number of evaluation cycles. + pub eval_cycles: Option, + /// Default cycle count when learn/eval are omitted. + pub terminate_lifetime: usize, + /// Logging interval in steps. + pub log_every: usize, + /// Whether to print throughput diagnostics. + pub perf: bool, + /// Whether to run VM perf-only mode. + pub vm_perf_only: bool, + /// Extra epsilon exploration used during execution. + pub explore_epsilon: f64, + /// Exponential decay for extra exploration. + pub explore_gamma: f64, +} + +/// Canonical planner-run specification. +#[derive(Clone)] +#[non_exhaustive] +pub struct PlannerRunSpec { + /// External asset bindings referenced by this run. + pub assets: Vec, + /// Planner-facing environment. + pub environment: EnvironmentSpec, + /// Planner observation/reward/action contract. + pub interface: PlannerInterfaceSpec, + /// Controller configuration. + pub controller: ControllerSpec, + /// Operational run controls. + pub runtime: PlannerRuntimeSpec, +} + +/// Tuning controller kind specified by the formal tuner document. +#[cfg(feature = "tuner")] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub enum TuneControllerKind { + /// Annealed hill climbing. + AnnealedHillClimbing, + /// MC-AIXI(FAC-CTW). + McAixiFacCtw, + /// Discounted AIQI. + AiqiDiscounted, + /// Warm-start exact-\u{1d4a5}_H controller. + #[cfg(feature = "aixi")] + AiqiWarmstartExactJh, +} + +/// Annealed hill-climbing controller settings for tuning. +#[cfg(feature = "tuner")] +#[derive(Clone, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub struct AnnealedHillClimbingTuneControllerSpec { + /// Maximum mutation radius applied to a candidate step. + pub max_mutation_radius: usize, +} + +/// MC-AIXI(FAC-CTW) controller settings for tuning. +#[cfg(feature = "tuner")] +#[derive(Clone, Debug, PartialEq)] +#[non_exhaustive] +pub struct McAixiFacCtwTuneControllerSpec { + /// Planner/environment observation/reward/action contract. + pub interface: TunePlannerInterfaceSpec, + /// Simulation budget per planner step. + pub planner_simulations_per_step: usize, +} + +/// Discounted AIQI controller settings for tuning. +#[cfg(feature = "tuner")] +#[derive(Clone, Debug, PartialEq)] +#[non_exhaustive] +pub struct AiqiDiscountedTuneControllerSpec { + /// Planner/environment observation/reward/action contract. + pub interface: TunePlannerInterfaceSpec, + /// Simulation budget per planner step. + pub planner_simulations_per_step: usize, + /// Return horizon. + pub return_horizon: usize, + /// Number of return bins. + pub return_bins: usize, + /// Discount factor used to construct returns. + pub discount_factor: f64, + /// Minimum clipped improvement value. + pub min_improvement: f64, + /// Maximum clipped improvement value. + pub max_improvement: f64, +} + +/// Warm-start exact-J_H controller settings for tuning. +#[cfg(feature = "tuner")] +#[derive(Clone, Debug, PartialEq)] +#[non_exhaustive] +pub struct WarmStartExactJhTuneControllerSpec { + /// Planner/environment observation/reward/action contract. + pub interface: TunePlannerInterfaceSpec, + /// Canonical direct-evaluator budget marker. + /// + /// Warm-start exact-\(J_H\) performs deterministic full return-law + /// evaluation, not MCTS-style simulation. Valid canonical specs use `1`. + pub planner_simulations_per_step: usize, + /// Return horizon. + pub return_horizon: usize, + /// Teacher dataset asset for warm-start labels. + pub warmstart_teacher_dataset_asset: AssetId, + /// Label phase period. + pub label_phase_period: usize, +} + +/// Runtime-selectable controller configuration for the tuning runtime. +#[cfg(feature = "tuner")] +#[derive(Clone, Debug, PartialEq)] +#[non_exhaustive] +pub enum TuneControllerSpec { + /// Annealed hill climbing. + AnnealedHillClimbing(AnnealedHillClimbingTuneControllerSpec), + /// MC-AIXI(FAC-CTW). + McAixiFacCtw(McAixiFacCtwTuneControllerSpec), + /// Discounted AIQI. + AiqiDiscounted(AiqiDiscountedTuneControllerSpec), + /// Warm-start exact-J_H. + #[cfg(feature = "aixi")] + AiqiWarmstartExactJh(WarmStartExactJhTuneControllerSpec), +} + +#[cfg(feature = "tuner")] +impl TuneControllerSpec { + /// Controller family tag for this tuning controller configuration. + pub fn kind(&self) -> TuneControllerKind { + match self { + Self::AnnealedHillClimbing(_) => TuneControllerKind::AnnealedHillClimbing, + Self::McAixiFacCtw(_) => TuneControllerKind::McAixiFacCtw, + Self::AiqiDiscounted(_) => TuneControllerKind::AiqiDiscounted, + #[cfg(feature = "aixi")] + Self::AiqiWarmstartExactJh(_) => TuneControllerKind::AiqiWarmstartExactJh, + } + } +} + +#[cfg(all(test, feature = "tuner"))] +mod tests { + use super::*; + + fn action_alphabet(n: usize) -> ActionAlphabet { + ActionAlphabet::try_from_usize(n).expect("test action alphabet must be non-zero") + } + + fn sample_tune_interface() -> TunePlannerInterfaceSpec { + TunePlannerInterfaceSpec { + observation_bits: 8, + observation_stream_len: 1, + observation_key_mode: ObservationKeyMode::FullStream, + reward_bits: 8, + agent_actions: action_alphabet(2), + } + } + + #[test] + fn tune_controller_kind_matches_each_variant() { + let annealed = + TuneControllerSpec::AnnealedHillClimbing(AnnealedHillClimbingTuneControllerSpec { + max_mutation_radius: 3, + }); + assert_eq!(annealed.kind(), TuneControllerKind::AnnealedHillClimbing); + + let mc_aixi = TuneControllerSpec::McAixiFacCtw(McAixiFacCtwTuneControllerSpec { + interface: sample_tune_interface(), + planner_simulations_per_step: 8, + }); + assert_eq!(mc_aixi.kind(), TuneControllerKind::McAixiFacCtw); + + let aiqi = TuneControllerSpec::AiqiDiscounted(AiqiDiscountedTuneControllerSpec { + interface: sample_tune_interface(), + planner_simulations_per_step: 8, + return_horizon: 2, + return_bins: 8, + discount_factor: 0.5, + min_improvement: -1.0, + max_improvement: 1.0, + }); + assert_eq!(aiqi.kind(), TuneControllerKind::AiqiDiscounted); + + #[cfg(feature = "aixi")] + { + let warmstart = + TuneControllerSpec::AiqiWarmstartExactJh(WarmStartExactJhTuneControllerSpec { + interface: sample_tune_interface(), + planner_simulations_per_step: 1, + return_horizon: 2, + warmstart_teacher_dataset_asset: "teacher".to_string(), + label_phase_period: 3, + }); + assert_eq!(warmstart.kind(), TuneControllerKind::AiqiWarmstartExactJh); + } + } +} + +/// Bounded numeric range for a named canonical tuning parameter. +#[cfg(feature = "tuner")] +#[derive(Clone, Debug, PartialEq)] +#[non_exhaustive] +pub struct TuneParameterRangeSpec { + /// Canonical parameter path or name. + pub parameter: String, + /// Inclusive lower bound. + pub min: f64, + /// Inclusive upper bound. + pub max: f64, +} + +/// Canonical bounds specification for the future tuning runtime. +#[cfg(feature = "tuner")] +#[derive(Clone, Debug, PartialEq)] +#[non_exhaustive] +pub struct TuneBoundsSpec { + /// Allowed canonical backend names. + pub allowed_backends: Vec, + /// Forbidden canonical backend names. + pub forbidden_backends: Vec, + /// Inclusive ranges for canonical numeric parameters. + pub parameter_ranges: Vec, + /// Maximum experts per mixture node. + pub max_experts: usize, + /// Maximum recursive mixture nesting depth. + pub max_mixture_nesting_depth: usize, + /// Optional minimum experts per mixture node. + pub min_experts: Option, + /// Whether duplicate experts are permitted. + pub allow_duplicate_experts: Option, + /// Expert names that must appear in the candidate set. + pub required_experts: Vec, + /// Canonical expert-pair combinations that are forbidden together. + pub forbidden_expert_pairs: Vec<(String, String)>, +} + +/// Canonical future-facing tune request document. +#[cfg(feature = "tuner")] +#[derive(Clone)] +#[non_exhaustive] +pub struct TuneSpec { + /// External asset bindings referenced by this request. + pub assets: Vec, + /// Asset identifier for the input dataset. + pub input_asset: AssetId, + /// Baseline candidate configuration. + pub baseline_candidate: CompressionBackend, + /// Runtime-selectable controller used by the tuning runtime. + pub controller: TuneControllerSpec, + /// Candidate bounds specification. + pub bounds: TuneBoundsSpec, + /// Per-candidate evaluation time limit in seconds. + pub eval_time_limit_seconds: f64, + /// Total search budget in seconds. + pub time_budget_seconds: f64, + /// Minimum required throughput in bytes/second. + pub min_throughput_bytes_per_second: f64, + /// Maximum allowed memory use in bytes. + pub max_memory_bytes: u64, + /// Output path for the best canonical candidate. + pub output_config_path: String, + /// Deterministic search seed. + pub seed: u64, + /// Optional report output path. + pub report_path: Option, +} + +/// Compiled planner controller with precompiled predictor backends. +#[derive(Clone)] +#[non_exhaustive] +pub enum CompiledPlannerController { + /// MC-AIXI controller. + McAixi { + /// Compiled predictor backend. + predictor: CompiledRateBackend, + /// Bit-stream semantics used to adapt the predictor to AIXI symbols. + bit_stream_semantics: BitStreamSemantics, + /// Planning horizon. + agent_horizon: usize, + /// Number of simulations per planning step. + num_simulations: usize, + /// Explicit MCTS strategy used by the planner. + mcts_strategy: MctsStrategy, + /// UCT exploration constant. + exploration_exploitation_ratio: f64, + /// Reward discount factor. + discount_gamma: f64, + }, + /// Discounted AIQI controller. + AiqiDiscounted { + /// Compiled predictor backend. + predictor: CompiledRateBackend, + /// Bit-stream semantics used to adapt the predictor to AIQI symbols. + bit_stream_semantics: BitStreamSemantics, + /// Discount factor used for return construction. + discount_gamma: f64, + /// Return horizon. + return_horizon: usize, + /// Number of return bins. + return_bins: usize, + /// Label augmentation period. + augmentation_period: usize, + /// Optional bounded-history retention hint. + history_prune_keep_steps: Option, + /// Baseline exploration probability. + baseline_exploration: f64, + }, + /// Warm-start exact-J_H controller. + #[cfg(feature = "aixi")] + AiqiWarmstartExactJh { + /// Compiled predictor backend. + predictor: CompiledRateBackend, + /// Bit-stream semantics used to adapt the predictor to warm-start symbols. + bit_stream_semantics: BitStreamSemantics, + /// Return horizon. + return_horizon: usize, + /// Number of return bins. + return_bins: usize, + /// Label phase period. + label_phase_period: usize, + /// Teacher dataset asset id. + teacher_dataset_asset: AssetId, + /// Canonical direct-evaluator budget marker. + planner_simulations_per_step: usize, + }, +} + +/// Compiled planner-run specification with resolved assets and compiled backends. +#[derive(Clone)] +#[non_exhaustive] +pub struct CompiledPlannerRunSpec { + pub(super) canonical_spec: Arc, + pub(super) canonical_bytes: CanonicalBytes, + pub(super) resolved_assets: Arc<[ResolvedAssetBinding]>, + pub(super) interface: PlannerInterfaceSpec, + pub(super) runtime: PlannerRuntimeSpec, + pub(super) controller: CompiledPlannerController, + pub(super) action_bits: usize, +} + +/// Compiled tuning controller configuration. +#[cfg(feature = "tuner")] +#[derive(Clone)] +#[non_exhaustive] +pub enum CompiledTuneController { + /// Annealed hill climbing. + AnnealedHillClimbing(AnnealedHillClimbingTuneControllerSpec), + /// MC-AIXI(FAC-CTW). + McAixiFacCtw(McAixiFacCtwTuneControllerSpec), + /// Discounted AIQI. + AiqiDiscounted(AiqiDiscountedTuneControllerSpec), + /// Warm-start exact-J_H. + #[cfg(feature = "aixi")] + AiqiWarmstartExactJh(WarmStartExactJhTuneControllerSpec), +} + +/// Compiled tune request with resolved assets and compiled baseline candidate. +#[cfg(feature = "tuner")] +#[derive(Clone)] +#[non_exhaustive] +pub struct CompiledTuneSpec { + pub(super) canonical_spec: Arc, + pub(super) canonical_bytes: CanonicalBytes, + pub(super) base_dir: PathBuf, + pub(super) resolved_assets: Arc<[ResolvedAssetBinding]>, + pub(super) baseline_candidate: CompiledCompressionBackend, + pub(super) controller: CompiledTuneController, + pub(super) candidate_canonicalization_version: &'static str, +} + +/// Universal top-level spec document. +#[derive(Clone)] +#[non_exhaustive] +#[allow(clippy::large_enum_variant)] +pub enum SpecDocument { + /// Planner-run configuration document. + PlannerRun(PlannerRunSpec), + /// Tune request document. + #[cfg(feature = "tuner")] + Tune(TuneSpec), + /// Standalone rate-backend document. + RateBackend(RateBackend), + /// Standalone compression-backend document. + CompressionBackend(CompressionBackend), +} + +/// Parsed top-level spec document paired with its parse base directory. +/// +/// This is the first stage of the canonical pipeline: +/// parse -> validate -> compile. +#[derive(Clone)] +#[non_exhaustive] +pub struct ParsedSpecDocument { + pub(super) document: SpecDocument, + pub(super) base_dir: PathBuf, +} + +/// Canonicalized and validated planner-run document. +#[derive(Clone)] +#[non_exhaustive] +pub struct ValidatedPlannerRunSpec { + pub(super) canonical_spec: Arc, + pub(super) canonical_bytes: CanonicalBytes, + pub(super) base_dir: PathBuf, +} + +/// Canonicalized and validated tune request document. +#[cfg(feature = "tuner")] +#[derive(Clone)] +#[non_exhaustive] +pub struct ValidatedTuneSpec { + pub(super) canonical_spec: Arc, + pub(super) canonical_bytes: CanonicalBytes, + #[cfg(feature = "tuner")] + pub(super) base_dir: PathBuf, +} + +/// Validated top-level spec document. +/// +/// This is the second stage of the canonical pipeline and can be compiled into +/// runtime-ready plans/backends. +#[derive(Clone)] +#[non_exhaustive] +pub enum ValidatedSpecDocument { + /// Validated planner-run document. + PlannerRun(ValidatedPlannerRunSpec), + /// Validated tune document. + #[cfg(feature = "tuner")] + Tune(ValidatedTuneSpec), + /// Validated standalone rate-backend document. + RateBackend(ValidatedRateBackend), + /// Validated standalone compression-backend document. + CompressionBackend(ValidatedCompressionBackend), +} + +/// Compiled top-level spec document. +/// +/// This is the final stage of the canonical pipeline and is executable by +/// runtime adapters. +#[derive(Clone)] +#[non_exhaustive] +#[allow(clippy::large_enum_variant)] +pub enum CompiledSpecDocument { + /// Compiled planner-run document. + PlannerRun(CompiledPlannerRunSpec), + /// Compiled tune document. + #[cfg(feature = "tuner")] + Tune(CompiledTuneSpec), + /// Compiled standalone rate-backend document. + RateBackend(CompiledRateBackend), + /// Compiled standalone compression-backend document. + CompressionBackend(CompiledCompressionBackend), +} diff --git a/crates/infotheory/src/tuner.rs b/crates/infotheory/src/tuner.rs new file mode 100644 index 00000000..a3dc9a18 --- /dev/null +++ b/crates/infotheory/src/tuner.rs @@ -0,0 +1,2405 @@ +//! Tuner execution profile and CLI-facing tune runner. +//! +//! `SpecDocument::Tune` remains the canonical candidate/request surface. +//! Runtime controls in this module are executor-side and must not mutate +//! canonical candidate identity. + +use crate::aixi::agent::Agent; +use crate::aixi::aiqi::AiqiAgent; +use crate::aixi::common::{ + Action, MctsStrategy, ObservationKeyMode, PerceptVal, RandomGenerator, Reward, +}; +use crate::aixi::warmstart::{ + WarmStartExactJhAgent, WarmStartExactJhTeacherDataset, WarmStartExactJhTeacherTrace, + merge_warmstart_teacher_trace_deterministic, +}; +use crate::api::RateBackend; +use crate::runtime::CompressionRuntime; +use crate::spec::{ + AiqiDiscountedControllerSpec, AssetRef, BuiltinEnvironmentSpec, CanonicalJson, + CompiledPlannerRunSpec, ControllerSpec, EnvironmentSpec, McAixiControllerSpec, + PlannerInterfaceSpec, PlannerRunSpec, PlannerRuntimeSpec, SpecDocument, SpecEnvironment, + TuneInvalidReason, WarmStartExactJhControllerSpec, canonical_json_bytes, load_spec_document, +}; +use crc32fast::Hasher; +use serde_json::Value; +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::fs; +use std::io::Write; +use std::path::Path; +use std::time::{Duration, Instant}; + +mod annealer; +mod causal_dataset; +mod certificates; +use certificates::validate_complete_finite_reward_interval; +#[cfg(test)] +use certificates::{parse_finite_reward_map, project_observation_output}; +mod config; +mod eval; +mod planner_bridge; +mod report; +use crate::aixi::common::max_nonnegative_reward_for_bits; +use annealer::{ + annealer_acceptance_probability, annealer_active_radius, annealer_progress, + annealer_runtime_path_name, annealer_temperature, collect_numeric_leaves, + sample_annealed_proposal, +}; +#[cfg(test)] +use annealer::{annealer_progress_from_elapsed, compile_canonical_proposal_kernel}; +use causal_dataset::load_dataset; +pub use config::{ + AnnealerKernelProfile, PeakMemoryMode, TimingCertificationTier, TuneCommandRequest, + TuneExecutionConfig, TuneTheoremConfig, parse_tune_command_args, +}; +use config::{ + annealer_kernel_profile_name, compiled_feature_set, peak_memory_mode_name, timing_tier_name, +}; +#[cfg(test)] +use eval::evaluate_candidate_causal_loss; +pub use eval::run_tuner_eval_worker_from_env; +use eval::{ + ResolvedEvaluatorRuntimeProfile, cache_key_for_candidate, evaluate_candidate, + resolve_evaluator_runtime_profile, timeout_eval_result, +}; +#[cfg(test)] +use planner_bridge::{ + TunerRawObservation, compile_tuner_planner_run_spec, encode_tuner_planner_percept, + merge_warmstart_trace_deterministic, planner_controller_contract, + validate_theorem_planner_mutation_domain, +}; +use planner_bridge::{ + exact_nonnegative_i64_from_f64, key_less, normalized_clipped_improvement, + run_planner_family_controller, +}; +#[cfg(test)] +use report::exact_finite_mdp_missing_prereqs; +use report::{ + causal_profile_report, controller_kind_name, dataset_kind_name, diagnostic_chunking_report, + evaluator_execution_model, executor_controls_report, objective_target_name, + observation_key_mode_name, planner_completed_status, planner_deployability_report, + planner_runtime_path_name, theorem_claims_report, theorem_timing_basis, +}; + +const PASSIVE_DATASET_LOWERING_VERSION: &str = "passive-bytes-v1"; +const INTERACTIVE_TRACE_LOWERING_VERSION: &str = "interactive-trace-events-v1"; +const CAUSAL_PREFIX_LOWERING_VERSION: &str = "causal-prefix-examples-v1"; +const TUNER_EVALUATOR_INTERFACE_VERSION: &str = "typed-causal-evaluator-v1"; +const OBSERVATION_ADAPTER_DECLARATION: &str = "single-channel-conditional-byte-adapter-v1"; +const SCALAR_REPRESENTATION_DECLARATION: &str = "finite-ieee754-f64-nonfinite-forbidden-v1"; +const TUNER_MCAIXI_HORIZON: usize = 5; +const TUNER_MCAIXI_FAC_CTW_BASE_DEPTH: usize = 8; +const ANNEALER_T0_BITS: f64 = 1.0; +const ANNEALER_T_MIN_BITS: f64 = 1.0e-3; + +enum ExactObjectiveDifferenceController<'a> { + McAixiFacCtw(&'a crate::spec::McAixiFacCtwTuneControllerSpec), + AiqiWarmstartExactJh(&'a crate::spec::WarmStartExactJhTuneControllerSpec), +} + +impl ExactObjectiveDifferenceController<'_> { + fn reward_bits(&self) -> usize { + match self { + Self::McAixiFacCtw(controller) => controller.interface.reward_bits, + Self::AiqiWarmstartExactJh(controller) => controller.interface.reward_bits, + } + } +} + +impl crate::spec::CompiledTuneController { + fn exact_objective_difference_controller( + &self, + ) -> Option> { + match self { + Self::McAixiFacCtw(controller) => { + Some(ExactObjectiveDifferenceController::McAixiFacCtw(controller)) + } + Self::AiqiWarmstartExactJh(controller) => Some( + ExactObjectiveDifferenceController::AiqiWarmstartExactJh(controller), + ), + Self::AnnealedHillClimbing(_) | Self::AiqiDiscounted(_) => None, + } + } +} + +fn observation_adapter_spec_value() -> Value { + serde_json::json!({ + "kind": OBSERVATION_ADAPTER_DECLARATION, + "schema_version": 1, + "stream": "fixed_len_packed_u64_little_endian", + "fields": [ + "fail_flag", + "normalized_physical_size", + "normalized_target_loss", + "normalized_eval_time", + "physical_size_delta", + "eval_time_delta", + "candidate_signature_crc32", + "terminal" + ], + "missing_sentinel": 0xff_u8, + "nonfinite_float_encoding": "forbidden_before_encoding", + "delta_time_epsilon": 1.0e-9_f64, + }) +} + +fn observation_adapter_content_hash() -> Result { + canonical_json_bytes(&observation_adapter_spec_value()) + .map(|bytes| crc32_hex(&bytes)) + .map_err(|err| format!("failed to encode observation adapter spec: {err}")) +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum DatasetKind { + PassiveBytes, + InteractiveTrace, + CausalPrefixDataset, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum ObjectiveTarget { + PassiveAc, + InteractiveCausalAc, + PlannerDeployableModel, +} + +#[derive(Clone, Debug)] +struct LoadedDataset { + kind: DatasetKind, + objective_target: ObjectiveTarget, + lowering_version: &'static str, + codec_hash: String, + event_grammar_hash: String, + target_domain_support_hash: String, + causal_header_profile_hash: String, + target_size_function: &'static str, + canonical_content_hash: String, + lowered_skeleton_hash: String, + resolved_path: String, + source_size_bytes: usize, + raw_bytes: Vec, + events: Vec, + causal_profile: Option, + dataset_units: f64, + target_events: usize, +} + +#[derive(Clone, Debug)] +struct CausalEvaluationProfile { + domains: BTreeMap, + channel_set: BTreeSet, + domain_support_hash: String, + byte_alphabet_symbol_width: usize, + header_profile_hash: String, + event_grammar: CausalEventGrammar, + action_alphabet_size: usize, + collection_policy: String, + percept_channels: BTreeSet, + reward_channel: CausalChannelDomain, + terminal_channel: CausalChannelDomain, +} + +#[derive(Clone, Debug)] +enum CausalTargetDomain { + ByteAlphabet, + EnumeratedPayloads { payloads: Vec> }, +} + +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd)] +struct CausalChannelDomain { + channel: String, + domain: String, +} + +#[derive(Clone, Debug)] +struct CausalEventGrammar { + context_channels: BTreeSet, + observe_target_no_score: BTreeSet, + target: BTreeSet, +} + +struct PreparedTuneContext { + compiled: crate::spec::CompiledTuneSpec, + dataset: LoadedDataset, + runtime_profile: ResolvedEvaluatorRuntimeProfile, + evaluator_profile: EvaluatorProfile, + initial_eval_limit: f64, + tune_started: Instant, +} + +fn prepare_tune_context(request: &TuneCommandRequest) -> Result { + request.execution.validate()?; + let config_path = Path::new(&request.spec_path); + let config_dir = config_path.parent().unwrap_or(Path::new(".")); + let document = load_spec_document(&request.spec_path).map_err(|err| err.to_string())?; + let SpecDocument::Tune(spec) = document else { + return Err(format!( + "tune expects a tune document, found kind '{}'", + document.kind_str() + )); + }; + let tune_env = SpecEnvironment::new(config_dir); + let compiled = spec.compile_in(&tune_env).map_err(|err| err.to_string())?; + + validate_candidate_against_tune_bounds( + compiled.baseline_candidate().canonical_spec(), + &compiled.canonical_spec().bounds, + )?; + reject_candidate_local_external_artifacts(compiled.baseline_candidate().canonical_spec()) + .map_err(|err| err.diagnostic)?; + + let dataset = load_dataset(resolve_input_asset_path( + &compiled, + &compiled.canonical_spec().input_asset, + )?)?; + let tune_started = Instant::now(); + let initial_eval_limit = effective_eval_limit_seconds( + &compiled, + tune_started, + Some(compiled.canonical_spec().eval_time_limit_seconds), + None, + ); + let runtime_profile = resolve_evaluator_runtime_profile( + &request.execution, + request + .execution + .theorem + .deterministic_evaluator_table + .is_some(), + )?; + let evaluator_profile = EvaluatorProfile { + dataset_kind: dataset.kind, + objective_target: if request.execution.planner_deployable_model { + ObjectiveTarget::PlannerDeployableModel + } else { + dataset.objective_target + }, + dataset_lowering_version: dataset.lowering_version, + dataset_codec_hash: dataset.codec_hash.clone(), + event_grammar_hash: dataset.event_grammar_hash.clone(), + target_domain_support_hash: dataset.target_domain_support_hash.clone(), + causal_header_profile_hash: dataset.causal_header_profile_hash.clone(), + target_size_function: dataset.target_size_function, + evaluator_interface_version: TUNER_EVALUATOR_INTERFACE_VERSION, + candidate_canonicalization_version: compiled + .candidate_canonicalization_version() + .to_string(), + warmup_baseline_runs: request.execution.warmup_baseline_runs, + diagnostic_chunk_bytes: request.execution.diagnostic_chunk_bytes, + eval_time_limit_seconds: initial_eval_limit, + evaluator_threads: request.execution.evaluator_threads(), + worker_isolation_mode: "spawn_exec_worker", + worker_executable_identity: runtime_profile.worker_executable_identity.clone(), + resolved_memory_accounting_kind: runtime_profile.memory_accounting_kind.name(), + resolved_memory_accounting_strict_theorem_facing: runtime_profile + .strict_theorem_memory_certified(), + resolved_evaluator_cgroup_parent: runtime_profile.resolved_cgroup_parent_string(), + backend_report_component_policy: runtime_profile + .memory_accounting_kind + .backend_report_component_policy(), + evaluator_determinism: request.execution.evaluator_determinism(), + rss_mode: request.execution.rss_mode, + timing_certification_tier: request.execution.theorem.timing_certification_tier, + build_profile: option_env!("PROFILE").unwrap_or("unknown"), + feature_set: compiled_feature_set(), + }; + + Ok(PreparedTuneContext { + compiled, + dataset, + runtime_profile, + evaluator_profile, + initial_eval_limit, + tune_started, + }) +} + +fn emit_exact_reward_encoding_certificate( + request: &TuneCommandRequest, + path: &str, + prepared: &PreparedTuneContext, +) -> Result<(), String> { + let controller_kind = controller_kind_name(prepared.compiled.controller()); + let scalar_representation = request + .execution + .theorem + .scalar_representation_ref + .as_deref() + .unwrap_or(SCALAR_REPRESENTATION_DECLARATION); + let Some(exact_controller) = prepared + .compiled + .controller() + .exact_objective_difference_controller() + else { + return Err(format!( + "exact reward-encoding certificate emission is only supported for exact-objective controller families (mc_aixi_fac_ctw, aiqi_warmstart_exact_jh); found '{controller_kind}'" + )); + }; + let reward_bits = exact_controller.reward_bits(); + let action_alphabet_size = planner_action_count(&prepared.compiled)?; + let cert = serde_json::json!({ + "schema_version": 1, + "kind": "exact_reward_encoding", + "dataset_crc32": prepared.dataset.canonical_content_hash, + "bounds_crc32": bounds_hash(&prepared.compiled.canonical_spec().bounds)?, + "evaluator_profile_crc32": prepared.evaluator_profile.hash()?, + "controller_kind": controller_kind, + "action_alphabet_size": action_alphabet_size, + "encoding": "integer_objective_difference", + "scalar_representation": scalar_representation, + "reward_bits": reward_bits, + "max_reward": max_nonnegative_reward_for_bits(reward_bits)?, + }); + let bytes = serde_json::to_vec_pretty(&cert) + .map_err(|err| format!("failed to serialize exact reward certificate JSON: {err}"))?; + fs::write(path, bytes) + .map_err(|err| format!("failed to write exact reward certificate '{}': {err}", path))?; + Ok(()) +} + +#[derive(Clone, Debug)] +struct CausalHeaderProfile { + action_alphabet_size: usize, + collection_policy: String, + percept_channels: BTreeSet, + reward_channel: CausalChannelDomain, + terminal_channel: CausalChannelDomain, + event_grammar: CausalEventGrammar, + profile_hash: String, +} + +#[derive(Clone, Debug, PartialEq)] +struct EvaluatorProfile { + dataset_kind: DatasetKind, + objective_target: ObjectiveTarget, + dataset_lowering_version: &'static str, + dataset_codec_hash: String, + event_grammar_hash: String, + target_domain_support_hash: String, + causal_header_profile_hash: String, + target_size_function: &'static str, + evaluator_interface_version: &'static str, + candidate_canonicalization_version: String, + warmup_baseline_runs: usize, + diagnostic_chunk_bytes: Option, + eval_time_limit_seconds: f64, + evaluator_threads: usize, + worker_isolation_mode: &'static str, + worker_executable_identity: Option, + resolved_memory_accounting_kind: &'static str, + resolved_memory_accounting_strict_theorem_facing: bool, + resolved_evaluator_cgroup_parent: Option, + backend_report_component_policy: &'static str, + evaluator_determinism: &'static str, + rss_mode: PeakMemoryMode, + timing_certification_tier: TimingCertificationTier, + build_profile: &'static str, + feature_set: Vec<&'static str>, +} + +impl EvaluatorProfile { + fn with_eval_time_limit(&self, eval_time_limit_seconds: f64) -> Self { + let mut profile = self.clone(); + profile.eval_time_limit_seconds = eval_time_limit_seconds; + profile + } + + fn to_json_value(&self) -> Value { + serde_json::json!({ + "dataset_kind": dataset_kind_name(self.dataset_kind), + "objective_target": objective_target_name(self.objective_target), + "dataset_lowering_version": self.dataset_lowering_version, + "dataset_codec_hash": self.dataset_codec_hash, + "event_grammar_hash": self.event_grammar_hash, + "target_domain_support_hash": self.target_domain_support_hash, + "causal_header_profile_hash": self.causal_header_profile_hash, + "target_size_function": self.target_size_function, + "evaluator_interface_version": self.evaluator_interface_version, + "candidate_canonicalization_version": self.candidate_canonicalization_version, + "warmup_baseline_runs": self.warmup_baseline_runs, + "diagnostic_chunk_bytes": self.diagnostic_chunk_bytes, + "effective_eval_time_limit_seconds": self.eval_time_limit_seconds, + "evaluator_threads": self.evaluator_threads, + "worker_isolation_mode": self.worker_isolation_mode, + "worker_executable_identity": self.worker_executable_identity.as_deref(), + "resolved_memory_accounting_kind": self.resolved_memory_accounting_kind, + "resolved_memory_accounting_strict_theorem_facing": self.resolved_memory_accounting_strict_theorem_facing, + "resolved_evaluator_cgroup_parent": self.resolved_evaluator_cgroup_parent.as_deref(), + "backend_report_component_policy": self.backend_report_component_policy, + "evaluator_determinism": self.evaluator_determinism, + "rss_mode": peak_memory_mode_name(self.rss_mode), + "timing_certification_tier": timing_tier_name(self.timing_certification_tier), + "build_profile": self.build_profile, + "feature_set": self.feature_set, + }) + } + + fn hash(&self) -> Result { + Ok(crc32_hex(&self.cache_identity_bytes()?)) + } + + fn cache_identity_bytes(&self) -> Result, String> { + let value = serde_json::json!({ + "dataset_kind": dataset_kind_name(self.dataset_kind), + "objective_target": objective_target_name(self.objective_target), + "dataset_lowering_version": self.dataset_lowering_version, + "dataset_codec_hash": self.dataset_codec_hash, + "event_grammar_hash": self.event_grammar_hash, + "target_domain_support_hash": self.target_domain_support_hash, + "causal_header_profile_hash": self.causal_header_profile_hash, + "target_size_function": self.target_size_function, + "evaluator_interface_version": self.evaluator_interface_version, + "candidate_canonicalization_version": self.candidate_canonicalization_version, + "warmup_baseline_runs": self.warmup_baseline_runs, + "diagnostic_chunk_bytes": self.diagnostic_chunk_bytes, + "effective_eval_time_limit_seconds_bits": self.eval_time_limit_seconds.to_bits(), + "evaluator_threads": self.evaluator_threads, + "worker_isolation_mode": self.worker_isolation_mode, + "worker_executable_identity": self.worker_executable_identity.as_deref(), + "resolved_memory_accounting_kind": self.resolved_memory_accounting_kind, + "resolved_memory_accounting_strict_theorem_facing": self.resolved_memory_accounting_strict_theorem_facing, + "resolved_evaluator_cgroup_parent": self.resolved_evaluator_cgroup_parent.as_deref(), + "backend_report_component_policy": self.backend_report_component_policy, + "evaluator_determinism": self.evaluator_determinism, + "rss_mode": peak_memory_mode_name(self.rss_mode), + "timing_certification_tier": timing_tier_name(self.timing_certification_tier), + "build_profile": self.build_profile, + "feature_set": self.feature_set, + }); + canonical_json_bytes(&value) + .map_err(|err| format!("failed to encode evaluator profile JSON: {err}")) + } +} + +#[derive(Clone, Debug)] +struct CandidateEvalResult { + status: CandidateEvalStatus, + compressed_bytes: usize, + elapsed_seconds: f64, + effective_eval_time_limit_seconds: f64, + throughput_bytes_per_second: f64, + peak_memory_bytes: u64, + target_loss_bits: f64, + objective_bits: f64, + deployable: bool, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum CandidateEvalStatus { + Success, + Timeout, + Invalid, + Error, +} + +impl CandidateEvalStatus { + fn name(self) -> &'static str { + match self { + CandidateEvalStatus::Success => "success", + CandidateEvalStatus::Timeout => "timeout", + CandidateEvalStatus::Invalid => "invalid", + CandidateEvalStatus::Error => "error", + } + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +enum CandidateEvalFailure { + FatalEvaluatorFailure { diagnostic: String }, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +struct CandidateInvalidDiagnostic { + reason: TuneInvalidReason, + diagnostic: String, +} + +#[derive(Clone, Debug, Default)] +struct CandidateResultCounts { + success_deployable: usize, + success_non_deployable: usize, + timeout: usize, + invalid: usize, + error_recoverable: usize, +} + +impl CandidateResultCounts { + fn record_admitted_result(&mut self, value: &CandidateEvalResult) { + match value.status { + CandidateEvalStatus::Success if value.deployable => { + self.success_deployable = self.success_deployable.saturating_add(1); + } + CandidateEvalStatus::Success => { + self.success_non_deployable = self.success_non_deployable.saturating_add(1); + } + CandidateEvalStatus::Timeout => { + self.timeout = self.timeout.saturating_add(1); + } + CandidateEvalStatus::Invalid => { + self.invalid = self.invalid.saturating_add(1); + } + CandidateEvalStatus::Error => { + self.error_recoverable = self.error_recoverable.saturating_add(1); + } + } + } +} + +#[derive(Clone, Debug, Default)] +struct InvalidReasonCounts { + candidate_external_asset_forbidden: usize, + candidate_out_of_bounds: usize, + candidate_compile_error: usize, + invalid_action_index: usize, + inapplicable_action: usize, +} + +impl InvalidReasonCounts { + fn record(&mut self, reason: TuneInvalidReason) { + match reason { + TuneInvalidReason::CandidateExternalAssetForbidden => { + self.candidate_external_asset_forbidden = + self.candidate_external_asset_forbidden.saturating_add(1); + } + TuneInvalidReason::CandidateOutOfBounds => { + self.candidate_out_of_bounds = self.candidate_out_of_bounds.saturating_add(1); + } + TuneInvalidReason::CandidateCompileError => { + self.candidate_compile_error = self.candidate_compile_error.saturating_add(1); + } + TuneInvalidReason::InvalidActionIndex => { + self.invalid_action_index = self.invalid_action_index.saturating_add(1); + } + TuneInvalidReason::InapplicableAction => { + self.inapplicable_action = self.inapplicable_action.saturating_add(1); + } + } + } +} + +#[derive(Clone, Debug)] +struct CandidateTopologyStats { + backend_families: BTreeSet, + max_mixture_nesting_depth: usize, + mixture_node_expert_counts: Vec, +} + +#[derive(Clone)] +struct SearchSummary { + status: &'static str, + warning: Option, + fatal_evaluator_failure: Option, + fatal_evaluator_failures: usize, + best_candidate: crate::api::CompressionBackend, + best_candidate_crc32: String, + best_eval: CandidateEvalResult, + cache_key_digest: String, + cache_hits: usize, + cache_misses: usize, + candidate_evaluations_executed: usize, + non_warmup_candidate_results_seen: usize, + post_baseline_candidate_results_seen: usize, + proposals_attempted: usize, + proposals_invalid: usize, + self_loop_proposals: usize, + invalid_reason_counts: InvalidReasonCounts, + successful_non_deployable: usize, + candidate_result_counts: CandidateResultCounts, + final_best_move_reward: f64, + realized_trace_counts_by_round: Option>, + trace_refresh_merges_by_round: Option>, + controller_report: Value, +} + +#[derive(Clone, Debug)] +struct NumericLeaf { + path: String, + pointer: String, + kind: NumericKind, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum NumericKind { + Unsigned, + Signed, + Float, +} + +#[derive(Clone)] +struct CanonicalProposalKernel { + transitions: Vec, + total_raw_actions: u64, +} + +#[derive(Clone)] +struct CanonicalProposal { + candidate: crate::api::CompressionBackend, + candidate_canonical_bytes: Vec, + raw_action_count: u64, +} + +#[derive(Clone)] +struct AnnealedProposal { + candidate: crate::api::CompressionBackend, + forward_raw_action_count: u64, + forward_total_raw_actions: u64, + reverse_raw_action_count: u64, + reverse_total_raw_actions: u64, +} + +#[derive(Clone)] +enum AnnealedProposalDraw { + Proposal(AnnealedProposal), + SelfLoop, + Exhausted, +} + +impl CanonicalProposalKernel { + fn proposal_mass_to_canonical_bytes(&self, candidate_canonical_bytes: &[u8]) -> u64 { + self.transitions + .iter() + .find(|proposal| proposal.candidate_canonical_bytes == candidate_canonical_bytes) + .map(|proposal| proposal.raw_action_count) + .unwrap_or(0) + } + + fn sample<'a>(&'a self, rng: &mut RandomGenerator) -> Option<&'a CanonicalProposal> { + if self.total_raw_actions == 0 { + return None; + } + let total_raw_actions = usize::try_from(self.total_raw_actions).ok()?; + let mut draw = rng.gen_range(total_raw_actions) as u64; + for proposal in &self.transitions { + if draw < proposal.raw_action_count { + return Some(proposal); + } + draw -= proposal.raw_action_count; + } + None + } +} + +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +struct CandidateCacheKey { + candidate_canonical_bytes: Vec, + evaluator_profile_bytes: Vec, + dataset_identity: String, +} + +impl CandidateCacheKey { + fn digest_crc32(&self) -> String { + let mut payload = Vec::with_capacity( + self.candidate_canonical_bytes.len() + + self.evaluator_profile_bytes.len() + + self.dataset_identity.len(), + ); + payload.extend_from_slice(&self.candidate_canonical_bytes); + payload.extend_from_slice(&self.evaluator_profile_bytes); + payload.extend_from_slice(self.dataset_identity.as_bytes()); + crc32_hex(&payload) + } +} + +#[derive(Clone, Debug, Default)] +struct VerifiedTheoremInputs { + finite_planner_state: Option, + no_hidden_state: Option, + exact_reward_encoding: Option, + exact_state_observation: Option, + determinism_deadline: Option, + deterministic_table: Option, +} + +#[derive(Clone, Debug)] +struct VerifiedCertificate { + ref_value: String, + content_hash: String, +} + +#[derive(Clone, Debug)] +struct VerifiedExactRewardEncodingCertificate { + base: VerifiedCertificate, + max_reward: Reward, + reward_bits: usize, + scalar_representation: String, + mode: VerifiedRewardEncodingMode, +} + +#[derive(Clone, Debug)] +enum VerifiedRewardEncodingMode { + IntegerObjectiveDifferenceInterval, + FiniteRewardMap { map: VerifiedFiniteRewardMap }, +} + +#[derive(Clone, Debug)] +struct VerifiedFiniteRewardMap { + objective_difference_to_symbol: BTreeMap, + complete_nonnegative_interval_max: Option, +} + +#[derive(Clone, Debug)] +struct VerifiedExactStateObservationCertificate { + base: VerifiedCertificate, + observation_key_mode: String, + exact_state_encoder_spec_ref: String, + observation_adapter_spec_ref: String, + observation_adapter_content_hash: String, + finite_planner_state_certificate_hash: String, + finite_state_count: usize, +} + +#[derive(Clone, Debug)] +struct VerifiedDeterministicEvaluatorTable { + base: VerifiedCertificate, + rows: HashMap, +} + +#[derive(Clone, Debug)] +struct DeterministicEvaluatorRow { + status: CandidateEvalStatus, + compressed_bytes: usize, + target_loss_bits: f64, + elapsed_seconds: f64, + peak_memory_bytes: u64, +} + +#[derive(Clone, Debug)] +enum LoweredCausalEvent { + Reset, + Context { + channel: String, + bytes: Vec, + }, + ObserveTargetNoScore { + channel: String, + domain: String, + bytes: Vec, + }, + Target { + channel: String, + domain: String, + bytes: Vec, + weight: f64, + }, +} + +#[derive(Clone, Debug)] +enum PlannerMutationAction { + NumericStep { + path: String, + pointer: String, + kind: NumericKind, + delta: f64, + range: Option<(f64, f64)>, + }, + Noop, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum PlannerRewardSemantics { + ExactObjectiveDifference, + NormalizedClippedImprovement, +} + +impl PlannerRewardSemantics { + fn name(self) -> &'static str { + match self { + PlannerRewardSemantics::ExactObjectiveDifference => "exact_objective_difference", + PlannerRewardSemantics::NormalizedClippedImprovement => { + "normalized_clipped_improvement" + } + } + } +} + +#[derive(Clone, Debug)] +struct WarmstartTeacherDataset { + asset_id: String, + resolved_path: String, + content_hash: String, + records: usize, + traces: WarmStartExactJhTeacherDataset, +} + +#[derive(Clone, Debug)] +struct PlannerControllerContract { + interface: crate::spec::TunePlannerInterfaceSpec, + planner_simulations_per_step: usize, + return_horizon: Option, + label_phase_period: Option, + discount_factor: f64, + reward_semantics: PlannerRewardSemantics, + clipping_interval: Option<(f64, f64)>, + teacher: Option, + warmstart_self_improvement: bool, +} + +#[derive(Clone, Debug)] +struct PlannerEncodedPercept { + observations: Vec, + reward: Reward, +} + +impl PlannerEncodedPercept { + fn observation_slice(&self) -> &[PerceptVal] { + &self.observations + } +} + +impl PlannerControllerContract { + fn reward_encoder( + &self, + dataset: &LoadedDataset, + baseline_eval: &CandidateEvalResult, + verified_theorem: &VerifiedTheoremInputs, + ) -> Result { + match self.reward_semantics { + PlannerRewardSemantics::ExactObjectiveDifference => { + TunerRewardEncoder::exact_integer_objective_difference( + self.interface.reward_bits, + dataset, + baseline_eval, + verified_theorem.exact_reward_encoding.as_ref(), + ) + } + PlannerRewardSemantics::NormalizedClippedImprovement => { + let (min_improvement, max_improvement) = + self.clipping_interval.ok_or_else(|| { + "normalized clipped reward contract was not initialized".to_string() + })?; + TunerRewardEncoder::normalized_clipped( + self.interface.reward_bits, + min_improvement, + max_improvement, + ) + } + } + } +} + +#[derive(Clone, Debug)] +enum TunerRewardEncoder { + ExactIntegerObjectiveDifference { + max_reward: Reward, + objective_difference_to_symbol: Option>, + }, + NormalizedClipped { + min_improvement: f64, + max_improvement: f64, + max_reward: Reward, + }, +} + +impl TunerRewardEncoder { + fn exact_integer_objective_difference( + reward_bits: usize, + dataset: &LoadedDataset, + baseline_eval: &CandidateEvalResult, + certificate: Option<&VerifiedExactRewardEncodingCertificate>, + ) -> Result { + let certificate = certificate.ok_or_else(|| { + "reward_encoding_unsafe: exact objective-difference planner controllers require a verified exact_reward_encoding_certificate" + .to_string() + })?; + if certificate.reward_bits != reward_bits { + return Err(format!( + "reward_encoding_unsafe: certificate reward_bits={} does not match controller reward_bits={reward_bits}", + certificate.reward_bits + )); + } + let baseline_objective = + exact_nonnegative_i64_from_f64(baseline_eval.objective_bits, "baseline objective")?; + let max_encoded = max_nonnegative_reward_for_bits(reward_bits)?; + if baseline_objective > max_encoded { + return Err(format!( + "reward_encoding_unsafe: reward_bits={reward_bits} cannot injectively encode reachable exact objective differences up to baseline objective {baseline_objective} (max encoded {max_encoded})" + )); + } + if baseline_objective > certificate.max_reward { + return Err(format!( + "reward_encoding_unsafe: baseline objective {baseline_objective} exceeds verified exact reward maximum {}", + certificate.max_reward + )); + } + let objective_difference_to_symbol = match &certificate.mode { + VerifiedRewardEncodingMode::IntegerObjectiveDifferenceInterval => None, + VerifiedRewardEncodingMode::FiniteRewardMap { map } => { + let objective_difference_to_symbol = &map.objective_difference_to_symbol; + if !objective_difference_to_symbol.contains_key(&0) { + return Err( + "reward_encoding_unsafe: finite reward map must encode zero improvement" + .to_string(), + ); + } + if !objective_difference_to_symbol.contains_key(&baseline_objective) { + return Err(format!( + "reward_encoding_unsafe: finite reward map must include baseline objective difference {baseline_objective}" + )); + } + let complete_max = map.complete_nonnegative_interval_max.ok_or_else(|| { + "reward_encoding_unsafe: finite reward map certificates used by exact controllers must declare complete_nonnegative_interval_max" + .to_string() + })?; + if complete_max < baseline_objective { + return Err(format!( + "reward_encoding_unsafe: finite reward map complete_nonnegative_interval_max {complete_max} is below baseline objective difference {baseline_objective}" + )); + } + validate_complete_finite_reward_interval( + objective_difference_to_symbol, + complete_max, + )?; + Some(objective_difference_to_symbol.clone()) + } + }; + if dataset.kind != DatasetKind::PassiveBytes + && certificate.scalar_representation == SCALAR_REPRESENTATION_DECLARATION + { + return Err( + "reward_encoding_unsafe: non-passive exact objective-difference runs require a task-specific finite scalar certificate, not the passive default declaration" + .to_string(), + ); + } + Ok(Self::ExactIntegerObjectiveDifference { + max_reward: certificate.max_reward.min(baseline_objective), + objective_difference_to_symbol, + }) + } + + fn normalized_clipped( + reward_bits: usize, + min_improvement: f64, + max_improvement: f64, + ) -> Result { + if !min_improvement.is_finite() || !max_improvement.is_finite() { + return Err("normalized clipped reward bounds must be finite".to_string()); + } + if max_improvement <= min_improvement { + return Err( + "normalized clipped reward contract requires max_improvement > min_improvement" + .to_string(), + ); + } + Ok(Self::NormalizedClipped { + min_improvement, + max_improvement, + max_reward: max_nonnegative_reward_for_bits(reward_bits)?, + }) + } + + fn max_reward(&self) -> Reward { + match self { + Self::ExactIntegerObjectiveDifference { max_reward, .. } + | Self::NormalizedClipped { max_reward, .. } => *max_reward, + } + } + + fn encode(&self, raw_improvement: f64) -> Result { + if !raw_improvement.is_finite() { + return Err("planner reward improvement must be finite".to_string()); + } + match self { + Self::ExactIntegerObjectiveDifference { + max_reward, + objective_difference_to_symbol, + } => { + if raw_improvement < 0.0 { + return Err("exact objective-difference reward cannot be negative".to_string()); + } + let reward = + exact_nonnegative_i64_from_f64(raw_improvement, "planner reward improvement")?; + if let Some(map) = objective_difference_to_symbol { + return map.get(&reward).copied().ok_or_else(|| { + format!( + "reward_encoding_unsafe: exact objective difference {reward} is missing from verified finite reward map" + ) + }); + } + if reward > *max_reward { + return Err(format!( + "reward_encoding_unsafe: exact reward {reward} exceeds certified maximum {max_reward}" + )); + } + Ok(reward) + } + Self::NormalizedClipped { + min_improvement, + max_improvement, + max_reward, + } => { + let normalized = normalized_clipped_improvement( + raw_improvement, + *min_improvement, + *max_improvement, + )?; + Ok((normalized * *max_reward as f64).round() as Reward) + } + } + } +} + +enum TunerPlannerAgentRuntime { + McAixi { + agent: Agent, + prev_action: Action, + prev_percept: PlannerEncodedPercept, + }, + AiqiDiscounted { + agent: AiqiAgent, + }, + WarmStartExactJh { + agent: WarmStartExactJhAgent, + }, +} + +impl TunerPlannerAgentRuntime { + fn select_action(&mut self) -> Action { + match self { + Self::McAixi { + agent, + prev_action, + prev_percept, + } => { + agent.model_update_percept_stream( + prev_percept.observation_slice(), + prev_percept.reward, + ); + let action = agent.get_planned_action( + prev_percept.observation_slice(), + prev_percept.reward, + *prev_action, + ); + agent.model_update_action_external(action); + *prev_action = action; + action + } + Self::AiqiDiscounted { agent } => agent.get_planned_action(), + Self::WarmStartExactJh { agent } => agent.get_planned_action(), + } + } + + fn observe_transition( + &mut self, + action: Action, + percept: PlannerEncodedPercept, + ) -> Result<(), String> { + match self { + Self::McAixi { prev_percept, .. } => { + *prev_percept = percept; + Ok(()) + } + Self::AiqiDiscounted { agent } => agent + .observe_transition(action, percept.observation_slice(), percept.reward) + .map_err(|err| err.to_string()), + Self::WarmStartExactJh { agent } => agent + .observe_transition(action, percept.observation_slice(), percept.reward) + .map_err(|err| err.to_string()), + } + } + + fn same_task_live_trace(&self) -> Option { + match self { + Self::WarmStartExactJh { agent } => agent.same_task_live_trace(), + Self::McAixi { .. } | Self::AiqiDiscounted { .. } => None, + } + } + + fn rebuild_warmstart_agent( + &mut self, + planner_run: &CompiledPlannerRunSpec, + teacher: WarmStartExactJhTeacherDataset, + ) -> Result<(), String> { + match self { + Self::WarmStartExactJh { agent } => { + *agent = WarmStartExactJhAgent::from_compiled_planner_run(planner_run, teacher) + .map_err(|err| err.to_string())?; + Ok(()) + } + Self::McAixi { .. } | Self::AiqiDiscounted { .. } => Err( + "trace refresh is only defined for warm-start exact-J_H controllers".to_string(), + ), + } + } +} + +/// Execute tuning for one canonical tune document using executor-side controls. +/// +/// The runtime enforces canonical/executor separation, baseline deployability +/// preconditions, and controller-specific bounded search semantics. +pub fn run_tune(request: &TuneCommandRequest) -> Result<(), String> { + let prepared = prepare_tune_context(request)?; + if let Some(path) = request.emit_exact_reward_encoding_certificate.as_deref() { + emit_exact_reward_encoding_certificate(request, path, &prepared)?; + return Ok(()); + } + let PreparedTuneContext { + compiled, + dataset, + runtime_profile, + evaluator_profile, + initial_eval_limit, + tune_started, + } = prepared; + let verified_theorem = VerifiedTheoremInputs::load( + &request.execution.theorem, + &compiled, + &dataset, + &evaluator_profile, + compiled.base_dir(), + )?; + apply_executor_controls(&request.execution)?; + + for _ in 0..request.execution.warmup_baseline_runs { + match evaluate_candidate( + compiled.baseline_candidate(), + &dataset, + compiled.baseline_candidate_model_bytes(), + compiled.canonical_spec().min_throughput_bytes_per_second, + compiled.canonical_spec().max_memory_bytes, + initial_eval_limit, + request.execution.evaluator_threads(), + &runtime_profile, + verified_theorem.deterministic_table.as_ref(), + ) { + Ok(_) => {} + Err(CandidateEvalFailure::FatalEvaluatorFailure { diagnostic }) => { + return Err(format!( + "unrecoverable evaluator failure during baseline warmup: {diagnostic}" + )); + } + } + } + + let baseline_eval = match evaluate_candidate( + compiled.baseline_candidate(), + &dataset, + compiled.baseline_candidate_model_bytes(), + compiled.canonical_spec().min_throughput_bytes_per_second, + compiled.canonical_spec().max_memory_bytes, + initial_eval_limit, + request.execution.evaluator_threads(), + &runtime_profile, + verified_theorem.deterministic_table.as_ref(), + ) { + Ok(value) => value, + Err(CandidateEvalFailure::FatalEvaluatorFailure { diagnostic }) => { + return Err(format!( + "unrecoverable evaluator failure during baseline evaluation: {diagnostic}" + )); + } + }; + let baseline_key = cache_key_for_candidate( + compiled.baseline_candidate().canonical_bytes().as_slice(), + &evaluator_profile, + &dataset.canonical_content_hash, + )?; + let baseline_hash = crc32_hex(compiled.baseline_candidate().canonical_bytes().as_slice()); + let baseline_bytes = compiled + .baseline_candidate() + .canonical_bytes() + .as_slice() + .to_vec(); + let mut cache = HashMap::::new(); + cache.insert(baseline_key.clone(), baseline_eval.clone()); + let doc_hash = crc32_hex(compiled.canonical_bytes().as_slice()); + let evaluator_profile_hash = evaluator_profile.hash()?; + let search_summary = if baseline_eval.deployable { + run_controller_search( + &compiled, + request, + &dataset, + &evaluator_profile, + &verified_theorem, + tune_started, + baseline_eval.clone(), + baseline_hash.clone(), + baseline_bytes.clone(), + baseline_key.clone(), + &runtime_profile, + &mut cache, + )? + } else { + SearchSummary { + status: "baseline_not_deployable", + warning: None, + fatal_evaluator_failure: None, + fatal_evaluator_failures: 0, + best_candidate: compiled.canonical_spec().baseline_candidate.clone(), + best_candidate_crc32: baseline_hash.clone(), + best_eval: baseline_eval.clone(), + cache_key_digest: baseline_key.digest_crc32(), + cache_hits: 0, + cache_misses: 1, + candidate_evaluations_executed: 1, + non_warmup_candidate_results_seen: 1, + post_baseline_candidate_results_seen: 0, + proposals_attempted: 0, + proposals_invalid: 0, + self_loop_proposals: 0, + invalid_reason_counts: InvalidReasonCounts::default(), + successful_non_deployable: if baseline_eval.status == CandidateEvalStatus::Success + && !baseline_eval.deployable + { + 1 + } else { + 0 + }, + candidate_result_counts: { + let mut counts = CandidateResultCounts::default(); + counts.record_admitted_result(&baseline_eval); + counts + }, + final_best_move_reward: 0.0, + realized_trace_counts_by_round: None, + trace_refresh_merges_by_round: None, + controller_report: serde_json::json!({ + "kind": controller_kind_name(compiled.controller()), + "runtime_path": "baseline_precondition_failed", + "baseline_deployable_precondition": false, + }), + } + }; + + let output_path = compiled.canonical_spec().output_config_path.as_str(); + let output_written = if search_summary.best_eval.deployable { + let output_json = search_summary + .best_candidate + .to_canonical_json() + .map_err(|err| format!("failed to serialize output candidate: {err}"))?; + fs::write(output_path, output_json).map_err(|err| { + format!( + "failed to write output_config_path '{}': {err}", + output_path + ) + })?; + true + } else { + false + }; + let theorem_claims = theorem_claims_report( + &request.execution.theorem, + &verified_theorem, + compiled.controller(), + &dataset, + &search_summary, + runtime_profile.strict_theorem_memory_certified(), + ); + let bounds_hash = bounds_hash(&compiled.canonical_spec().bounds)?; + let observation_adapter_hash = observation_adapter_content_hash()?; + let evaluator_execution_model = + evaluator_execution_model(verified_theorem.deterministic_table.as_ref()); + let theorem_timing_basis = theorem_timing_basis(&request.execution.theorem, &verified_theorem); + let best_model_bytes = search_summary + .best_candidate + .compile_in(&SpecEnvironment::new(compiled.base_dir())) + .map_err(|err| format!("failed to compile best candidate for reporting: {err}"))? + .canonical_bytes() + .len(); + let throughput_runtime_cap_seconds = + dataset.dataset_units / compiled.canonical_spec().min_throughput_bytes_per_second; + let warmstart_self_improvement_enabled = matches!( + compiled.controller(), + crate::spec::CompiledTuneController::AiqiWarmstartExactJh(_) + ); + let same_task_trace_refresh_enabled = + warmstart_self_improvement_enabled && request.execution.warmstart_trace_refresh; + let effective_self_improvement_rounds = if warmstart_self_improvement_enabled { + request.execution.self_improvement_rounds.max(1) + } else { + 1 + }; + let self_improvement_round_deadlines_seconds = + if warmstart_self_improvement_enabled && effective_self_improvement_rounds > 1 { + Some( + (1..=effective_self_improvement_rounds) + .map(|round| { + ((round as f64) / (effective_self_improvement_rounds as f64)) + * compiled.canonical_spec().time_budget_seconds + }) + .collect::>(), + ) + } else { + None + }; + + let report = serde_json::json!({ + "schema_version": 1, + "kind": "tune_report", + "status": search_summary.status, + "warning": search_summary.warning, + "crate_version": env!("CARGO_PKG_VERSION"), + "feature_set": compiled_feature_set(), + "spec_path": request.spec_path, + "spec_crc32": doc_hash, + "seed": compiled.canonical_spec().seed, + "timing_certification_tier": timing_tier_name(request.execution.theorem.timing_certification_tier), + "determinism_deadline_certificate": request.execution.theorem.determinism_deadline_certificate, + "evaluator_execution_model": evaluator_execution_model, + "theorem_timing_basis": theorem_timing_basis, + "theorem_claims": theorem_claims, + "execution_profile": request.execution.to_json_value(), + "evaluator_profile": evaluator_profile.to_json_value(), + "evaluator_profile_crc32": evaluator_profile_hash, + "provenance": { + "bounds_crc32": bounds_hash, + "candidate_canonicalization_version": compiled.candidate_canonicalization_version(), + "observation_adapter_spec_ref": request.execution.theorem.observation_adapter_spec_ref.as_deref().unwrap_or(OBSERVATION_ADAPTER_DECLARATION), + "observation_adapter_content_crc32": observation_adapter_hash, + "exact_state_encoder_spec_ref": request.execution.theorem.exact_state_encoder_spec_ref, + "exact_state_observation_certified": verified_theorem.exact_state_observation.is_some(), + "scalar_representation_ref": request.execution.theorem.scalar_representation_ref.as_deref().unwrap_or(SCALAR_REPRESENTATION_DECLARATION), + "evaluator_execution_model": evaluator_execution_model, + "theorem_timing_basis": theorem_timing_basis, + "verified_theorem_inputs": verified_theorem.to_json_value(), + "resolved_evaluator_runtime_profile": runtime_profile.to_provenance_value(), + "canonical_code_certification": { + "basis": "structural_self_delimiting_binary_encoding_plus_tests", + "mechanized": false, + "sample_corpus_checked": true, + "trailing_bytes_rejected": true, + "top_level_length_prefix": true, + }, + "warmup_policy": { + "warmup_baseline_runs": request.execution.warmup_baseline_runs, + "excluded_from_cache": true, + "excluded_from_optimization_metrics": true, + }, + "self_improvement_policy": { + "rounds": effective_self_improvement_rounds, + "same_task_trace_refresh_enabled": same_task_trace_refresh_enabled, + "online_delayed_label_update_enabled": warmstart_self_improvement_enabled && !same_task_trace_refresh_enabled, + "deterministic_round_deadlines_seconds": self_improvement_round_deadlines_seconds, + "realized_trace_counts_by_round": search_summary.realized_trace_counts_by_round.clone(), + "trace_refresh_merges_by_round": search_summary.trace_refresh_merges_by_round.clone(), + }, + "stagnation_policy": { + "stagnation_reset_evals": request.execution.stagnation_reset_evals, + }, + "executor_controls": executor_controls_report( + &request.execution, + &runtime_profile, + ), + "diagnostic_chunking": diagnostic_chunking_report( + &dataset, + request.execution.diagnostic_chunk_bytes, + ), + }, + "cache": { + "key_candidate_crc32": search_summary.best_candidate_crc32, + "key_digest_crc32": search_summary.cache_key_digest, + "dataset_content_crc32": dataset.canonical_content_hash, + "warmup_runs_excluded_from_cache": true, + "actual_evaluator_calls_excluding_warmups": search_summary.candidate_evaluations_executed, + "candidate_evaluations_executed": search_summary.candidate_evaluations_executed, + "cache_hits": search_summary.cache_hits, + "cache_misses": search_summary.cache_misses, + }, + "search": { + "termination_reason": search_summary.status, + "fatal_evaluator_failures": search_summary.fatal_evaluator_failures, + "fatal_evaluator_failure": search_summary.fatal_evaluator_failure.clone(), + "proposals_attempted": search_summary.proposals_attempted, + "proposals_invalid": search_summary.proposals_invalid, + "self_loop_proposals": search_summary.self_loop_proposals, + "invalid_reason_counts": { + "candidate_external_asset_forbidden": search_summary.invalid_reason_counts.candidate_external_asset_forbidden, + "candidate_out_of_bounds": search_summary.invalid_reason_counts.candidate_out_of_bounds, + "candidate_compile_error": search_summary.invalid_reason_counts.candidate_compile_error, + "invalid_action_index": search_summary.invalid_reason_counts.invalid_action_index, + "inapplicable_action": search_summary.invalid_reason_counts.inapplicable_action, + }, + "successful_non_deployable": search_summary.successful_non_deployable, + "candidate_result_counts": { + "success_deployable": search_summary.candidate_result_counts.success_deployable, + "success_non_deployable": search_summary.candidate_result_counts.success_non_deployable, + "timeout": search_summary.candidate_result_counts.timeout, + "invalid": search_summary.candidate_result_counts.invalid, + "error_recoverable": search_summary.candidate_result_counts.error_recoverable, + }, + "self_improvement_rounds": effective_self_improvement_rounds, + "max_evaluations": request.execution.max_evaluations, + "max_evaluations_semantics": "baseline_included_warmups_excluded_cache_hits_included_for_search_steps", + "baseline_counts_toward_max_evaluations": true, + "non_warmup_candidate_results_seen": search_summary.non_warmup_candidate_results_seen, + "post_baseline_candidate_results_seen": search_summary.post_baseline_candidate_results_seen, + "time_budget_seconds": compiled.canonical_spec().time_budget_seconds, + "final_best_move_reward": search_summary.final_best_move_reward, + "controller": search_summary.controller_report, + "planner_deployability": planner_deployability_report( + request.execution.planner_deployable_model, + best_model_bytes, + search_summary.best_eval.elapsed_seconds, + search_summary.best_eval.deployable, + ), + }, + "input_asset": { + "id": compiled.canonical_spec().input_asset, + "resolved_path": dataset.resolved_path, + "content_crc32": dataset.canonical_content_hash, + "lowered_skeleton_crc32": dataset.lowered_skeleton_hash, + "target_domain_support_crc32": dataset.target_domain_support_hash, + "causal_header_profile_crc32": dataset.causal_header_profile_hash, + "causal_profile": causal_profile_report(&dataset), + "dataset_kind": dataset_kind_name(dataset.kind), + "source_size_bytes": dataset.source_size_bytes, + "charged_target_bytes": dataset.raw_bytes.len(), + "dataset_units": dataset.dataset_units, + "target_events": dataset.target_events, + "target_size_function": dataset.target_size_function, + "diagnostic_chunking": diagnostic_chunking_report( + &dataset, + request.execution.diagnostic_chunk_bytes, + ), + }, + "baseline": { + "candidate_crc32": baseline_hash, + "status": baseline_eval.status.name(), + "model_bytes": compiled.baseline_candidate_model_bytes(), + "compressed_bytes": baseline_eval.compressed_bytes, + "physical_compressed_bytes_diagnostic_only": true, + "target_loss_bits": baseline_eval.target_loss_bits, + "objective_bits": baseline_eval.objective_bits, + "elapsed_seconds": baseline_eval.elapsed_seconds, + "throughput_bytes_per_second": baseline_eval.throughput_bytes_per_second, + "peak_memory_bytes": baseline_eval.peak_memory_bytes, + "min_throughput_bytes_per_second": compiled.canonical_spec().min_throughput_bytes_per_second, + "throughput_runtime_cap_seconds": throughput_runtime_cap_seconds, + "max_memory_bytes": compiled.canonical_spec().max_memory_bytes, + "effective_eval_time_limit_seconds": baseline_eval.effective_eval_time_limit_seconds, + "deployable": baseline_eval.deployable, + }, + "best": { + "candidate_crc32": search_summary.best_candidate_crc32, + "status": search_summary.best_eval.status.name(), + "model_bytes": best_model_bytes, + "compressed_bytes": search_summary.best_eval.compressed_bytes, + "physical_compressed_bytes_diagnostic_only": true, + "target_loss_bits": search_summary.best_eval.target_loss_bits, + "objective_bits": search_summary.best_eval.objective_bits, + "elapsed_seconds": search_summary.best_eval.elapsed_seconds, + "throughput_bytes_per_second": search_summary.best_eval.throughput_bytes_per_second, + "peak_memory_bytes": search_summary.best_eval.peak_memory_bytes, + "effective_eval_time_limit_seconds": search_summary.best_eval.effective_eval_time_limit_seconds, + "deployable": search_summary.best_eval.deployable, + }, + "output": { + "output_config_path": output_path, + "output_written": output_written, + "output_candidate_crc32": if output_written { + Some(search_summary.best_candidate_crc32.clone()) + } else { + None:: + }, + }, + }); + + if let Some(path) = compiled.canonical_spec().report_path.as_deref() { + let text = serde_json::to_string_pretty(&report) + .map_err(|err| format!("failed to serialize report: {err}"))?; + fs::write(path, text) + .map_err(|err| format!("failed to write report_path '{}': {err}", path))?; + } + log_executor_event( + &request.execution, + "finish", + serde_json::json!({ + "status": search_summary.status, + "output_written": output_written, + "best_eval_status": search_summary.best_eval.status.name(), + }), + )?; + + if let Some(diagnostic) = search_summary.fatal_evaluator_failure.as_deref() { + Err(format!( + "tuning terminated due to unrecoverable evaluator failure: {diagnostic}" + )) + } else if search_summary.best_eval.deployable { + Ok(()) + } else if search_summary.best_eval.status == CandidateEvalStatus::Timeout { + Err(format!( + "baseline candidate timed out under effective evaluator limit {:.6} seconds", + initial_eval_limit + )) + } else { + Err(format!( + "baseline candidate is not deployable under tune constraints: throughput {:.3} B/s (required >= {:.3}), peak memory {} bytes (required <= {})", + search_summary.best_eval.throughput_bytes_per_second, + compiled.canonical_spec().min_throughput_bytes_per_second, + search_summary.best_eval.peak_memory_bytes, + compiled.canonical_spec().max_memory_bytes + )) + } +} + +fn resolve_input_asset_path<'a>( + compiled: &'a crate::spec::CompiledTuneSpec, + input_asset_id: &str, +) -> Result<&'a Path, String> { + let binding = compiled + .resolved_assets() + .iter() + .find(|entry| entry.id == input_asset_id) + .ok_or_else(|| format!("unknown input_asset id '{}'", input_asset_id))?; + let AssetRef::Filesystem(path) = &binding.asset; + Ok(path.as_path()) +} + +fn effective_eval_limit_seconds( + compiled: &crate::spec::CompiledTuneSpec, + tune_started: Instant, + full_limit_seconds: Option, + round_deadline_seconds: Option, +) -> f64 { + let elapsed = tune_started.elapsed().as_secs_f64(); + let remaining_total = (compiled.canonical_spec().time_budget_seconds - elapsed).max(0.0); + let remaining_round = round_deadline_seconds + .map(|deadline| (deadline - elapsed).max(0.0)) + .unwrap_or(f64::INFINITY); + full_limit_seconds + .unwrap_or(compiled.canonical_spec().eval_time_limit_seconds) + .min(remaining_total) + .min(remaining_round) + .max(0.0) +} + +fn apply_executor_controls(config: &TuneExecutionConfig) -> Result<(), String> { + if let Some(cpu_affinity) = config.cpu_affinity.as_deref() { + apply_cpu_affinity(cpu_affinity)?; + } + log_executor_event(config, "start", serde_json::json!({}))?; + Ok(()) +} + +#[cfg(target_os = "linux")] +fn apply_cpu_affinity(raw: &str) -> Result<(), String> { + // SAFETY: `cpu_set_t` is a plain C bitset type for the Linux affinity API. + // Zero-initialization is the documented starting state before `CPU_ZERO`. + let mut set = unsafe { std::mem::zeroed::() }; + // SAFETY: `set` is a valid, writable `cpu_set_t` local variable. + unsafe { + libc::CPU_ZERO(&mut set); + } + let mut count = 0usize; + for part in raw.split(',') { + let cpu = part + .trim() + .parse::() + .map_err(|_| format!("invalid cpu_affinity entry '{}'", part.trim()))?; + // SAFETY: `set` is valid and writable for the duration of the call; CPU_SET + // only mutates the bitset. Kernel validation of out-of-range CPU indices is + // handled by the subsequent `sched_setaffinity` call. + unsafe { + libc::CPU_SET(cpu, &mut set); + } + count = count.saturating_add(1); + } + if count == 0 { + return Err("cpu_affinity must name at least one CPU".to_string()); + } + // SAFETY: The pointer references a live `cpu_set_t`, the size matches that type, + // and pid 0 intentionally targets the current process per sched_setaffinity(2). + let result = + unsafe { libc::sched_setaffinity(0, std::mem::size_of::(), &set) }; + if result == 0 { + Ok(()) + } else { + Err(format!( + "failed to apply cpu_affinity '{}': {}", + raw, + std::io::Error::last_os_error() + )) + } +} + +#[cfg(not(target_os = "linux"))] +fn apply_cpu_affinity(raw: &str) -> Result<(), String> { + Err(format!( + "cpu_affinity '{}' is only supported on Linux by this tuner runtime", + raw + )) +} + +fn log_executor_event( + config: &TuneExecutionConfig, + event: &str, + payload: Value, +) -> Result<(), String> { + let Some(path) = config.log_path.as_deref() else { + return Ok(()); + }; + let mut file = fs::OpenOptions::new() + .create(true) + .append(true) + .open(path) + .map_err(|err| format!("failed to open log_path '{}': {err}", path))?; + let line = serde_json::to_string(&serde_json::json!({ + "kind": "tune_executor_event", + "event": event, + "payload": payload, + })) + .map_err(|err| format!("failed to serialize executor log event: {err}"))?; + writeln!(file, "{line}").map_err(|err| format!("failed to write log_path '{}': {err}", path)) +} + +fn planner_action_count(compiled: &crate::spec::CompiledTuneSpec) -> Result { + match compiled.controller() { + crate::spec::CompiledTuneController::AnnealedHillClimbing(_) => Ok(0), + crate::spec::CompiledTuneController::McAixiFacCtw(inner) => { + Ok(inner.interface.agent_actions.get()) + } + crate::spec::CompiledTuneController::AiqiDiscounted(inner) => { + Ok(inner.interface.agent_actions.get()) + } + crate::spec::CompiledTuneController::AiqiWarmstartExactJh(inner) => { + Ok(inner.interface.agent_actions.get()) + } + } +} + +#[allow(clippy::too_many_arguments)] +fn run_controller_search( + compiled: &crate::spec::CompiledTuneSpec, + request: &TuneCommandRequest, + dataset: &LoadedDataset, + evaluator_profile: &EvaluatorProfile, + verified_theorem: &VerifiedTheoremInputs, + tune_started: Instant, + baseline_eval: CandidateEvalResult, + baseline_hash: String, + baseline_bytes: Vec, + baseline_key: CandidateCacheKey, + runtime_profile: &ResolvedEvaluatorRuntimeProfile, + cache: &mut HashMap, +) -> Result { + match compiled.controller() { + crate::spec::CompiledTuneController::AnnealedHillClimbing(inner) => { + run_annealed_hill_climbing( + compiled, + request, + dataset, + evaluator_profile, + verified_theorem, + tune_started, + baseline_eval, + baseline_hash, + baseline_bytes, + baseline_key, + runtime_profile, + cache, + inner.max_mutation_radius, + ) + } + crate::spec::CompiledTuneController::McAixiFacCtw(_) + | crate::spec::CompiledTuneController::AiqiDiscounted(_) + | crate::spec::CompiledTuneController::AiqiWarmstartExactJh(_) => { + run_planner_family_controller( + compiled, + request, + dataset, + evaluator_profile, + verified_theorem, + tune_started, + baseline_eval, + baseline_hash, + baseline_bytes, + baseline_key, + runtime_profile, + cache, + ) + } + } +} + +#[allow(clippy::too_many_arguments)] +fn run_annealed_hill_climbing( + compiled: &crate::spec::CompiledTuneSpec, + request: &TuneCommandRequest, + dataset: &LoadedDataset, + evaluator_profile: &EvaluatorProfile, + verified_theorem: &VerifiedTheoremInputs, + tune_started: Instant, + baseline_eval: CandidateEvalResult, + baseline_hash: String, + baseline_bytes: Vec, + baseline_key: CandidateCacheKey, + runtime_profile: &ResolvedEvaluatorRuntimeProfile, + cache: &mut HashMap, + max_mutation_radius: usize, +) -> Result { + let env = SpecEnvironment::new(compiled.base_dir()); + let mut rng = RandomGenerator::from_seed(compiled.canonical_spec().seed); + + let mut best_candidate = compiled.canonical_spec().baseline_candidate.clone(); + let mut best_eval = baseline_eval.clone(); + let mut best_hash = baseline_hash; + let mut best_bytes = baseline_bytes; + let mut best_key = baseline_key; + + let mut current_candidate = best_candidate.clone(); + let mut current_eval = baseline_eval; + let mut candidate_result_counts = CandidateResultCounts::default(); + candidate_result_counts.record_admitted_result(¤t_eval); + let mut cache_hits: usize = 0; + let mut cache_misses: usize = 1; + let mut candidate_evaluations_executed: usize = 1; + let mut proposals_attempted: usize = 0; + let mut proposals_invalid: usize = 0; + let mut self_loop_proposals: usize = 0; + let mut invalid_reason_counts = InvalidReasonCounts::default(); + let mut successful_non_deployable: usize = 0; + let mut final_best_move_reward: f64 = 0.0; + let mut evaluations_seen: usize = 1; + let max_evaluations = request.execution.max_evaluations.unwrap_or(usize::MAX); + let mut fatal_evaluator_failure: Option = None; + + let mut stagnation_counter: usize = 0; + 'search: loop { + if evaluations_seen >= max_evaluations { + break; + } + if tune_started.elapsed().as_secs_f64() >= compiled.canonical_spec().time_budget_seconds { + break; + } + proposals_attempted = proposals_attempted.saturating_add(1); + + let progress = + annealer_progress(tune_started, compiled.canonical_spec().time_budget_seconds); + let temperature = annealer_temperature(progress); + let active_radius = annealer_active_radius(max_mutation_radius, temperature); + let proposal = match sample_annealed_proposal( + ¤t_candidate, + &compiled.canonical_spec().bounds, + max_mutation_radius, + active_radius, + &env, + &mut rng, + )? { + AnnealedProposalDraw::Proposal(proposal) => proposal, + AnnealedProposalDraw::SelfLoop => { + self_loop_proposals = self_loop_proposals.saturating_add(1); + continue; + } + AnnealedProposalDraw::Exhausted => { + self_loop_proposals = self_loop_proposals.saturating_add(1); + break; + } + }; + let proposed_candidate = proposal.candidate.clone(); + + if let Err(err) = reject_candidate_local_external_artifacts(&proposed_candidate) { + proposals_invalid = proposals_invalid.saturating_add(1); + invalid_reason_counts.record(err.reason); + continue; + } + if validate_candidate_against_tune_bounds( + &proposed_candidate, + &compiled.canonical_spec().bounds, + ) + .is_err() + { + proposals_invalid = proposals_invalid.saturating_add(1); + invalid_reason_counts.record(TuneInvalidReason::CandidateOutOfBounds); + continue; + } + + let compiled_candidate = match proposed_candidate.compile_in(&env) { + Ok(value) => value, + Err(_) => { + proposals_invalid = proposals_invalid.saturating_add(1); + invalid_reason_counts.record(TuneInvalidReason::CandidateCompileError); + continue; + } + }; + + let candidate_bytes = compiled_candidate.canonical_bytes().as_slice().to_vec(); + let candidate_hash = crc32_hex(&candidate_bytes); + let effective_limit = effective_eval_limit_seconds( + compiled, + tune_started, + Some(compiled.canonical_spec().eval_time_limit_seconds), + None, + ); + let candidate_profile = evaluator_profile.with_eval_time_limit(effective_limit); + let cache_key = cache_key_for_candidate( + compiled_candidate.canonical_bytes().as_slice(), + &candidate_profile, + &dataset.canonical_content_hash, + )?; + let candidate_eval = if let Some(cached) = cache.get(&cache_key) { + cache_hits = cache_hits.saturating_add(1); + cached.clone() + } else { + let evaluated = match evaluate_candidate( + &compiled_candidate, + dataset, + compiled_candidate.canonical_bytes().len(), + compiled.canonical_spec().min_throughput_bytes_per_second, + compiled.canonical_spec().max_memory_bytes, + effective_limit, + request.execution.evaluator_threads(), + runtime_profile, + verified_theorem.deterministic_table.as_ref(), + ) { + Ok(value) => value, + Err(CandidateEvalFailure::FatalEvaluatorFailure { diagnostic }) => { + fatal_evaluator_failure = Some(diagnostic); + break 'search; + } + }; + cache.insert(cache_key.clone(), evaluated.clone()); + cache_misses = cache_misses.saturating_add(1); + candidate_evaluations_executed = candidate_evaluations_executed.saturating_add(1); + evaluated + }; + evaluations_seen = evaluations_seen.saturating_add(1); + candidate_result_counts.record_admitted_result(&candidate_eval); + + if candidate_eval.status == CandidateEvalStatus::Success && !candidate_eval.deployable { + successful_non_deployable = successful_non_deployable.saturating_add(1); + } + + if !candidate_eval.deployable { + continue; + } + + let delta = candidate_eval.objective_bits - current_eval.objective_bits; + let accept_probability = annealer_acceptance_probability( + request.execution.annealer_kernel_profile, + delta, + temperature, + &proposal, + )?; + let accept = rng.gen_f64() < accept_probability; + + if accept { + current_candidate = proposed_candidate.clone(); + current_eval = candidate_eval.clone(); + } + + if key_less(&candidate_eval, &candidate_bytes, &best_eval, &best_bytes) { + final_best_move_reward = + (best_eval.objective_bits - candidate_eval.objective_bits).max(0.0); + best_candidate = proposed_candidate; + best_eval = candidate_eval; + best_hash = candidate_hash; + best_bytes = candidate_bytes; + best_key = cache_key; + stagnation_counter = 0; + } else { + stagnation_counter = stagnation_counter.saturating_add(1); + } + + if let Some(reset_after) = request.execution.stagnation_reset_evals + && stagnation_counter >= reset_after + { + current_candidate = best_candidate.clone(); + current_eval = best_eval.clone(); + stagnation_counter = 0; + } + } + + let status = if fatal_evaluator_failure.is_some() { + "terminated_unrecoverable_evaluator_failure" + } else { + "completed_annealed" + }; + let warning = fatal_evaluator_failure + .as_ref() + .map(|_| "terminated due to unrecoverable evaluator failure".to_string()); + + Ok(SearchSummary { + status, + warning, + fatal_evaluator_failure: fatal_evaluator_failure.clone(), + fatal_evaluator_failures: usize::from(fatal_evaluator_failure.is_some()), + best_candidate, + best_candidate_crc32: best_hash, + best_eval, + cache_key_digest: best_key.digest_crc32(), + cache_hits, + cache_misses, + candidate_evaluations_executed, + non_warmup_candidate_results_seen: evaluations_seen, + post_baseline_candidate_results_seen: evaluations_seen.saturating_sub(1), + proposals_attempted, + proposals_invalid, + self_loop_proposals, + invalid_reason_counts, + successful_non_deployable, + candidate_result_counts, + final_best_move_reward, + realized_trace_counts_by_round: None, + trace_refresh_merges_by_round: None, + controller_report: serde_json::json!({ + "kind": "annealed_hill_climbing", + "runtime_path": annealer_runtime_path_name(request.execution.annealer_kernel_profile), + "max_mutation_radius": max_mutation_radius, + "annealer_kernel_profile": annealer_kernel_profile_name(request.execution.annealer_kernel_profile), + "proposal_mass_accounting": request.execution.annealer_kernel_profile == AnnealerKernelProfile::CompiledUniformMetropolisHastings, + "proposal_action_distribution": "uniform_finite_bounded_numeric_elementary_descriptors", + }), + }) +} + +#[allow(clippy::too_many_arguments)] +fn validate_candidate_against_tune_bounds( + candidate: &crate::api::CompressionBackend, + bounds: &crate::spec::TuneBoundsSpec, +) -> Result<(), String> { + let topology = collect_candidate_topology(candidate)?; + validate_candidate_backend_family_bounds(&topology, bounds)?; + if bounds.allow_duplicate_experts == Some(false) && candidate_has_duplicate_experts(candidate) { + return Err( + "candidate violates bounds.allow_duplicate_experts=false due to duplicate mixture experts" + .to_string(), + ); + } + validate_candidate_parameter_ranges(candidate, bounds)?; + Ok(()) +} + +fn reject_candidate_local_external_artifacts( + candidate: &crate::api::CompressionBackend, +) -> Result<(), CandidateInvalidDiagnostic> { + if candidate_contains_external_artifact(candidate) { + Err(CandidateInvalidDiagnostic { + reason: TuneInvalidReason::CandidateExternalAssetForbidden, + diagnostic: format!( + "{}: candidate-local external filesystem/model path references are not allowed in tune candidates", + TuneInvalidReason::CandidateExternalAssetForbidden.as_str() + ), + }) + } else { + Ok(()) + } +} + +fn collect_candidate_topology( + candidate: &crate::api::CompressionBackend, +) -> Result { + let mut stats = CandidateTopologyStats { + backend_families: BTreeSet::new(), + max_mixture_nesting_depth: 0, + mixture_node_expert_counts: Vec::new(), + }; + match candidate { + crate::api::CompressionBackend::Zpaq { .. } => { + stats.backend_families.insert("zpaq".to_string()); + } + #[cfg(feature = "backend-rwkv")] + crate::api::CompressionBackend::Rwkv7 { .. } => { + stats.backend_families.insert("rwkv7".to_string()); + } + crate::api::CompressionBackend::Rate { rate_backend, .. } => { + collect_rate_backend_topology(rate_backend, 0, &mut stats)? + } + } + Ok(stats) +} + +fn collect_rate_backend_topology( + backend: &crate::api::RateBackend, + mixture_depth: usize, + stats: &mut CandidateTopologyStats, +) -> Result<(), String> { + let canonical_name = backend + .descriptor() + .map_err(|err| format!("failed to resolve backend descriptor: {err}"))? + .canonical + .to_string(); + stats.backend_families.insert(canonical_name); + + match backend { + crate::api::RateBackend::Mixture { spec } => { + let depth = mixture_depth + 1; + if depth > stats.max_mixture_nesting_depth { + stats.max_mixture_nesting_depth = depth; + } + stats.mixture_node_expert_counts.push(spec.experts.len()); + for expert in &spec.experts { + collect_rate_backend_topology(&expert.backend, depth, stats)?; + } + } + crate::api::RateBackend::Calibrated { spec } => { + collect_rate_backend_topology(&spec.base, mixture_depth, stats)?; + } + _ => {} + } + Ok(()) +} + +fn validate_candidate_backend_family_bounds( + topology: &CandidateTopologyStats, + bounds: &crate::spec::TuneBoundsSpec, +) -> Result<(), String> { + if topology.max_mixture_nesting_depth > bounds.max_mixture_nesting_depth { + return Err(format!( + "candidate mixture nesting depth {} exceeds bounds.max_mixture_nesting_depth {}", + topology.max_mixture_nesting_depth, bounds.max_mixture_nesting_depth + )); + } + if topology + .mixture_node_expert_counts + .iter() + .any(|count| *count > bounds.max_experts) + { + return Err(format!( + "candidate mixture expert count exceeds bounds.max_experts {}", + bounds.max_experts + )); + } + if let Some(min_experts) = bounds.min_experts + && topology + .mixture_node_expert_counts + .iter() + .any(|count| *count < min_experts) + { + return Err(format!( + "candidate mixture expert count is below bounds.min_experts {}", + min_experts + )); + } + + if !bounds.allowed_backends.is_empty() + && topology.backend_families.iter().any(|name| { + !bounds + .allowed_backends + .iter() + .any(|allowed| allowed == name) + }) + { + return Err( + "candidate contains backend family not listed in bounds.allowed_backends".into(), + ); + } + if topology.backend_families.iter().any(|name| { + bounds + .forbidden_backends + .iter() + .any(|blocked| blocked == name) + }) { + return Err("candidate contains backend family listed in bounds.forbidden_backends".into()); + } + if bounds + .required_experts + .iter() + .any(|required| !topology.backend_families.contains(required)) + { + return Err( + "candidate is missing at least one backend listed in bounds.required_experts".into(), + ); + } + if bounds.forbidden_expert_pairs.iter().any(|(left, right)| { + topology.backend_families.contains(left) && topology.backend_families.contains(right) + }) { + return Err("candidate violates bounds.forbidden_expert_pairs".into()); + } + Ok(()) +} + +fn validate_candidate_parameter_ranges( + candidate: &crate::api::CompressionBackend, + bounds: &crate::spec::TuneBoundsSpec, +) -> Result<(), String> { + if bounds.parameter_ranges.is_empty() { + return Ok(()); + } + let json = crate::spec::compression_backend_to_json_value(candidate) + .map_err(|err| format!("failed to materialize candidate JSON for range checks: {err}"))?; + let mut values = BTreeMap::::new(); + collect_numeric_parameter_paths("", &json, &mut values); + for range in &bounds.parameter_ranges { + let Some(value) = values.get(&range.parameter) else { + return Err(format!( + "candidate is missing bounded parameter path '{}'", + range.parameter + )); + }; + if *value < range.min || *value > range.max { + return Err(format!( + "candidate parameter '{}' = {} violates bounds [{}, {}]", + range.parameter, value, range.min, range.max + )); + } + } + Ok(()) +} + +fn collect_numeric_parameter_paths( + prefix: &str, + value: &Value, + output: &mut BTreeMap, +) { + match value { + Value::Object(map) => { + for (key, child) in map { + let next = if prefix.is_empty() { + key.clone() + } else { + format!("{prefix}.{key}") + }; + collect_numeric_parameter_paths(&next, child, output); + } + } + Value::Array(items) => { + for (idx, child) in items.iter().enumerate() { + let next = format!("{prefix}[{idx}]"); + collect_numeric_parameter_paths(&next, child, output); + } + } + Value::Number(number) => { + if let Some(value) = number.as_f64() { + output.insert(prefix.to_string(), value); + } + } + _ => {} + } +} + +fn candidate_contains_external_artifact(candidate: &crate::api::CompressionBackend) -> bool { + match candidate { + crate::api::CompressionBackend::Zpaq { method, .. } => { + zpaq_method_contains_external_file(method) + } + #[cfg(feature = "backend-rwkv")] + crate::api::CompressionBackend::Rwkv7 { method, .. } => { + rwkv_method_contains_external_artifact(method) + } + crate::api::CompressionBackend::Rate { rate_backend, .. } => { + rate_backend_contains_external_artifact(rate_backend) + } + } +} + +fn candidate_has_duplicate_experts(candidate: &crate::api::CompressionBackend) -> bool { + match candidate { + crate::api::CompressionBackend::Rate { rate_backend, .. } => { + rate_backend_has_duplicate_experts(rate_backend) + } + _ => false, + } +} + +fn rate_backend_has_duplicate_experts(backend: &crate::api::RateBackend) -> bool { + match backend { + crate::api::RateBackend::Mixture { spec } => { + let mut seen = BTreeSet::::new(); + for expert in &spec.experts { + let key = expert + .backend + .to_canonical_json() + .unwrap_or_else(|_| "".to_string()); + if !seen.insert(key) { + return true; + } + } + spec.experts + .iter() + .any(|expert| rate_backend_has_duplicate_experts(&expert.backend)) + } + crate::api::RateBackend::Calibrated { spec } => { + rate_backend_has_duplicate_experts(&spec.base) + } + _ => false, + } +} + +fn rate_backend_contains_external_artifact(backend: &crate::api::RateBackend) -> bool { + match backend { + #[cfg(feature = "backend-rwkv")] + crate::api::RateBackend::Rwkv7Method { method } => { + rwkv_method_contains_external_artifact(method) + } + #[cfg(feature = "backend-mamba")] + crate::api::RateBackend::MambaMethod { method } => { + mamba_method_contains_external_artifact(method) + } + crate::api::RateBackend::Mixture { spec } => spec + .experts + .iter() + .any(|expert| rate_backend_contains_external_artifact(&expert.backend)), + crate::api::RateBackend::Calibrated { spec } => { + rate_backend_contains_external_artifact(&spec.base) + } + _ => false, + } +} + +#[cfg(feature = "backend-rwkv")] +fn rwkv_method_contains_external_artifact(method: &crate::rwkvzip::MethodSpec) -> bool { + match method { + crate::rwkvzip::MethodSpec::File { .. } => true, + crate::rwkvzip::MethodSpec::Online { policy, .. } => policy + .as_ref() + .and_then(|policy| policy.load_from.as_ref()) + .is_some(), + } +} + +#[cfg(feature = "backend-mamba")] +fn mamba_method_contains_external_artifact(method: &crate::mambazip::MethodSpec) -> bool { + match method { + crate::mambazip::MethodSpec::File { .. } => true, + crate::mambazip::MethodSpec::Online { policy, .. } => policy + .as_ref() + .and_then(|policy| policy.load_from.as_ref()) + .is_some(), + } +} + +fn zpaq_method_contains_external_file(method: &crate::api::ZpaqMethodSpec) -> bool { + match method { + crate::api::ZpaqMethodSpec::Literal { value } => { + let trimmed = value.trim_start(); + trimmed.starts_with("file:") || trimmed.contains("://") + } + } +} + +fn bounds_hash(bounds: &crate::spec::TuneBoundsSpec) -> Result { + let value = serde_json::json!({ + "allowed_backends": bounds.allowed_backends, + "forbidden_backends": bounds.forbidden_backends, + "parameter_ranges": bounds.parameter_ranges.iter().map(|range| serde_json::json!({ + "parameter": range.parameter, + "min_bits": range.min.to_bits(), + "max_bits": range.max.to_bits(), + })).collect::>(), + "max_experts": bounds.max_experts, + "max_mixture_nesting_depth": bounds.max_mixture_nesting_depth, + "min_experts": bounds.min_experts, + "allow_duplicate_experts": bounds.allow_duplicate_experts, + "required_experts": bounds.required_experts, + "forbidden_expert_pairs": bounds.forbidden_expert_pairs, + }); + canonical_json_bytes(&value) + .map(|bytes| crc32_hex(&bytes)) + .map_err(|err| format!("failed to serialize bounds for hash: {err}")) +} + +fn crc32_hex(bytes: &[u8]) -> String { + let mut hasher = Hasher::new(); + hasher.update(bytes); + format!("{:08x}", hasher.finalize()) +} + +fn peak_rss_bytes() -> u64 { + #[cfg(target_os = "linux")] + { + if let Ok(status) = fs::read_to_string("/proc/self/status") { + for line in status.lines() { + if let Some(raw) = line.strip_prefix("VmHWM:") { + let kb = raw + .split_whitespace() + .next() + .and_then(|token| token.parse::().ok()); + if let Some(kb) = kb { + return kb.saturating_mul(1024); + } + } + } + } + } + 0 +} + +fn peak_memory_bytes(mode: PeakMemoryMode) -> u64 { + let process = peak_rss_bytes(); + let cgroup = cgroup_peak_memory_bytes(); + match mode { + PeakMemoryMode::ProcessRssPeak => process, + PeakMemoryMode::BackendReported => cgroup.unwrap_or(process), + PeakMemoryMode::HybridStrictMax => cgroup.unwrap_or(process).max(process), + } +} + +#[cfg(target_os = "linux")] +fn cgroup_peak_memory_bytes() -> Option { + cgroup_v2_peak_memory_bytes().or_else(cgroup_v1_peak_memory_bytes) +} + +#[cfg(not(target_os = "linux"))] +fn cgroup_peak_memory_bytes() -> Option { + None +} + +#[cfg(target_os = "linux")] +fn cgroup_v2_peak_memory_bytes() -> Option { + let relative = current_cgroup_path("0::")?; + let path = Path::new("/sys/fs/cgroup") + .join(relative.trim_start_matches('/')) + .join("memory.peak"); + read_u64_from_file(&path).ok() +} + +#[cfg(target_os = "linux")] +fn cgroup_v1_peak_memory_bytes() -> Option { + let relative = current_cgroup_memory_path()?; + let path = Path::new("/sys/fs/cgroup/memory") + .join(relative.trim_start_matches('/')) + .join("memory.max_usage_in_bytes"); + read_u64_from_file(&path).ok() +} + +#[cfg(target_os = "linux")] +fn current_cgroup_path(prefix: &str) -> Option { + let content = fs::read_to_string("/proc/self/cgroup").ok()?; + content + .lines() + .find_map(|line| line.strip_prefix(prefix).map(ToOwned::to_owned)) +} + +#[cfg(target_os = "linux")] +fn current_cgroup_memory_path() -> Option { + let content = fs::read_to_string("/proc/self/cgroup").ok()?; + content.lines().find_map(|line| { + let mut parts = line.splitn(3, ':'); + let _hierarchy = parts.next()?; + let controllers = parts.next()?; + let path = parts.next()?; + controllers + .split(',') + .any(|controller| controller == "memory") + .then(|| path.to_string()) + }) +} + +#[cfg(target_os = "linux")] +fn read_u64_from_file(path: &Path) -> Result { + let raw = fs::read_to_string(path) + .map_err(|err| format!("failed to read '{}': {err}", path.display()))?; + let trimmed = raw.trim(); + if trimmed == "max" { + return Err(format!( + "'{}' contains unbounded sentinel 'max'", + path.display() + )); + } + trimmed + .parse::() + .map_err(|err| format!("failed to parse '{}': {err}", path.display())) +} + +#[cfg(target_os = "linux")] +fn peak_rss_bytes_for_pid(pid: u32) -> Option { + let path = format!("/proc/{pid}/status"); + let status = fs::read_to_string(path).ok()?; + for line in status.lines() { + if let Some(raw) = line.strip_prefix("VmHWM:") { + let kb = raw + .split_whitespace() + .next() + .and_then(|token| token.parse::().ok())?; + return Some(kb.saturating_mul(1024)); + } + } + None +} + +#[cfg(all(unix, not(target_os = "linux")))] +fn peak_memory_bytes_for_pid(_pid: u32, _mode: PeakMemoryMode) -> Option { + None +} + +#[cfg(test)] +mod tests; diff --git a/crates/infotheory/src/tuner/annealer.rs b/crates/infotheory/src/tuner/annealer.rs new file mode 100644 index 00000000..56567aa6 --- /dev/null +++ b/crates/infotheory/src/tuner/annealer.rs @@ -0,0 +1,519 @@ +use super::*; + +pub(super) fn annealer_progress(tune_started: Instant, time_budget_seconds: f64) -> f64 { + annealer_progress_from_elapsed(tune_started.elapsed().as_secs_f64(), time_budget_seconds) +} + +pub(super) fn annealer_progress_from_elapsed( + elapsed_seconds: f64, + time_budget_seconds: f64, +) -> f64 { + if time_budget_seconds <= 0.0 { + return 1.0; + } + (elapsed_seconds / time_budget_seconds).clamp(0.0, 1.0) +} + +pub(super) fn annealer_temperature(progress: f64) -> f64 { + let u = progress.clamp(0.0, 1.0); + let temperature = ANNEALER_T_MIN_BITS * (ANNEALER_T0_BITS / ANNEALER_T_MIN_BITS).powf(1.0 - u); + temperature.clamp(ANNEALER_T_MIN_BITS, ANNEALER_T0_BITS) +} + +pub(super) fn annealer_active_radius(max_mutation_radius: usize, temperature: f64) -> usize { + ((max_mutation_radius as f64) * temperature) + .floor() + .max(1.0) as usize +} + +pub(super) fn annealer_runtime_path_name(profile: AnnealerKernelProfile) -> &'static str { + match profile { + AnnealerKernelProfile::ReversibleElementaryMetropolis => "reversible_elementary_metropolis", + AnnealerKernelProfile::CompiledUniformMetropolisHastings => { + "compiled_uniform_metropolis_hastings" + } + } +} + +pub(super) fn annealer_acceptance_probability( + profile: AnnealerKernelProfile, + delta: f64, + temperature: f64, + proposal: &AnnealedProposal, +) -> Result { + if proposal.forward_raw_action_count == 0 || proposal.forward_total_raw_actions == 0 { + return Ok(0.0); + } + let metropolis = (-delta / temperature).exp(); + match profile { + AnnealerKernelProfile::ReversibleElementaryMetropolis => { + let forward = (proposal.forward_raw_action_count as u128) + * (proposal.reverse_total_raw_actions as u128); + let reverse = (proposal.reverse_raw_action_count as u128) + * (proposal.forward_total_raw_actions as u128); + if forward != reverse { + return Err( + "compiled elementary proposal kernel failed reversibility check".to_string(), + ); + } + Ok(metropolis.clamp(0.0, 1.0)) + } + AnnealerKernelProfile::CompiledUniformMetropolisHastings => { + if proposal.reverse_raw_action_count == 0 || proposal.reverse_total_raw_actions == 0 { + return Ok(0.0); + } + let hastings_ratio = ((proposal.reverse_raw_action_count as f64) + * (proposal.forward_total_raw_actions as f64)) + / ((proposal.forward_raw_action_count as f64) + * (proposal.reverse_total_raw_actions as f64)); + Ok((metropolis * hastings_ratio).clamp(0.0, 1.0)) + } + } +} + +pub(super) fn sample_annealed_proposal( + candidate: &crate::api::CompressionBackend, + bounds: &crate::spec::TuneBoundsSpec, + max_mutation_radius: usize, + active_radius: usize, + env: &SpecEnvironment, + rng: &mut RandomGenerator, +) -> Result { + let current_compiled = candidate + .compile_in(env) + .map_err(|err| format!("failed to compile current annealer candidate: {err}"))?; + let current_canonical_bytes = current_compiled.canonical_bytes().as_slice().to_vec(); + let forward = compile_canonical_proposal_kernel( + candidate, + bounds, + max_mutation_radius, + active_radius, + env, + ¤t_canonical_bytes, + )?; + if forward.transitions.is_empty() { + return Ok(AnnealedProposalDraw::Exhausted); + } + let Some(proposed) = forward.sample(rng) else { + return Ok(AnnealedProposalDraw::SelfLoop); + }; + let reverse = compile_canonical_proposal_kernel( + &proposed.candidate, + bounds, + max_mutation_radius, + active_radius, + env, + &proposed.candidate_canonical_bytes, + )?; + Ok(AnnealedProposalDraw::Proposal(AnnealedProposal { + candidate: proposed.candidate.clone(), + forward_raw_action_count: proposed.raw_action_count, + forward_total_raw_actions: forward.total_raw_actions, + reverse_raw_action_count: reverse + .proposal_mass_to_canonical_bytes(¤t_canonical_bytes), + reverse_total_raw_actions: reverse.total_raw_actions, + })) +} + +pub(super) fn compile_canonical_proposal_kernel( + candidate: &crate::api::CompressionBackend, + bounds: &crate::spec::TuneBoundsSpec, + max_mutation_radius: usize, + active_radius: usize, + env: &SpecEnvironment, + current_canonical_bytes: &[u8], +) -> Result { + let json = crate::spec::compression_backend_to_json_value(candidate) + .map_err(|err| format!("failed to serialize candidate for proposal kernel: {err}"))?; + let range_map = bounds + .parameter_ranges + .iter() + .map(|range| (range.parameter.clone(), (range.min, range.max))) + .collect::>(); + let mut leaves = collect_numeric_leaves(&json); + if !range_map.is_empty() { + leaves.retain(|leaf| range_map.contains_key(&leaf.path)); + } + let descriptors = compile_numeric_mutation_descriptors(leaves, &range_map, max_mutation_radius); + let total_raw_actions = (descriptors.len() as u64) + .saturating_mul(2) + .saturating_mul(max_mutation_radius.max(1) as u64); + let mut transitions = BTreeMap::, CanonicalProposal>::new(); + for descriptor in &descriptors { + for magnitude in 1..=max_mutation_radius.max(1) { + for sign in [-1i8, 1i8] { + if magnitude > active_radius { + continue; + } + let mut next_json = json.clone(); + if !apply_numeric_descriptor(&mut next_json, descriptor, magnitude, sign) { + continue; + } + let parsed = match crate::spec::parse_compression_backend_json( + &next_json, + Path::new("."), + None, + crate::compression::FramingMode::Framed, + ) { + Ok(value) => value, + Err(_) => continue, + }; + if reject_candidate_local_external_artifacts(&parsed).is_err() + || validate_candidate_against_tune_bounds(&parsed, bounds).is_err() + { + continue; + } + let compiled = match parsed.compile_in(env) { + Ok(value) => value, + Err(_) => continue, + }; + let candidate_canonical_bytes = compiled.canonical_bytes().as_slice().to_vec(); + if candidate_canonical_bytes == current_canonical_bytes { + continue; + } + transitions + .entry(candidate_canonical_bytes.clone()) + .and_modify(|proposal| { + proposal.raw_action_count = proposal.raw_action_count.saturating_add(1); + }) + .or_insert(CanonicalProposal { + candidate: parsed, + candidate_canonical_bytes, + raw_action_count: 1, + }); + } + } + } + Ok(CanonicalProposalKernel { + transitions: transitions.into_values().collect(), + total_raw_actions, + }) +} + +#[derive(Clone)] +struct NumericMutationDescriptor { + leaf: NumericLeaf, + domain: NumericMutationDomain, +} + +#[derive(Clone, Copy)] +enum NumericMutationDomain { + Integer { + kind: NumericKind, + min_bound: i128, + max_bound: i128, + }, + Float { + min_key: u64, + max_key: u64, + stride: u64, + }, +} + +fn compile_numeric_mutation_descriptors( + leaves: Vec, + range_map: &BTreeMap, + max_mutation_radius: usize, +) -> Vec { + leaves + .into_iter() + .filter_map(|leaf| { + let range = range_map.get(&leaf.path).copied(); + let domain = match leaf.kind { + NumericKind::Unsigned | NumericKind::Signed => { + let effective_kind = effective_integer_kind(leaf.kind, range); + let (min_bound, max_bound) = integer_leaf_bounds(effective_kind, range)?; + NumericMutationDomain::Integer { + kind: effective_kind, + min_bound, + max_bound, + } + } + NumericKind::Float => { + let (min_key, max_key, stride) = float_leaf_bounds(range, max_mutation_radius)?; + NumericMutationDomain::Float { + min_key, + max_key, + stride, + } + } + }; + Some(NumericMutationDescriptor { leaf, domain }) + }) + .collect() +} + +pub(super) fn integer_leaf_bounds( + kind: NumericKind, + range: Option<(f64, f64)>, +) -> Option<(i128, i128)> { + let (type_min, type_max) = match kind { + NumericKind::Unsigned => (0i128, u64::MAX as i128), + NumericKind::Signed => (i64::MIN as i128, i64::MAX as i128), + NumericKind::Float => return None, + }; + let (min, max) = match range { + Some((min, max)) => { + if !min.is_finite() || !max.is_finite() { + return None; + } + ( + (min.ceil() as i128).clamp(type_min, type_max), + (max.floor() as i128).clamp(type_min, type_max), + ) + } + None => (type_min, type_max), + }; + (min <= max).then_some((min, max)) +} + +fn float_leaf_bounds( + range: Option<(f64, f64)>, + max_mutation_radius: usize, +) -> Option<(u64, u64, u64)> { + let (min, max) = range?; + if !min.is_finite() || !max.is_finite() || min > max { + return None; + } + let min_key = f64_to_ordered_key(min)?; + let max_key = f64_to_ordered_key(max)?; + if min_key > max_key { + return None; + } + let span = max_key - min_key; + let denominator = (max_mutation_radius.max(1) as u128) + .saturating_mul(2) + .saturating_add(1); + let stride = ((span as u128) / denominator).max(1); + Some((min_key, max_key, u64::try_from(stride).unwrap_or(u64::MAX))) +} + +fn f64_to_ordered_key(value: f64) -> Option { + if !value.is_finite() { + return None; + } + let bits = value.to_bits(); + let sign_mask = 1_u64 << 63; + if bits & sign_mask == 0 { + Some(bits | sign_mask) + } else { + Some(!bits) + } +} + +fn ordered_key_to_f64(key: u64) -> f64 { + let sign_mask = 1_u64 << 63; + let bits = if key & sign_mask == 0 { + !key + } else { + key & !sign_mask + }; + f64::from_bits(bits) +} + +fn apply_numeric_descriptor( + json: &mut Value, + descriptor: &NumericMutationDescriptor, + magnitude: usize, + sign: i8, +) -> bool { + match descriptor.domain { + NumericMutationDomain::Integer { + kind, + min_bound, + max_bound, + } => apply_integer_descriptor( + json, + &descriptor.leaf, + kind, + min_bound, + max_bound, + magnitude, + sign, + ), + NumericMutationDomain::Float { + min_key, + max_key, + stride, + } => apply_float_descriptor( + json, + &descriptor.leaf, + min_key, + max_key, + stride, + magnitude, + sign, + ), + } +} + +pub(super) fn apply_integer_descriptor( + json: &mut Value, + leaf: &NumericLeaf, + kind: NumericKind, + min_bound: i128, + max_bound: i128, + magnitude: usize, + sign: i8, +) -> bool { + let Some(slot) = json.pointer_mut(&leaf.pointer) else { + return false; + }; + let current = match kind { + NumericKind::Unsigned => slot.as_u64().map(i128::from), + NumericKind::Signed => slot.as_i64().map(i128::from), + NumericKind::Float => None, + }; + let Some(current) = current else { + return false; + }; + let Ok(magnitude) = i128::try_from(magnitude) else { + return false; + }; + let delta = if sign < 0 { -magnitude } else { magnitude }; + let Some(next) = current.checked_add(delta) else { + return false; + }; + if next < min_bound || next > max_bound || next == current { + return false; + } + match kind { + NumericKind::Unsigned => { + let Ok(value) = u64::try_from(next) else { + return false; + }; + *slot = Value::Number(serde_json::Number::from(value)); + true + } + NumericKind::Signed => { + let Ok(value) = i64::try_from(next) else { + return false; + }; + *slot = Value::Number(serde_json::Number::from(value)); + true + } + NumericKind::Float => false, + } +} + +fn effective_integer_kind(kind: NumericKind, range: Option<(f64, f64)>) -> NumericKind { + if matches!(kind, NumericKind::Unsigned) + && range.is_some_and(|(min, _)| min.is_finite() && min < 0.0) + { + NumericKind::Signed + } else { + kind + } +} + +fn apply_float_descriptor( + json: &mut Value, + leaf: &NumericLeaf, + min_key: u64, + max_key: u64, + stride: u64, + magnitude: usize, + sign: i8, +) -> bool { + let Some(slot) = json.pointer_mut(&leaf.pointer) else { + return false; + }; + let Some(current) = slot.as_f64() else { + return false; + }; + let Some(current_key) = f64_to_ordered_key(current) else { + return false; + }; + if current_key < min_key || current_key > max_key { + return false; + } + let step = (magnitude as u128).saturating_mul(stride as u128); + let Ok(step) = u64::try_from(step) else { + return false; + }; + let next_key = if sign < 0 { + let Some(value) = current_key.checked_sub(step) else { + return false; + }; + value + } else { + let Some(value) = current_key.checked_add(step) else { + return false; + }; + value + }; + if next_key < min_key || next_key > max_key || next_key == current_key { + return false; + } + let next = ordered_key_to_f64(next_key); + if !next.is_finite() { + return false; + } + let Some(number) = serde_json::Number::from_f64(next) else { + return false; + }; + *slot = Value::Number(number); + true +} + +pub(super) fn collect_numeric_leaves(root: &Value) -> Vec { + let mut out = Vec::::new(); + collect_numeric_leaves_inner(root, "", "", &mut out); + out +} + +fn collect_numeric_leaves_inner( + value: &Value, + path_prefix: &str, + pointer_prefix: &str, + out: &mut Vec, +) { + match value { + Value::Object(object) => { + for (key, child) in object { + let next_path = if path_prefix.is_empty() { + key.to_string() + } else { + format!("{path_prefix}.{key}") + }; + let escaped = key.replace('~', "~0").replace('/', "~1"); + let next_pointer = if pointer_prefix.is_empty() { + format!("/{escaped}") + } else { + format!("{pointer_prefix}/{escaped}") + }; + collect_numeric_leaves_inner(child, &next_path, &next_pointer, out); + } + } + Value::Array(items) => { + for (index, child) in items.iter().enumerate() { + let next_path = format!("{path_prefix}[{index}]"); + let next_pointer = if pointer_prefix.is_empty() { + format!("/{index}") + } else { + format!("{pointer_prefix}/{index}") + }; + collect_numeric_leaves_inner(child, &next_path, &next_pointer, out); + } + } + Value::Number(number) => { + let kind = if number.as_u64().is_some() { + Some(NumericKind::Unsigned) + } else if number.as_i64().is_some() { + Some(NumericKind::Signed) + } else if number.as_f64().is_some() { + Some(NumericKind::Float) + } else { + None + }; + if let Some(kind) = kind { + out.push(NumericLeaf { + path: path_prefix.to_string(), + pointer: pointer_prefix.to_string(), + kind, + }); + } + } + _ => {} + } +} diff --git a/crates/infotheory/src/tuner/causal_dataset.rs b/crates/infotheory/src/tuner/causal_dataset.rs new file mode 100644 index 00000000..980db4db --- /dev/null +++ b/crates/infotheory/src/tuner/causal_dataset.rs @@ -0,0 +1,1106 @@ +use super::*; + +pub(super) fn load_dataset(path: &Path) -> Result { + let bytes = fs::read(path) + .map_err(|err| format!("failed to read input_asset '{}': {err}", path.display()))?; + if let Ok(Value::Object(object)) = serde_json::from_slice::(&bytes) { + if object.contains_key("events") { + let value = Value::Object(object); + return lower_interactive_trace_dataset(path, bytes.len(), &value); + } + if object.contains_key("examples") || object.contains_key("prefixes") { + let value = Value::Object(object); + return lower_causal_prefix_dataset(path, bytes.len(), &value); + } + return Err( + "JSON object input_asset must match a canonical tuner causal dataset kind: provide 'events' or 'examples'/'prefixes'" + .to_string(), + ); + } + let hash = crc32_hex(&bytes); + Ok(LoadedDataset { + kind: DatasetKind::PassiveBytes, + objective_target: ObjectiveTarget::PassiveAc, + lowering_version: PASSIVE_DATASET_LOWERING_VERSION, + codec_hash: "passive-identity-bytes".to_string(), + event_grammar_hash: "passive-target-only-byte-stream".to_string(), + target_domain_support_hash: crc32_hex(b"passive-byte-alphabet"), + causal_header_profile_hash: crc32_hex(b"passive-none"), + target_size_function: "passive-bytes-len", + canonical_content_hash: hash, + lowered_skeleton_hash: crc32_hex(b"passive-bytes-target-only"), + resolved_path: path.to_string_lossy().to_string(), + source_size_bytes: bytes.len(), + dataset_units: bytes.len() as f64, + target_events: usize::from(!bytes.is_empty()), + events: Vec::new(), + causal_profile: None, + raw_bytes: bytes, + }) +} + +fn lower_interactive_trace_dataset( + path: &Path, + source_size_bytes: usize, + value: &Value, +) -> Result { + let (header_profile, domain_supports) = parse_causal_header_profile( + value, + "interactive trace dataset", + CausalPayloadKind::Events, + )?; + let events_value = value + .get("events") + .and_then(Value::as_array) + .ok_or_else(|| "interactive trace dataset requires an array field 'events'".to_string())?; + let events = events_value + .iter() + .enumerate() + .map(|(index, event)| parse_lowered_event(event, &format!("events[{index}]"))) + .collect::, _>>()?; + lowered_dataset_from_events( + DatasetKind::InteractiveTrace, + ObjectiveTarget::InteractiveCausalAc, + INTERACTIVE_TRACE_LOWERING_VERSION, + path, + source_size_bytes, + value, + header_profile, + domain_supports, + events, + "interactive-trace-target-bytes", + ) +} + +fn lower_causal_prefix_dataset( + path: &Path, + source_size_bytes: usize, + value: &Value, +) -> Result { + let (header_profile, domain_supports) = parse_causal_header_profile( + value, + "causal-prefix dataset", + CausalPayloadKind::ExamplesOrPrefixes, + )?; + let examples_value = value + .get("examples") + .or_else(|| value.get("prefixes")) + .and_then(Value::as_array) + .ok_or_else(|| { + "causal-prefix dataset requires an array field 'examples' or 'prefixes'".to_string() + })?; + let mut events = Vec::::new(); + for (index, example) in examples_value.iter().enumerate() { + let object = example + .as_object() + .ok_or_else(|| format!("examples[{index}] must be an object"))?; + events.push(LoweredCausalEvent::Reset); + if let Some(history) = object.get("history").and_then(Value::as_array) { + for (history_index, event) in history.iter().enumerate() { + let replay = parse_lowered_event( + event, + &format!("examples[{index}].history[{history_index}]"), + )?; + if matches!(replay, LoweredCausalEvent::Target { .. }) { + return Err(format!( + "examples[{index}].history[{history_index}] must replay targets with observe_target_no_score, not charged target events" + )); + } + events.push(replay); + } + } + if let Some(action) = object.get("action") { + events.push(LoweredCausalEvent::Context { + channel: "action".to_string(), + bytes: payload_bytes(action, &format!("examples[{index}].action"))?, + }); + } + let target = object + .get("target") + .or_else(|| object.get("percept")) + .ok_or_else(|| format!("examples[{index}] requires 'target' or 'percept'"))?; + let weight = object + .get("weight") + .map(|raw| { + raw.as_f64() + .filter(|value| value.is_finite() && *value > 0.0) + .ok_or_else(|| format!("examples[{index}].weight must be finite and > 0")) + }) + .transpose()? + .unwrap_or(1.0); + events.push(LoweredCausalEvent::Target { + channel: required_nonempty_string_field( + object.get("channel"), + &format!("examples[{index}].channel"), + )?, + domain: required_nonempty_string_field( + object.get("domain"), + &format!("examples[{index}].domain"), + )?, + bytes: payload_bytes(target, &format!("examples[{index}].target"))?, + weight, + }); + } + lowered_dataset_from_events( + DatasetKind::CausalPrefixDataset, + ObjectiveTarget::InteractiveCausalAc, + CAUSAL_PREFIX_LOWERING_VERSION, + path, + source_size_bytes, + value, + header_profile, + domain_supports, + events, + "weighted-target-bytes-sum", + ) +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum CausalPayloadKind { + Events, + ExamplesOrPrefixes, +} + +fn parse_causal_header_profile( + value: &Value, + label: &str, + payload_kind: CausalPayloadKind, +) -> Result<(CausalHeaderProfile, BTreeMap), String> { + let object = value + .as_object() + .ok_or_else(|| format!("{label} must be a JSON object"))?; + let allowed = match payload_kind { + CausalPayloadKind::Events => [ + "schema_version", + "environment_id", + "environment_config_crc32", + "codec_hash", + "reset_convention", + "action_alphabet", + "percept_schema", + "reward_encoding", + "terminal_encoding", + "collection_policy", + "target_domains", + "event_grammar", + "events", + ] + .as_slice(), + CausalPayloadKind::ExamplesOrPrefixes => [ + "schema_version", + "environment_id", + "environment_config_crc32", + "codec_hash", + "reset_convention", + "action_alphabet", + "percept_schema", + "reward_encoding", + "terminal_encoding", + "collection_policy", + "target_domains", + "event_grammar", + "examples", + "prefixes", + ] + .as_slice(), + }; + ensure_known_fields_in_object(object, allowed, label)?; + + let schema_version = object + .get("schema_version") + .and_then(Value::as_u64) + .ok_or_else(|| format!("{label} requires schema_version: 1"))?; + if schema_version != 1 { + return Err(format!("{label} schema_version must be 1")); + } + let _environment_id = + required_nonempty_string_field(object.get("environment_id"), "environment_id")?; + let env_crc32 = required_nonempty_string_field( + object.get("environment_config_crc32"), + "environment_config_crc32", + )?; + if !is_lower_hex_crc32(&env_crc32) { + return Err(format!( + "{label}.environment_config_crc32 must be an 8-character lowercase hex CRC32" + )); + } + let _codec_hash = required_nonempty_string_field(object.get("codec_hash"), "codec_hash")?; + let reset_convention = + required_nonempty_string_field(object.get("reset_convention"), "reset_convention")?; + if reset_convention != "reset-before-episode" { + return Err(format!( + "{label}.reset_convention must be 'reset-before-episode'" + )); + } + + let domains = parse_causal_target_domains( + object + .get("target_domains") + .ok_or_else(|| format!("{label} requires target_domains"))?, + )?; + let action_alphabet_size = parse_action_alphabet_header( + object + .get("action_alphabet") + .ok_or_else(|| format!("{label} requires action_alphabet"))?, + )?; + let percept_channels = parse_percept_schema_header( + object + .get("percept_schema") + .ok_or_else(|| format!("{label} requires percept_schema"))?, + )?; + let reward_channel = parse_single_encoding_channel_header( + object + .get("reward_encoding") + .ok_or_else(|| format!("{label} requires reward_encoding"))?, + "reward_encoding", + )?; + let terminal_channel = parse_single_encoding_channel_header( + object + .get("terminal_encoding") + .ok_or_else(|| format!("{label} requires terminal_encoding"))?, + "terminal_encoding", + )?; + let collection_policy = + required_nonempty_string_field(object.get("collection_policy"), "collection_policy")?; + let event_grammar = parse_event_grammar_header( + object + .get("event_grammar") + .ok_or_else(|| format!("{label} requires event_grammar"))?, + )?; + + validate_header_profile_consistency( + &domains, + &event_grammar, + &percept_channels, + &reward_channel, + &terminal_channel, + )?; + + if payload_kind == CausalPayloadKind::ExamplesOrPrefixes + && object.get("examples").is_some() + && object.get("prefixes").is_some() + { + return Err(format!( + "{label} must declare exactly one of 'examples' or 'prefixes', not both" + )); + } + + let profile_hash = causal_header_profile_hash( + action_alphabet_size, + &collection_policy, + &percept_channels, + &reward_channel, + &terminal_channel, + &event_grammar, + )?; + Ok(( + CausalHeaderProfile { + action_alphabet_size, + collection_policy, + percept_channels, + reward_channel, + terminal_channel, + event_grammar, + profile_hash, + }, + domains, + )) +} + +fn ensure_known_fields_in_object( + object: &serde_json::Map, + allowed: &[&str], + label: &str, +) -> Result<(), String> { + for key in object.keys() { + if !allowed.contains(&key.as_str()) { + return Err(format!("{label} contains unknown field '{key}'")); + } + } + Ok(()) +} + +fn parse_action_alphabet_header(value: &Value) -> Result { + let object = value + .as_object() + .ok_or_else(|| "action_alphabet must be an object".to_string())?; + ensure_known_fields_in_object(object, &["size"], "action_alphabet")?; + let size = object + .get("size") + .and_then(Value::as_u64) + .ok_or_else(|| "action_alphabet.size must be an integer".to_string())?; + let size = usize::try_from(size).map_err(|_| "action_alphabet.size does not fit usize")?; + if size == 0 { + return Err("action_alphabet.size must be >= 1".to_string()); + } + Ok(size) +} + +fn parse_percept_schema_header(value: &Value) -> Result, String> { + let object = value + .as_object() + .ok_or_else(|| "percept_schema must be an object".to_string())?; + ensure_known_fields_in_object(object, &["encoding", "channels"], "percept_schema")?; + let encoding = object + .get("encoding") + .and_then(Value::as_str) + .ok_or_else(|| "percept_schema.encoding is required".to_string())?; + if encoding != "bytes" { + return Err("percept_schema.encoding must be 'bytes'".to_string()); + } + let channels = object + .get("channels") + .and_then(Value::as_array) + .ok_or_else(|| "percept_schema.channels must be an array".to_string())?; + if channels.is_empty() { + return Err("percept_schema.channels must contain at least one entry".to_string()); + } + let mut out = BTreeSet::::new(); + for (index, value) in channels.iter().enumerate() { + let pair = parse_channel_domain_pair(value, &format!("percept_schema.channels[{index}]"))?; + if !out.insert(pair.clone()) { + return Err(format!( + "percept_schema.channels[{index}] duplicates ({}, {})", + pair.channel, pair.domain + )); + } + } + Ok(out) +} + +fn parse_single_encoding_channel_header( + value: &Value, + label: &str, +) -> Result { + let object = value + .as_object() + .ok_or_else(|| format!("{label} must be an object"))?; + ensure_known_fields_in_object(object, &["encoding", "channel", "domain"], label)?; + let encoding = object + .get("encoding") + .and_then(Value::as_str) + .ok_or_else(|| format!("{label}.encoding is required"))?; + if encoding != "bytes" { + return Err(format!("{label}.encoding must be 'bytes'")); + } + Ok(CausalChannelDomain { + channel: required_nonempty_string_field( + object.get("channel"), + &format!("{label}.channel"), + )?, + domain: required_nonempty_string_field(object.get("domain"), &format!("{label}.domain"))?, + }) +} + +fn parse_event_grammar_header(value: &Value) -> Result { + let object = value + .as_object() + .ok_or_else(|| "event_grammar must be an object".to_string())?; + ensure_known_fields_in_object( + object, + &["context_channels", "observe_target_no_score", "target"], + "event_grammar", + )?; + let context_channels = object + .get("context_channels") + .and_then(Value::as_array) + .ok_or_else(|| "event_grammar.context_channels must be an array".to_string())?; + let mut context = BTreeSet::::new(); + for (index, raw) in context_channels.iter().enumerate() { + let channel = raw + .as_str() + .map(str::trim) + .filter(|value| !value.is_empty()) + .ok_or_else(|| { + format!( + "event_grammar.context_channels[{index}] must be a non-empty channel string" + ) + })? + .to_string(); + if !context.insert(channel.clone()) { + return Err(format!( + "event_grammar.context_channels[{index}] duplicates '{channel}'" + )); + } + } + let observe = parse_grammar_channel_domains( + object + .get("observe_target_no_score") + .ok_or_else(|| "event_grammar.observe_target_no_score is required".to_string())?, + "event_grammar.observe_target_no_score", + )?; + let target = parse_grammar_channel_domains( + object + .get("target") + .ok_or_else(|| "event_grammar.target is required".to_string())?, + "event_grammar.target", + )?; + Ok(CausalEventGrammar { + context_channels: context, + observe_target_no_score: observe, + target, + }) +} + +fn parse_grammar_channel_domains( + value: &Value, + label: &str, +) -> Result, String> { + let array = value + .as_array() + .ok_or_else(|| format!("{label} must be an array"))?; + let mut out = BTreeSet::::new(); + for (index, entry) in array.iter().enumerate() { + let pair = parse_channel_domain_pair(entry, &format!("{label}[{index}]"))?; + if !out.insert(pair.clone()) { + return Err(format!( + "{label}[{index}] duplicates ({}, {})", + pair.channel, pair.domain + )); + } + } + Ok(out) +} + +fn parse_channel_domain_pair(value: &Value, label: &str) -> Result { + let object = value + .as_object() + .ok_or_else(|| format!("{label} must be an object"))?; + ensure_known_fields_in_object(object, &["channel", "domain"], label)?; + Ok(CausalChannelDomain { + channel: required_nonempty_string_field( + object.get("channel"), + &format!("{label}.channel"), + )?, + domain: required_nonempty_string_field(object.get("domain"), &format!("{label}.domain"))?, + }) +} + +fn validate_header_profile_consistency( + domains: &BTreeMap, + event_grammar: &CausalEventGrammar, + percept_channels: &BTreeSet, + reward_channel: &CausalChannelDomain, + terminal_channel: &CausalChannelDomain, +) -> Result<(), String> { + for pair in event_grammar + .observe_target_no_score + .iter() + .chain(event_grammar.target.iter()) + { + if !domains.contains_key(&pair.domain) { + return Err(format!( + "event_grammar references undeclared target domain '{}'", + pair.domain + )); + } + } + for pair in percept_channels { + if !event_grammar.target.contains(pair) { + return Err(format!( + "percept_schema channel/domain ({}, {}) must appear in event_grammar.target", + pair.channel, pair.domain + )); + } + } + if !event_grammar.target.contains(reward_channel) { + return Err(format!( + "reward_encoding channel/domain ({}, {}) must appear in event_grammar.target", + reward_channel.channel, reward_channel.domain + )); + } + if !event_grammar.target.contains(terminal_channel) { + return Err(format!( + "terminal_encoding channel/domain ({}, {}) must appear in event_grammar.target", + terminal_channel.channel, terminal_channel.domain + )); + } + Ok(()) +} + +fn causal_header_profile_hash( + action_alphabet_size: usize, + collection_policy: &str, + percept_channels: &BTreeSet, + reward_channel: &CausalChannelDomain, + terminal_channel: &CausalChannelDomain, + event_grammar: &CausalEventGrammar, +) -> Result { + let value = serde_json::json!({ + "action_alphabet_size": action_alphabet_size, + "collection_policy": collection_policy, + "percept_channels": percept_channels + .iter() + .map(|pair| { + serde_json::json!({"channel": pair.channel.as_str(), "domain": pair.domain.as_str()}) + }) + .collect::>(), + "reward_channel": {"channel": reward_channel.channel.as_str(), "domain": reward_channel.domain.as_str()}, + "terminal_channel": {"channel": terminal_channel.channel.as_str(), "domain": terminal_channel.domain.as_str()}, + "event_grammar": { + "context_channels": event_grammar.context_channels.iter().collect::>(), + "observe_target_no_score": event_grammar.observe_target_no_score + .iter() + .map(|pair| { + serde_json::json!({"channel": pair.channel.as_str(), "domain": pair.domain.as_str()}) + }) + .collect::>(), + "target": event_grammar.target + .iter() + .map(|pair| { + serde_json::json!({"channel": pair.channel.as_str(), "domain": pair.domain.as_str()}) + }) + .collect::>(), + } + }); + let bytes = canonical_json_bytes(&value) + .map_err(|err| format!("failed to encode causal header profile hash: {err}"))?; + Ok(crc32_hex(&bytes)) +} + +fn parse_causal_target_domains( + value: &Value, +) -> Result, String> { + let object = value + .as_object() + .ok_or_else(|| "target_domains must be an object".to_string())?; + if object.is_empty() { + return Err("target_domains must declare at least one target domain".to_string()); + } + object + .iter() + .map(|(domain, spec)| { + if domain.trim().is_empty() { + return Err("target_domains contains an empty domain tag".to_string()); + } + let spec_object = spec + .as_object() + .ok_or_else(|| format!("target_domains.{domain} must be an object"))?; + let kind = spec_object + .get("kind") + .and_then(Value::as_str) + .ok_or_else(|| format!("target_domains.{domain}.kind is required"))?; + match kind { + "byte_alphabet" => { + ensure_known_causal_domain_fields( + spec_object, + &["kind"], + &format!("target_domains.{domain}"), + )?; + Ok((domain.clone(), CausalTargetDomain::ByteAlphabet)) + } + "enumerated_payloads" => { + ensure_known_causal_domain_fields( + spec_object, + &["kind", "payloads"], + &format!("target_domains.{domain}"), + )?; + let payloads_value = spec_object + .get("payloads") + .and_then(Value::as_array) + .ok_or_else(|| { + format!("target_domains.{domain}.payloads must be an array") + })?; + if payloads_value.is_empty() { + return Err(format!( + "target_domains.{domain}.payloads must contain at least one payload" + )); + } + let mut seen = BTreeSet::>::new(); + let mut payloads = Vec::>::with_capacity(payloads_value.len()); + for (index, payload) in payloads_value.iter().enumerate() { + let bytes = payload_bytes( + payload, + &format!("target_domains.{domain}.payloads[{index}]"), + )?; + if !seen.insert(bytes.clone()) { + return Err(format!( + "target_domains.{domain}.payloads[{index}] duplicates an enumerated payload" + )); + } + payloads.push(bytes); + } + Ok((domain.clone(), CausalTargetDomain::EnumeratedPayloads { payloads })) + } + other => Err(format!( + "target_domains.{domain}.kind has unknown target-domain support kind '{other}'" + )), + } + }) + .collect() +} + +fn ensure_known_causal_domain_fields( + object: &serde_json::Map, + allowed: &[&str], + label: &str, +) -> Result<(), String> { + for key in object.keys() { + if !allowed.contains(&key.as_str()) { + return Err(format!( + "{label} contains unknown target-domain field '{key}'" + )); + } + } + Ok(()) +} + +#[allow(clippy::too_many_arguments)] +fn lowered_dataset_from_events( + kind: DatasetKind, + objective_target: ObjectiveTarget, + lowering_version: &'static str, + path: &Path, + source_size_bytes: usize, + source_value: &Value, + header_profile: CausalHeaderProfile, + domain_supports: BTreeMap, + events: Vec, + target_size_function: &'static str, +) -> Result { + let canonical_bytes = canonical_json_bytes(source_value) + .map_err(|err| format!("failed to canonicalize causal dataset JSON: {err}"))?; + let canonical_content_hash = crc32_hex(&canonical_bytes); + let normalized_events = expand_byte_alphabet_events(events, &domain_supports)?; + validate_events_against_causal_profile(&normalized_events, &domain_supports, &header_profile)?; + let mut charged = Vec::::new(); + let mut target_events = 0usize; + let mut dataset_units = 0.0f64; + for event in &normalized_events { + if let LoweredCausalEvent::Target { bytes, weight, .. } = event { + charged.extend_from_slice(bytes); + target_events = target_events.saturating_add(1); + dataset_units += (*weight) * (bytes.len() as f64); + } + } + let skeleton_bytes = lowered_event_skeleton_bytes(&normalized_events)?; + let domain_support_bytes = causal_domain_support_bytes(&domain_supports)?; + let domain_support_hash = crc32_hex(&domain_support_bytes); + let channel_set = causal_channel_set(&normalized_events); + Ok(LoadedDataset { + kind, + objective_target, + lowering_version, + codec_hash: causal_dataset_string_field(source_value, "codec_hash") + .unwrap_or_else(|| "json-causal-byte-events-v1".to_string()), + event_grammar_hash: crc32_hex(&skeleton_bytes), + target_domain_support_hash: domain_support_hash.clone(), + causal_header_profile_hash: header_profile.profile_hash.clone(), + target_size_function, + canonical_content_hash, + lowered_skeleton_hash: crc32_hex(&skeleton_bytes), + resolved_path: path.to_string_lossy().to_string(), + source_size_bytes, + raw_bytes: charged, + events: normalized_events, + causal_profile: Some(CausalEvaluationProfile { + domains: domain_supports, + channel_set, + domain_support_hash, + byte_alphabet_symbol_width: 1, + header_profile_hash: header_profile.profile_hash, + event_grammar: header_profile.event_grammar, + action_alphabet_size: header_profile.action_alphabet_size, + collection_policy: header_profile.collection_policy, + percept_channels: header_profile.percept_channels, + reward_channel: header_profile.reward_channel, + terminal_channel: header_profile.terminal_channel, + }), + dataset_units, + target_events, + }) +} + +fn parse_lowered_event(value: &Value, label: &str) -> Result { + let object = value + .as_object() + .ok_or_else(|| format!("{label} must be an object"))?; + let kind = object + .get("kind") + .and_then(Value::as_str) + .ok_or_else(|| format!("{label}.kind is required"))?; + match kind { + "reset" => { + ensure_known_event_fields(object, &["kind"], label)?; + Ok(LoweredCausalEvent::Reset) + } + "context" => { + ensure_known_event_fields(object, &["kind", "channel", "bytes"], label)?; + Ok(LoweredCausalEvent::Context { + channel: required_nonempty_string_field( + object.get("channel"), + &format!("{label}.channel"), + )?, + bytes: event_payload_bytes(value, label)?, + }) + } + "observe_target_no_score" => { + ensure_known_event_fields(object, &["kind", "channel", "domain", "bytes"], label)?; + Ok(LoweredCausalEvent::ObserveTargetNoScore { + channel: required_nonempty_string_field( + object.get("channel"), + &format!("{label}.channel"), + )?, + domain: required_nonempty_string_field( + object.get("domain"), + &format!("{label}.domain"), + )?, + bytes: event_payload_bytes(value, label)?, + }) + } + "target" => { + ensure_known_event_fields( + object, + &["kind", "channel", "domain", "bytes", "weight"], + label, + )?; + let channel = + required_nonempty_string_field(object.get("channel"), &format!("{label}.channel"))?; + let domain = + required_nonempty_string_field(object.get("domain"), &format!("{label}.domain"))?; + let weight = object + .get("weight") + .map(|raw| { + raw.as_f64() + .filter(|value| value.is_finite() && *value > 0.0) + .ok_or_else(|| format!("{label}.weight must be finite and > 0")) + }) + .transpose()? + .unwrap_or(1.0); + Ok(LoweredCausalEvent::Target { + channel, + domain, + bytes: event_payload_bytes(value, label)?, + weight, + }) + } + other => Err(format!( + "{label}.kind has unknown causal event kind '{other}'" + )), + } +} + +fn event_payload_bytes(value: &Value, label: &str) -> Result, String> { + let object = value + .as_object() + .ok_or_else(|| format!("{label} must be an object"))?; + let payload = object + .get("bytes") + .ok_or_else(|| format!("{label}.bytes is required"))?; + payload_bytes(payload, &format!("{label}.bytes")) +} + +fn payload_bytes(value: &Value, label: &str) -> Result, String> { + match value { + Value::String(text) => Ok(text.as_bytes().to_vec()), + Value::Array(items) => items + .iter() + .enumerate() + .map(|(index, item)| { + let byte = item + .as_u64() + .ok_or_else(|| format!("{label}[{index}] must be an integer byte"))?; + u8::try_from(byte).map_err(|_| format!("{label}[{index}] must be in 0..=255")) + }) + .collect(), + _ => Err(format!( + "{label} must be either a byte string or an array of integer bytes" + )), + } +} + +fn required_nonempty_string_field(value: Option<&Value>, label: &str) -> Result { + let text = value + .and_then(Value::as_str) + .map(str::trim) + .filter(|text| !text.is_empty()) + .ok_or_else(|| format!("{label} is required"))?; + Ok(text.to_string()) +} + +fn ensure_known_event_fields( + object: &serde_json::Map, + allowed: &[&str], + label: &str, +) -> Result<(), String> { + for key in object.keys() { + if !allowed.contains(&key.as_str()) { + return Err(format!("{label} contains unknown event field '{key}'")); + } + } + Ok(()) +} + +fn is_lower_hex_crc32(value: &str) -> bool { + value.len() == 8 + && value + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) +} + +fn lowered_event_skeleton_bytes(events: &[LoweredCausalEvent]) -> Result, String> { + let skeleton = events + .iter() + .map(|event| match event { + LoweredCausalEvent::Reset => serde_json::json!({"kind": "reset"}), + LoweredCausalEvent::Context { channel, bytes } => serde_json::json!({ + "kind": "context", + "channel": channel, + "bytes_len": bytes.len(), + }), + LoweredCausalEvent::ObserveTargetNoScore { + channel, + domain, + bytes, + } => serde_json::json!({ + "kind": "observe_target_no_score", + "channel": channel, + "domain": domain, + "bytes_len": bytes.len(), + }), + LoweredCausalEvent::Target { + channel, + domain, + bytes, + weight, + } => serde_json::json!({ + "kind": "target", + "channel": channel, + "domain": domain, + "bytes_len": bytes.len(), + "weight_bits": weight.to_bits(), + }), + }) + .collect::>(); + canonical_json_bytes(&serde_json::Value::Array(skeleton)) + .map_err(|err| format!("failed to serialize lowered event skeleton: {err}")) +} + +fn causal_domain_support_bytes( + domains: &BTreeMap, +) -> Result, String> { + let value = domains + .iter() + .map(|(domain, support)| match support { + CausalTargetDomain::ByteAlphabet => serde_json::json!({ + "domain": domain, + "kind": "byte_alphabet", + "symbol_width_bytes": 1usize, + "symbols": 256usize, + }), + CausalTargetDomain::EnumeratedPayloads { payloads } => serde_json::json!({ + "domain": domain, + "kind": "enumerated_payloads", + "payloads": payloads, + }), + }) + .collect::>(); + canonical_json_bytes(&serde_json::Value::Array(value)) + .map_err(|err| format!("failed to serialize causal target-domain supports: {err}")) +} + +fn causal_channel_set(events: &[LoweredCausalEvent]) -> BTreeSet { + let mut channels = BTreeSet::::new(); + for event in events { + match event { + LoweredCausalEvent::Reset => {} + LoweredCausalEvent::Context { channel, .. } + | LoweredCausalEvent::ObserveTargetNoScore { channel, .. } + | LoweredCausalEvent::Target { channel, .. } => { + channels.insert(channel.clone()); + } + } + } + channels +} + +fn validate_events_against_causal_profile( + events: &[LoweredCausalEvent], + domains: &BTreeMap, + header: &CausalHeaderProfile, +) -> Result<(), String> { + if header.action_alphabet_size > 256 && header.event_grammar.context_channels.contains("action") + { + return Err( + "event_grammar contains action context but action_alphabet.size exceeds byte encoding capacity (must be <= 256)" + .to_string(), + ); + } + for event in events { + match event { + LoweredCausalEvent::Reset => {} + LoweredCausalEvent::Context { channel, bytes } => { + if !header.event_grammar.context_channels.contains(channel) { + return Err(format!( + "causal context event channel '{channel}' is not declared in event_grammar.context_channels" + )); + } + if channel == "action" { + if bytes.len() != 1 { + return Err( + "action context payload must encode exactly one byte".to_string() + ); + } + if bytes[0] as usize >= header.action_alphabet_size { + return Err(format!( + "action context value {} is outside action_alphabet.size={}", + bytes[0], header.action_alphabet_size + )); + } + } + } + LoweredCausalEvent::ObserveTargetNoScore { domain, bytes, .. } + | LoweredCausalEvent::Target { domain, bytes, .. } => { + let support = domains.get(domain).ok_or_else(|| { + format!("causal event references undeclared target domain '{domain}'") + })?; + let descriptor = match event { + LoweredCausalEvent::ObserveTargetNoScore { + channel, domain, .. + } => CausalChannelDomain { + channel: channel.clone(), + domain: domain.clone(), + }, + LoweredCausalEvent::Target { + channel, domain, .. + } => CausalChannelDomain { + channel: channel.clone(), + domain: domain.clone(), + }, + _ => unreachable!(), + }; + match event { + LoweredCausalEvent::ObserveTargetNoScore { .. } => { + if !header + .event_grammar + .observe_target_no_score + .contains(&descriptor) + { + return Err(format!( + "observe_target_no_score ({}, {}) is not declared in event_grammar.observe_target_no_score", + descriptor.channel, descriptor.domain + )); + } + } + LoweredCausalEvent::Target { .. } => { + if !header.event_grammar.target.contains(&descriptor) { + return Err(format!( + "target ({}, {}) is not declared in event_grammar.target", + descriptor.channel, descriptor.domain + )); + } + } + _ => {} + } + if !causal_support_contains(support, bytes) { + return Err(format!( + "causal event payload is outside target-domain support '{domain}'" + )); + } + } + } + } + Ok(()) +} + +fn causal_support_contains(support: &CausalTargetDomain, bytes: &[u8]) -> bool { + match support { + CausalTargetDomain::ByteAlphabet => bytes.len() == 1, + CausalTargetDomain::EnumeratedPayloads { payloads } => { + payloads.iter().any(|payload| payload == bytes) + } + } +} + +fn expand_byte_alphabet_events( + events: Vec, + domains: &BTreeMap, +) -> Result, String> { + let mut expanded = Vec::::new(); + for event in events { + match event { + LoweredCausalEvent::Reset | LoweredCausalEvent::Context { .. } => expanded.push(event), + LoweredCausalEvent::ObserveTargetNoScore { + channel, + domain, + bytes, + } => { + let Some(support) = domains.get(&domain) else { + return Err(format!( + "causal event references undeclared target domain '{domain}'" + )); + }; + match support { + CausalTargetDomain::ByteAlphabet => { + if bytes.is_empty() { + return Err( + "byte_alphabet payloads must contain at least one byte".to_string() + ); + } + for byte in bytes { + expanded.push(LoweredCausalEvent::ObserveTargetNoScore { + channel: channel.clone(), + domain: domain.clone(), + bytes: vec![byte], + }); + } + } + CausalTargetDomain::EnumeratedPayloads { .. } => { + expanded.push(LoweredCausalEvent::ObserveTargetNoScore { + channel, + domain, + bytes, + }); + } + } + } + LoweredCausalEvent::Target { + channel, + domain, + bytes, + weight, + } => { + let Some(support) = domains.get(&domain) else { + return Err(format!( + "causal event references undeclared target domain '{domain}'" + )); + }; + match support { + CausalTargetDomain::ByteAlphabet => { + if bytes.is_empty() { + return Err( + "byte_alphabet payloads must contain at least one byte".to_string() + ); + } + for byte in bytes { + expanded.push(LoweredCausalEvent::Target { + channel: channel.clone(), + domain: domain.clone(), + bytes: vec![byte], + weight, + }); + } + } + CausalTargetDomain::EnumeratedPayloads { .. } => { + expanded.push(LoweredCausalEvent::Target { + channel, + domain, + bytes, + weight, + }); + } + } + } + } + } + Ok(expanded) +} + +fn causal_dataset_string_field(value: &Value, field: &str) -> Option { + value + .get(field) + .and_then(Value::as_str) + .map(str::trim) + .filter(|text| !text.is_empty()) + .map(ToOwned::to_owned) +} diff --git a/crates/infotheory/src/tuner/certificates.rs b/crates/infotheory/src/tuner/certificates.rs new file mode 100644 index 00000000..55cb224b --- /dev/null +++ b/crates/infotheory/src/tuner/certificates.rs @@ -0,0 +1,943 @@ +use super::*; + +impl VerifiedTheoremInputs { + pub(super) fn load( + theorem: &TuneTheoremConfig, + compiled: &crate::spec::CompiledTuneSpec, + dataset: &LoadedDataset, + evaluator_profile: &EvaluatorProfile, + config_dir: &Path, + ) -> Result { + let bounds_digest = bounds_hash(&compiled.canonical_spec().bounds)?; + let controller_kind = controller_kind_name(compiled.controller()); + let scalar_representation = theorem + .scalar_representation_ref + .as_deref() + .unwrap_or(SCALAR_REPRESENTATION_DECLARATION); + let mut verified = Self::default(); + + verified.finite_planner_state = load_generic_certificate( + theorem.finite_planner_state_certificate.as_deref(), + "finite_planner_state", + compiled, + dataset, + evaluator_profile, + config_dir, + &bounds_digest, + controller_kind, + )?; + verified.no_hidden_state = load_generic_certificate( + theorem.no_hidden_state_certificate.as_deref(), + "no_hidden_state", + compiled, + dataset, + evaluator_profile, + config_dir, + &bounds_digest, + controller_kind, + )?; + verified.determinism_deadline = load_generic_certificate( + theorem.determinism_deadline_certificate.as_deref(), + "determinism_deadline", + compiled, + dataset, + evaluator_profile, + config_dir, + &bounds_digest, + controller_kind, + )?; + verified.exact_state_observation = load_exact_state_observation_certificate( + theorem, + verified.finite_planner_state.as_ref(), + compiled, + dataset, + evaluator_profile, + config_dir, + &bounds_digest, + controller_kind, + )?; + verified.exact_reward_encoding = load_exact_reward_certificate( + theorem.exact_reward_encoding_certificate.as_deref(), + compiled, + dataset, + evaluator_profile, + config_dir, + &bounds_digest, + controller_kind, + scalar_representation, + )?; + verified.deterministic_table = load_deterministic_evaluator_table( + theorem.deterministic_evaluator_table.as_deref(), + compiled, + dataset, + evaluator_profile, + config_dir, + &bounds_digest, + controller_kind, + )?; + Ok(verified) + } + + pub(super) fn timing_certified(&self, theorem: &TuneTheoremConfig) -> bool { + match theorem.timing_certification_tier { + TimingCertificationTier::RealTime => self.determinism_deadline.is_some(), + TimingCertificationTier::DeterministicTable => self.deterministic_table.is_some(), + TimingCertificationTier::BestEffort | TimingCertificationTier::Isolated => false, + } + } + + pub(super) fn to_json_value(&self) -> Value { + serde_json::json!({ + "finite_planner_state": self.finite_planner_state.as_ref().map(VerifiedCertificate::to_json_value), + "no_hidden_state": self.no_hidden_state.as_ref().map(VerifiedCertificate::to_json_value), + "exact_reward_encoding": self.exact_reward_encoding.as_ref().map(VerifiedExactRewardEncodingCertificate::to_json_value), + "exact_state_observation": self.exact_state_observation.as_ref().map(VerifiedExactStateObservationCertificate::to_json_value), + "determinism_deadline": self.determinism_deadline.as_ref().map(VerifiedCertificate::to_json_value), + "deterministic_evaluator_table": self.deterministic_table.as_ref().map(VerifiedDeterministicEvaluatorTable::to_json_value), + }) + } +} + +impl VerifiedCertificate { + pub(super) fn to_json_value(&self) -> Value { + serde_json::json!({ + "ref": self.ref_value, + "content_crc32": self.content_hash, + "verified": true, + }) + } +} + +impl VerifiedExactRewardEncodingCertificate { + pub(super) fn to_json_value(&self) -> Value { + serde_json::json!({ + "ref": self.base.ref_value, + "content_crc32": self.base.content_hash, + "verified": true, + "scalar_representation": self.scalar_representation, + "reward_bits": self.reward_bits, + "max_reward": self.max_reward, + "encoding": match &self.mode { + VerifiedRewardEncodingMode::IntegerObjectiveDifferenceInterval => "integer_objective_difference", + VerifiedRewardEncodingMode::FiniteRewardMap { .. } => "finite_reward_map", + }, + "finite_reward_values": match &self.mode { + VerifiedRewardEncodingMode::IntegerObjectiveDifferenceInterval => None, + VerifiedRewardEncodingMode::FiniteRewardMap { map } => Some(map.objective_difference_to_symbol.len()), + }, + "complete_nonnegative_interval_max": match &self.mode { + VerifiedRewardEncodingMode::IntegerObjectiveDifferenceInterval => None, + VerifiedRewardEncodingMode::FiniteRewardMap { map } => map.complete_nonnegative_interval_max, + }, + }) + } + + pub(super) fn is_identity_or_interval_encoding(&self) -> bool { + match &self.mode { + VerifiedRewardEncodingMode::IntegerObjectiveDifferenceInterval => true, + VerifiedRewardEncodingMode::FiniteRewardMap { map } => map + .objective_difference_to_symbol + .iter() + .all(|(objective_difference, symbol)| objective_difference == symbol), + } + } +} + +impl VerifiedExactStateObservationCertificate { + pub(super) fn to_json_value(&self) -> Value { + serde_json::json!({ + "ref": self.base.ref_value, + "content_crc32": self.base.content_hash, + "verified": true, + "observation_key_mode": self.observation_key_mode, + "exact_state_encoder_spec_ref": self.exact_state_encoder_spec_ref, + "observation_adapter_spec_ref": self.observation_adapter_spec_ref, + "observation_adapter_content_crc32": self.observation_adapter_content_hash, + "finite_planner_state_certificate_crc32": self.finite_planner_state_certificate_hash, + "finite_state_count": self.finite_state_count, + }) + } +} + +impl VerifiedDeterministicEvaluatorTable { + pub(super) fn to_json_value(&self) -> Value { + serde_json::json!({ + "ref": self.base.ref_value, + "content_crc32": self.base.content_hash, + "verified": true, + "rows": self.rows.len(), + }) + } + + pub(super) fn evaluate( + &self, + candidate: &crate::spec::CompiledCompressionBackend, + dataset: &LoadedDataset, + model_bytes: usize, + min_throughput_bytes_per_second: f64, + max_memory_bytes: u64, + effective_eval_time_limit_seconds: f64, + ) -> Result { + let candidate_crc32 = crc32_hex(candidate.canonical_bytes().as_slice()); + let row = self.rows.get(&candidate_crc32).ok_or_else(|| { + format!( + "deterministic evaluator table missing row for candidate_crc32 '{candidate_crc32}'" + ) + })?; + if row.status == CandidateEvalStatus::Timeout { + return Ok(timeout_eval_result( + row.elapsed_seconds, + row.peak_memory_bytes, + effective_eval_time_limit_seconds, + )); + } + if row.status != CandidateEvalStatus::Success { + return Ok(CandidateEvalResult { + status: row.status, + compressed_bytes: row.compressed_bytes, + elapsed_seconds: row.elapsed_seconds, + effective_eval_time_limit_seconds, + throughput_bytes_per_second: 0.0, + peak_memory_bytes: row.peak_memory_bytes, + target_loss_bits: f64::INFINITY, + objective_bits: f64::INFINITY, + deployable: false, + }); + } + if row.elapsed_seconds >= effective_eval_time_limit_seconds { + return Ok(timeout_eval_result( + row.elapsed_seconds, + row.peak_memory_bytes, + effective_eval_time_limit_seconds, + )); + } + let throughput_bytes_per_second = if row.elapsed_seconds <= 0.0 { + f64::INFINITY + } else { + dataset.dataset_units / row.elapsed_seconds + }; + let deployable = throughput_bytes_per_second >= min_throughput_bytes_per_second + && row.peak_memory_bytes <= max_memory_bytes; + let objective_bits = if deployable { + ((model_bytes as f64) * 8.0) + row.target_loss_bits + } else { + f64::INFINITY + }; + + Ok(CandidateEvalResult { + status: row.status, + compressed_bytes: row.compressed_bytes, + elapsed_seconds: row.elapsed_seconds, + effective_eval_time_limit_seconds, + throughput_bytes_per_second, + peak_memory_bytes: row.peak_memory_bytes, + target_loss_bits: row.target_loss_bits, + objective_bits, + deployable, + }) + } +} + +#[allow(clippy::too_many_arguments)] +pub(super) fn load_generic_certificate( + reference: Option<&str>, + expected_kind: &str, + compiled: &crate::spec::CompiledTuneSpec, + dataset: &LoadedDataset, + evaluator_profile: &EvaluatorProfile, + config_dir: &Path, + bounds_digest: &str, + controller_kind: &str, +) -> Result, String> { + let Some((ref_value, value, content_hash)) = load_certificate_value(reference, config_dir)? + else { + return Ok(None); + }; + validate_certificate_common( + &value, + expected_kind, + compiled, + dataset, + evaluator_profile, + bounds_digest, + controller_kind, + )?; + Ok(Some(VerifiedCertificate { + ref_value, + content_hash, + })) +} + +#[allow(clippy::too_many_arguments)] +pub(super) fn load_exact_reward_certificate( + reference: Option<&str>, + compiled: &crate::spec::CompiledTuneSpec, + dataset: &LoadedDataset, + evaluator_profile: &EvaluatorProfile, + config_dir: &Path, + bounds_digest: &str, + controller_kind: &str, + scalar_representation: &str, +) -> Result, String> { + let Some((ref_value, value, content_hash)) = load_certificate_value(reference, config_dir)? + else { + return Ok(None); + }; + validate_certificate_common( + &value, + "exact_reward_encoding", + compiled, + dataset, + evaluator_profile, + bounds_digest, + controller_kind, + )?; + let object = certificate_object(&value, "exact_reward_encoding")?; + let encoding = required_cert_str(object, "encoding", "exact_reward_encoding")?; + if encoding != "integer_objective_difference" && encoding != "finite_reward_map" { + return Err(format!( + "exact_reward_encoding certificate uses unsupported encoding '{encoding}'" + )); + } + let cert_scalar = required_cert_str(object, "scalar_representation", "exact_reward_encoding")?; + if cert_scalar != scalar_representation { + return Err(format!( + "exact_reward_encoding certificate scalar_representation '{cert_scalar}' does not match requested '{scalar_representation}'" + )); + } + let reward_bits = required_cert_u64(object, "reward_bits", "exact_reward_encoding")?; + let reward_bits = usize::try_from(reward_bits) + .map_err(|_| "exact_reward_encoding.reward_bits does not fit usize".to_string())?; + let max_reward = required_cert_u64(object, "max_reward", "exact_reward_encoding")?; + let max_reward = Reward::try_from(max_reward) + .map_err(|_| "exact_reward_encoding.max_reward does not fit Reward".to_string())?; + let max_encoded = max_nonnegative_reward_for_bits(reward_bits)?; + if max_reward > max_encoded { + return Err(format!( + "exact_reward_encoding.max_reward {max_reward} exceeds reward_bits={reward_bits} maximum {max_encoded}" + )); + } + let mode = if encoding == "integer_objective_difference" { + VerifiedRewardEncodingMode::IntegerObjectiveDifferenceInterval + } else { + let map = parse_finite_reward_map(object, reward_bits, max_reward)?; + if compiled + .controller() + .exact_objective_difference_controller() + .is_some() + && map.complete_nonnegative_interval_max.is_none() + { + return Err( + "exact_reward_encoding finite_reward_map certificates for exact-objective controllers must declare complete_nonnegative_interval_max" + .to_string(), + ); + } + VerifiedRewardEncodingMode::FiniteRewardMap { map } + }; + Ok(Some(VerifiedExactRewardEncodingCertificate { + base: VerifiedCertificate { + ref_value, + content_hash, + }, + max_reward, + reward_bits, + scalar_representation: cert_scalar.to_string(), + mode, + })) +} + +pub(super) fn parse_finite_reward_map( + object: &serde_json::Map, + reward_bits: usize, + declared_max_reward: Reward, +) -> Result { + let entries = object + .get("values") + .and_then(Value::as_array) + .ok_or_else(|| { + "exact_reward_encoding finite_reward_map requires a 'values' array".to_string() + })?; + if entries.is_empty() { + return Err("exact_reward_encoding finite_reward_map values must be nonempty".to_string()); + } + let max_encoded = max_nonnegative_reward_for_bits(reward_bits)?; + let mut by_difference = BTreeMap::::new(); + let mut seen_symbols = BTreeSet::::new(); + for (index, entry) in entries.iter().enumerate() { + let entry_object = entry + .as_object() + .ok_or_else(|| format!("exact_reward_encoding.values[{index}] must be an object"))?; + let difference = required_cert_u64( + entry_object, + "objective_difference", + &format!("exact_reward_encoding.values[{index}]"), + )?; + let difference = Reward::try_from(difference).map_err(|_| { + format!( + "exact_reward_encoding.values[{index}].objective_difference does not fit Reward" + ) + })?; + let symbol = required_cert_u64( + entry_object, + "symbol", + &format!("exact_reward_encoding.values[{index}]"), + )?; + let symbol = Reward::try_from(symbol).map_err(|_| { + format!("exact_reward_encoding.values[{index}].symbol does not fit Reward") + })?; + if symbol > max_encoded { + return Err(format!( + "exact_reward_encoding.values[{index}].symbol {symbol} exceeds reward_bits={reward_bits} maximum {max_encoded}" + )); + } + if symbol > declared_max_reward { + return Err(format!( + "exact_reward_encoding.values[{index}].symbol {symbol} exceeds declared max_reward {declared_max_reward}" + )); + } + if by_difference.insert(difference, symbol).is_some() { + return Err(format!( + "exact_reward_encoding finite_reward_map duplicates objective_difference {difference}" + )); + } + if !seen_symbols.insert(symbol) { + return Err(format!( + "exact_reward_encoding finite_reward_map duplicates reward symbol {symbol}" + )); + } + } + let complete_nonnegative_interval_max = object + .get("complete_nonnegative_interval_max") + .map(|value| { + value.as_u64().ok_or_else(|| { + "exact_reward_encoding.complete_nonnegative_interval_max must be an unsigned integer" + .to_string() + }) + }) + .transpose()? + .map(|max| { + Reward::try_from(max).map_err(|_| { + "exact_reward_encoding.complete_nonnegative_interval_max does not fit Reward" + .to_string() + }) + }) + .transpose()?; + if let Some(complete_max) = complete_nonnegative_interval_max { + if complete_max > declared_max_reward { + return Err(format!( + "exact_reward_encoding.complete_nonnegative_interval_max {complete_max} exceeds declared max_reward {declared_max_reward}" + )); + } + validate_complete_finite_reward_interval(&by_difference, complete_max)?; + } + Ok(VerifiedFiniteRewardMap { + objective_difference_to_symbol: by_difference, + complete_nonnegative_interval_max, + }) +} + +pub(super) fn validate_complete_finite_reward_interval( + objective_difference_to_symbol: &BTreeMap, + complete_max: Reward, +) -> Result<(), String> { + for objective_difference in 0..=complete_max { + if !objective_difference_to_symbol.contains_key(&objective_difference) { + return Err(format!( + "exact_reward_encoding finite_reward_map complete interval is missing objective_difference {objective_difference}" + )); + } + } + Ok(()) +} + +#[allow(clippy::too_many_arguments)] +pub(super) fn load_exact_state_observation_certificate( + theorem: &TuneTheoremConfig, + finite_planner_state: Option<&VerifiedCertificate>, + compiled: &crate::spec::CompiledTuneSpec, + dataset: &LoadedDataset, + evaluator_profile: &EvaluatorProfile, + config_dir: &Path, + bounds_digest: &str, + controller_kind: &str, +) -> Result, String> { + let Some((ref_value, value, content_hash)) = load_certificate_value( + theorem.exact_state_observation_certificate.as_deref(), + config_dir, + )? + else { + return Ok(None); + }; + validate_certificate_common( + &value, + "exact_state_observation", + compiled, + dataset, + evaluator_profile, + bounds_digest, + controller_kind, + )?; + let object = certificate_object(&value, "exact_state_observation")?; + let observation_key_mode = + required_cert_str(object, "observation_key_mode", "exact_state_observation")?; + let requested_adapter = theorem + .observation_adapter_spec_ref + .as_deref() + .unwrap_or(OBSERVATION_ADAPTER_DECLARATION); + let observed_adapter = required_cert_str( + object, + "observation_adapter_spec_ref", + "exact_state_observation", + )?; + if observed_adapter != requested_adapter { + return Err(format!( + "exact_state_observation certificate observation_adapter_spec_ref '{observed_adapter}' does not match requested '{requested_adapter}'" + )); + } + require_cert_string_match( + object, + "observation_adapter_content_crc32", + &observation_adapter_content_hash()?, + "exact_state_observation", + )?; + let finite_planner_state = finite_planner_state.ok_or_else(|| { + "exact_state_observation certificate requires a verified finite_planner_state_certificate" + .to_string() + })?; + require_cert_string_match( + object, + "finite_planner_state_certificate_crc32", + &finite_planner_state.content_hash, + "exact_state_observation", + )?; + let observed_encoder = required_cert_str( + object, + "exact_state_encoder_spec_ref", + "exact_state_observation", + )?; + if let Some(expected) = theorem.exact_state_encoder_spec_ref.as_deref() + && observed_encoder != expected + { + return Err(format!( + "exact_state_observation certificate encoder ref '{observed_encoder}' does not match requested '{expected}'" + )); + } + let interface = planner_interface_for_controller(compiled.controller()).ok_or_else(|| { + "exact_state_observation certificate can only be validated for planner-family controllers" + .to_string() + })?; + let mode_matches = match interface.observation_key_mode { + crate::aixi::common::ObservationKeyMode::FullStream => { + observation_key_mode == "full_stream" + } + crate::aixi::common::ObservationKeyMode::First => { + observation_key_mode == "first" || observation_key_mode == "first_symbol" + } + crate::aixi::common::ObservationKeyMode::Last => { + observation_key_mode == "last" || observation_key_mode == "last_symbol" + } + crate::aixi::common::ObservationKeyMode::StreamHash => { + observation_key_mode == "stream_hash" + } + }; + if !mode_matches { + return Err(format!( + "exact_state_observation certificate observation_key_mode '{observation_key_mode}' is not compatible with the controller's configured projection" + )); + } + let finite_state_count = + validate_exact_state_observation_artifact(object, observation_key_mode, interface)?; + Ok(Some(VerifiedExactStateObservationCertificate { + base: VerifiedCertificate { + ref_value, + content_hash, + }, + observation_key_mode: observation_key_mode.to_string(), + exact_state_encoder_spec_ref: observed_encoder.to_string(), + observation_adapter_spec_ref: observed_adapter.to_string(), + observation_adapter_content_hash: observation_adapter_content_hash()?, + finite_planner_state_certificate_hash: finite_planner_state.content_hash.clone(), + finite_state_count, + })) +} + +fn planner_interface_for_controller( + controller: &crate::spec::CompiledTuneController, +) -> Option<&crate::spec::TunePlannerInterfaceSpec> { + match controller { + crate::spec::CompiledTuneController::McAixiFacCtw(inner) => Some(&inner.interface), + crate::spec::CompiledTuneController::AiqiDiscounted(inner) => Some(&inner.interface), + crate::spec::CompiledTuneController::AiqiWarmstartExactJh(inner) => Some(&inner.interface), + crate::spec::CompiledTuneController::AnnealedHillClimbing(_) => None, + } +} + +pub(super) fn validate_exact_state_observation_artifact( + object: &serde_json::Map, + observation_key_mode: &str, + interface: &crate::spec::TunePlannerInterfaceSpec, +) -> Result { + let states = object + .get("psi_h_outputs") + .or_else(|| object.get("state_outputs")) + .and_then(Value::as_array) + .ok_or_else(|| { + "exact_state_observation certificate requires a checkable psi_h_outputs array" + .to_string() + })?; + if states.is_empty() { + return Err("exact_state_observation psi_h_outputs must be nonempty".to_string()); + } + let stream_len = interface.observation_stream_len.max(1); + let max_symbol = max_observation_symbol_for_bits(interface.observation_bits)?; + let mut seen_state_ids = BTreeSet::::new(); + let mut seen_projected_outputs = BTreeSet::>::new(); + for (index, state_value) in states.iter().enumerate() { + let state_object = state_value.as_object().ok_or_else(|| { + format!("exact_state_observation.psi_h_outputs[{index}] must be an object") + })?; + let state_id = required_cert_str( + state_object, + "state_id", + &format!("exact_state_observation.psi_h_outputs[{index}]"), + )?; + if !seen_state_ids.insert(state_id.to_string()) { + return Err(format!( + "exact_state_observation duplicate state_id '{state_id}'" + )); + } + let observations = state_object + .get("observations") + .and_then(Value::as_array) + .ok_or_else(|| { + format!( + "exact_state_observation.psi_h_outputs[{index}].observations must be an array" + ) + })?; + if observations.len() != stream_len { + return Err(format!( + "exact_state_observation.psi_h_outputs[{index}].observations length {} does not match observation_stream_len {stream_len}", + observations.len() + )); + } + let mut output = Vec::::with_capacity(stream_len); + for (symbol_index, symbol_value) in observations.iter().enumerate() { + let symbol = symbol_value.as_u64().ok_or_else(|| { + format!( + "exact_state_observation.psi_h_outputs[{index}].observations[{symbol_index}] must be an integer" + ) + })?; + if symbol > max_symbol { + return Err(format!( + "exact_state_observation.psi_h_outputs[{index}].observations[{symbol_index}]={symbol} exceeds observation_bits={} maximum {max_symbol}", + interface.observation_bits + )); + } + output.push(symbol); + } + let projected = + project_observation_output(observation_key_mode, &output, interface.observation_bits)?; + if !seen_projected_outputs.insert(projected) { + return Err(format!( + "exact_state_observation psi_h_outputs are not injective under observation_key_mode '{observation_key_mode}'" + )); + } + } + Ok(states.len()) +} + +pub(super) fn project_observation_output( + observation_key_mode: &str, + output: &[PerceptVal], + observation_bits: usize, +) -> Result, String> { + match observation_key_mode { + "full_stream" => Ok(output.to_vec()), + "first_symbol" | "first" => output + .first() + .copied() + .map(|value| vec![value]) + .ok_or_else(|| "observation stream cannot be empty".to_string()), + "last_symbol" | "last" => output + .last() + .copied() + .map(|value| vec![value]) + .ok_or_else(|| "observation stream cannot be empty".to_string()), + "stream_hash" => Ok(vec![crate::aixi::common::observation_key_from_stream( + ObservationKeyMode::StreamHash, + output, + observation_bits, + )]), + other => Err(format!( + "exact_state_observation certificate uses unsupported observation_key_mode '{other}'" + )), + } +} + +pub(super) fn max_observation_symbol_for_bits( + observation_bits: usize, +) -> Result { + if observation_bits == 0 { + Ok(0) + } else if observation_bits >= 64 { + Ok(PerceptVal::MAX) + } else { + Ok((1_u64 << observation_bits) - 1) + } +} + +#[allow(clippy::too_many_arguments)] +pub(super) fn load_deterministic_evaluator_table( + reference: Option<&str>, + compiled: &crate::spec::CompiledTuneSpec, + dataset: &LoadedDataset, + evaluator_profile: &EvaluatorProfile, + config_dir: &Path, + bounds_digest: &str, + controller_kind: &str, +) -> Result, String> { + let Some((ref_value, value, content_hash)) = load_certificate_value(reference, config_dir)? + else { + return Ok(None); + }; + validate_certificate_common( + &value, + "deterministic_evaluator_table", + compiled, + dataset, + evaluator_profile, + bounds_digest, + controller_kind, + )?; + let object = certificate_object(&value, "deterministic_evaluator_table")?; + let rows_value = object + .get("rows") + .and_then(Value::as_array) + .ok_or_else(|| "deterministic_evaluator_table.rows must be an array".to_string())?; + let mut rows = HashMap::::new(); + for (index, row_value) in rows_value.iter().enumerate() { + let row_object = row_value.as_object().ok_or_else(|| { + format!("deterministic_evaluator_table.rows[{index}] must be an object") + })?; + let candidate_crc32 = required_cert_str( + row_object, + "candidate_crc32", + &format!("deterministic_evaluator_table.rows[{index}]"), + )?; + let status = match required_cert_str( + row_object, + "status", + &format!("deterministic_evaluator_table.rows[{index}]"), + )? { + "success" => CandidateEvalStatus::Success, + "timeout" => CandidateEvalStatus::Timeout, + "invalid" => CandidateEvalStatus::Invalid, + "error" => CandidateEvalStatus::Error, + other => { + return Err(format!( + "deterministic_evaluator_table.rows[{index}].status has unknown status '{other}'" + )); + } + }; + let compressed_bytes = required_cert_u64( + row_object, + "compressed_bytes", + &format!("deterministic_evaluator_table.rows[{index}]"), + )?; + let compressed_bytes = usize::try_from(compressed_bytes).map_err(|_| { + format!( + "deterministic_evaluator_table.rows[{index}].compressed_bytes does not fit usize" + ) + })?; + let target_loss_bits = required_cert_f64( + row_object, + "target_loss_bits", + &format!("deterministic_evaluator_table.rows[{index}]"), + )?; + let elapsed_seconds = required_cert_f64( + row_object, + "elapsed_seconds", + &format!("deterministic_evaluator_table.rows[{index}]"), + )?; + let peak_memory_bytes = required_cert_u64( + row_object, + "peak_memory_bytes", + &format!("deterministic_evaluator_table.rows[{index}]"), + )?; + if rows + .insert( + candidate_crc32.to_string(), + DeterministicEvaluatorRow { + status, + compressed_bytes, + target_loss_bits, + elapsed_seconds, + peak_memory_bytes, + }, + ) + .is_some() + { + return Err(format!( + "deterministic_evaluator_table duplicate candidate_crc32 '{candidate_crc32}'" + )); + } + } + if rows.is_empty() { + return Err("deterministic_evaluator_table.rows must not be empty".to_string()); + } + Ok(Some(VerifiedDeterministicEvaluatorTable { + base: VerifiedCertificate { + ref_value, + content_hash, + }, + rows, + })) +} + +fn load_certificate_value( + reference: Option<&str>, + config_dir: &Path, +) -> Result, String> { + let Some(raw_reference) = reference else { + return Ok(None); + }; + let raw_ref = raw_reference.trim(); + if raw_ref.is_empty() { + return Err("theorem certificate reference must be non-empty when set".to_string()); + }; + if raw_ref.contains("://") && !raw_ref.starts_with("file://") { + return Err(format!( + "unsupported theorem certificate reference scheme in '{raw_ref}'; use a filesystem path or file:// URI" + )); + } + let path_text = raw_ref.strip_prefix("file://").unwrap_or(raw_ref); + let path = Path::new(path_text); + let resolved = if path.is_absolute() { + path.to_path_buf() + } else { + config_dir.join(path) + }; + let raw = fs::read(&resolved).map_err(|err| { + format!( + "failed to read theorem certificate '{}': {err}", + resolved.display() + ) + })?; + let value: Value = serde_json::from_slice(&raw).map_err(|err| { + format!( + "invalid theorem certificate JSON '{}': {err}", + resolved.display() + ) + })?; + Ok(Some((raw_ref.to_string(), value, crc32_hex(&raw)))) +} + +#[allow(clippy::too_many_arguments)] +fn validate_certificate_common( + value: &Value, + expected_kind: &str, + compiled: &crate::spec::CompiledTuneSpec, + dataset: &LoadedDataset, + evaluator_profile: &EvaluatorProfile, + bounds_digest: &str, + controller_kind: &str, +) -> Result<(), String> { + let object = certificate_object(value, expected_kind)?; + let schema_version = required_cert_u64(object, "schema_version", expected_kind)?; + if schema_version != 1 { + return Err(format!( + "{expected_kind} certificate schema_version must be 1, got {schema_version}" + )); + } + let kind = required_cert_str(object, "kind", expected_kind)?; + if kind != expected_kind { + return Err(format!("{expected_kind} certificate has kind '{kind}'")); + } + require_cert_string_match( + object, + "dataset_crc32", + &dataset.canonical_content_hash, + expected_kind, + )?; + require_cert_string_match(object, "bounds_crc32", bounds_digest, expected_kind)?; + require_cert_string_match( + object, + "evaluator_profile_crc32", + &evaluator_profile.hash()?, + expected_kind, + )?; + require_cert_string_match(object, "controller_kind", controller_kind, expected_kind)?; + if let Some(action_alphabet_size) = object.get("action_alphabet_size") { + let expected = planner_action_count(compiled)?; + let observed = action_alphabet_size + .as_u64() + .ok_or_else(|| format!("{expected_kind}.action_alphabet_size must be an integer"))?; + if observed != expected as u64 { + return Err(format!( + "{expected_kind}.action_alphabet_size {observed} does not match compiled {expected}" + )); + } + } + Ok(()) +} + +fn certificate_object<'a>( + value: &'a Value, + label: &str, +) -> Result<&'a serde_json::Map, String> { + value + .as_object() + .ok_or_else(|| format!("{label} certificate must be a JSON object")) +} + +fn required_cert_str<'a>( + object: &'a serde_json::Map, + field: &str, + label: &str, +) -> Result<&'a str, String> { + object + .get(field) + .and_then(Value::as_str) + .ok_or_else(|| format!("{label}.{field} must be a string")) +} + +fn required_cert_u64( + object: &serde_json::Map, + field: &str, + label: &str, +) -> Result { + object + .get(field) + .and_then(Value::as_u64) + .ok_or_else(|| format!("{label}.{field} must be an unsigned integer")) +} + +fn required_cert_f64( + object: &serde_json::Map, + field: &str, + label: &str, +) -> Result { + let value = object + .get(field) + .and_then(Value::as_f64) + .ok_or_else(|| format!("{label}.{field} must be a finite number"))?; + if !value.is_finite() || value < 0.0 { + return Err(format!("{label}.{field} must be finite and nonnegative")); + } + Ok(value) +} + +fn require_cert_string_match( + object: &serde_json::Map, + field: &str, + expected: &str, + label: &str, +) -> Result<(), String> { + let observed = required_cert_str(object, field, label)?; + if observed != expected { + return Err(format!( + "{label}.{field} '{observed}' does not match expected '{expected}'" + )); + } + Ok(()) +} diff --git a/crates/infotheory/src/tuner/config.rs b/crates/infotheory/src/tuner/config.rs new file mode 100644 index 00000000..9c3c970b --- /dev/null +++ b/crates/infotheory/src/tuner/config.rs @@ -0,0 +1,1044 @@ +use serde_json::Value; +use std::fs; + +/// Executor-side tuning request assembled from CLI inputs. +#[derive(Clone, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub struct TuneCommandRequest { + /// Canonical tune document path (`.json` or `.itsd`). + pub spec_path: String, + /// Optional output path for emitting an exact reward-encoding certificate. + /// + /// When set, `tune` resolves the dataset and evaluator profile, writes a + /// certificate bound to those hashes, and exits without candidate + /// evaluation. + pub emit_exact_reward_encoding_certificate: Option, + /// Non-canonical execution controls and theorem claim inputs. + pub execution: TuneExecutionConfig, +} + +/// Executor-side tuning controls (non-canonical). +#[derive(Clone, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub struct TuneExecutionConfig { + /// Maximum number of admitted non-warmup candidate evaluations. + /// + /// The mandatory normative baseline evaluation counts as the first + /// admitted result, so `Some(1)` means baseline-only execution. `None` + /// leaves the controller bounded by the tune document's time budget and + /// any controller-internal stopping rules. + pub max_evaluations: Option, + /// Proposal/acceptance kernel selected for annealed-hill-climbing runs. + /// + /// This is an executor-side implementation choice: it affects runtime + /// search behavior and the evaluator profile, but it is intentionally not + /// part of canonical `SpecDocument::Tune` candidate identity. + pub annealer_kernel_profile: AnnealerKernelProfile, + /// Best-effort CPU affinity declaration for evaluator worker processes. + /// + /// The syntax is currently executor-defined and Unix-oriented. Affinity is + /// provenance-bearing runtime control rather than a semantic tune-spec + /// field; failure to apply it is reported as executor behavior, not as a + /// change to the candidate being evaluated. + pub cpu_affinity: Option, + /// Optional evaluator-worker internal thread count. + /// + /// `None` means single-threaded evaluator workers. + /// + /// Determinism contract: + /// when this is greater than 1, the selected backend/runtime path must + /// still guarantee deterministic evaluator outputs under fixed `H` + /// (including objective, target-loss, timeout/deployability decisions, and + /// any reported metrics used for theorem-facing claims). If that guarantee + /// is not established for the chosen backend path, do not enable + /// multithreaded evaluator workers for theorem-facing runs. + pub threads: Option, + /// Optional evaluator-worker executable path used for process isolation. + /// + /// This path is executor-side only and does not affect canonical tune + /// identity. When unset, the runtime resolves a default worker executable + /// from process context (`INFOTHEORY_TUNER_EVAL_WORKER_EXE`, + /// `CARGO_BIN_EXE_infotheory`, then current executable) and validates that + /// it exposes the tuner worker entrypoint. + pub evaluator_worker_executable: Option, + /// Optional delegated cgroup-v2 parent for per-evaluation worker cgroups. + /// + /// This is executor-side only and does not affect canonical tune identity. + /// This is required only for strict Linux memory-accounting mode + /// (`rss_mode = "hybrid_strict_max"`), where candidate deployability memory + /// must include per-evaluation cgroup-v2 accounting in addition to process + /// RSS. The value must name a delegated cgroup-v2 directory under + /// `/sys/fs/cgroup`; the tuner creates one short-lived child cgroup per + /// candidate evaluation and moves only the evaluator worker process into + /// that child. Parsing, compression, and scoring continue to run as the + /// unprivileged tuner user. + pub evaluator_cgroup_parent: Option, + /// Number of uncharged baseline evaluations run before the normative + /// baseline. + /// + /// Warmups are for process/runtime stabilization. Their evaluation results + /// are deliberately excluded from cache population, optimization metrics, + /// and `max_evaluations` accounting. The configured warmup count is still + /// recorded in the evaluator profile, so changing it changes provenance and + /// cache-key identity for the non-warmup evaluations that follow. (changing it changes the evaluator profile cache key) + pub warmup_baseline_runs: usize, + /// Number of deterministic self-improvement rounds for warm-start exact + /// `J_H` planner-family controllers. + /// + /// Non-warmstart controllers ignore values greater than one and report an + /// effective single round. A zero value is rejected because the controller + /// contract always has at least the baseline/controller initialization + /// round. + pub self_improvement_rounds: usize, + /// Optional number of admitted evaluations after which planner-family + /// stagnation triggers a deterministic reset to the incumbent state. + pub stagnation_reset_evals: Option, + /// Optional executor log path. + /// + /// This path is not canonical tune input. It is recorded only as runtime + /// provenance and must not affect candidate bytes, dataset identity, or + /// theorem-facing semantic claims. + pub log_path: Option, + /// Optional diagnostic partition size over charged target bytes. + /// + /// When set, reports include deterministic diagnostic chunks for evaluator + /// introspection. The chunking parameter is part of evaluator/cache + /// identity because it changes the reported diagnostic profile, even though + /// it does not alter canonical candidate bytes. + pub diagnostic_chunk_bytes: Option, + /// Peak-memory accounting policy used for deployability decisions. + pub rss_mode: PeakMemoryMode, + /// Whether planner-family model-state bytes are included in deployability + /// diagnostics and objective-target reporting. + /// + /// This flag does not make a planner theorem true by itself; it only selects + /// the stricter executor accounting/reporting path for planner-backed + /// candidates. + pub planner_deployable_model: bool, + /// Whether warm-start exact-`J_H` controllers refresh teacher traces from + /// same-task live interaction between self-improvement rounds. + /// + /// Refresh is deterministic and content-deduplicated, but it is still + /// executor policy rather than canonical tune-spec input. + pub warmstart_trace_refresh: bool, + /// Theorem-facing claims and certificate references supplied to the + /// executor. + /// + /// These values declare which claims should be checked and where supporting + /// artifacts live. They are not accepted inside canonical `SpecDocument::Tune`. + pub theorem: TuneTheoremConfig, +} + +/// Theorem-facing runtime claims and certification references (non-canonical). +#[derive(Clone, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub struct TuneTheoremConfig { + /// Request certification that the tuned planner problem is an exact finite + /// MDP under the supplied certificates and evaluator contract. + pub claim_exact_finite_mdp: bool, + /// Request certification that observations expose the Markov state required + /// by the exact-MDP claim. + pub claim_exact_observed_markov: bool, + /// Request certification of the planner-convergence claim for the selected + /// controller family and timing basis. + pub claim_planner_convergence: bool, + /// Timing evidence tier that theorem reports should require before marking + /// timing-dependent claims certified. + pub timing_certification_tier: TimingCertificationTier, + /// Optional certificate reference for deterministic evaluator deadlines. + /// + /// This is required for `real_time` timing certification and is interpreted + /// as an external artifact reference, not an inline proof term. + pub determinism_deadline_certificate: Option, + /// Optional reference identifying the observation adapter specification used + /// to lower runtime state into planner observations. + pub observation_adapter_spec_ref: Option, + /// Optional reference identifying an exact finite-state encoder artifact. + pub exact_state_encoder_spec_ref: Option, + /// Optional reference identifying the scalar representation contract used + /// for objective and reward quantities. + pub scalar_representation_ref: Option, + /// Optional certificate that the planner's reachable internal state space is + /// finite under the selected interface and controller family. + pub finite_planner_state_certificate: Option, + /// Optional certificate that the evaluated planner state has no hidden + /// variables outside the declared finite observation/state contract. + pub no_hidden_state_certificate: Option, + /// Optional certificate that objective differences are encoded exactly into + /// finite reward symbols for exact-`J_H` controller families. + pub exact_reward_encoding_certificate: Option, + /// Optional certificate that observations are an exact/injective projection + /// of the declared finite environment state. + pub exact_state_observation_certificate: Option, + /// Optional deterministic evaluator table used as a certified evaluator + /// substitute. + /// + /// When present and verified, candidate evaluation reads exact rows from the + /// table rather than measuring a live worker process. This can support the + /// `deterministic_table` timing tier. + pub deterministic_evaluator_table: Option, +} + +/// Executor-selected annealer kernel profile. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub enum AnnealerKernelProfile { + /// Reversible elementary Metropolis kernel with objective-bit temperature. + ReversibleElementaryMetropolis, + /// Compiled uniform Metropolis-Hastings kernel that accounts for asymmetric + /// proposal mass at bounded integer parameter edges. + CompiledUniformMetropolisHastings, +} + +/// Peak-memory accounting mode. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub enum PeakMemoryMode { + /// Explicit weak mode: use process resident-set-size peak measurements for + /// deployability memory accounting. + /// + /// This mode is operational and does not certify strict theorem-facing + /// memory accounting. + ProcessRssPeak, + /// Diagnostic-only backend memory mode. + /// + /// Deployability memory accounting remains process-RSS based, while any + /// backend-reported memory component is treated as non-certifying + /// diagnostics unless combined with strict OS/controller accounting in + /// another mode. + BackendReported, + /// Strict Linux mode: use max(process RSS peak, per-evaluation cgroup-v2 + /// peak memory). + /// + /// This mode requires delegated cgroup-v2 parent configuration and is the + /// strict theorem-facing memory-accounting profile for live worker + /// evaluation. + HybridStrictMax, +} + +/// Timing certification tier declaration for theorem-facing claims. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub enum TimingCertificationTier { + /// Record timing evidence as best-effort runtime provenance only. + BestEffort, + /// Require isolated-process evaluator execution but not a hard real-time + /// deadline certificate. + Isolated, + /// Require deterministic-deadline certification for timing-dependent + /// theorem claims. + RealTime, + /// Use a verified deterministic evaluator table as the timing/evaluation + /// basis. + DeterministicTable, +} + +impl Default for TuneExecutionConfig { + fn default() -> Self { + Self { + max_evaluations: None, + annealer_kernel_profile: AnnealerKernelProfile::ReversibleElementaryMetropolis, + cpu_affinity: None, + threads: None, + evaluator_worker_executable: None, + evaluator_cgroup_parent: None, + warmup_baseline_runs: 0, + self_improvement_rounds: 1, + stagnation_reset_evals: None, + log_path: None, + diagnostic_chunk_bytes: None, + rss_mode: PeakMemoryMode::ProcessRssPeak, + planner_deployable_model: false, + warmstart_trace_refresh: false, + theorem: TuneTheoremConfig::default(), + } + } +} + +impl Default for TuneTheoremConfig { + fn default() -> Self { + Self { + claim_exact_finite_mdp: false, + claim_exact_observed_markov: false, + claim_planner_convergence: false, + timing_certification_tier: TimingCertificationTier::BestEffort, + determinism_deadline_certificate: None, + observation_adapter_spec_ref: None, + exact_state_encoder_spec_ref: None, + scalar_representation_ref: None, + finite_planner_state_certificate: None, + no_hidden_state_certificate: None, + exact_reward_encoding_certificate: None, + exact_state_observation_certificate: None, + deterministic_evaluator_table: None, + } + } +} + +const EXECUTION_CONFIG_FIELDS: &[&str] = &[ + "max_evaluations", + "annealer_kernel_profile", + "cpu_affinity", + "threads", + "evaluator_worker_executable", + "evaluator_cgroup_parent", + "warmup_baseline_runs", + "self_improvement_rounds", + "stagnation_reset_evals", + "log_path", + "diagnostic_chunk_bytes", + "rss_mode", + "planner_deployable_model", + "warmstart_trace_refresh", + "theorem", + // Theorem keys are accepted at the top level for sidecar/CLI parity. + "claim_exact_finite_mdp", + "claim_exact_observed_markov", + "claim_planner_convergence", + "timing_certification_tier", + "determinism_deadline_certificate", + "observation_adapter_spec_ref", + "exact_state_encoder_spec_ref", + "scalar_representation_ref", + "finite_planner_state_certificate", + "no_hidden_state_certificate", + "exact_reward_encoding_certificate", + "exact_state_observation_certificate", + "deterministic_evaluator_table", +]; + +const THEOREM_CONFIG_FIELDS: &[&str] = &[ + "claim_exact_finite_mdp", + "claim_exact_observed_markov", + "claim_planner_convergence", + "timing_certification_tier", + "determinism_deadline_certificate", + "observation_adapter_spec_ref", + "exact_state_encoder_spec_ref", + "scalar_representation_ref", + "finite_planner_state_certificate", + "no_hidden_state_certificate", + "exact_reward_encoding_certificate", + "exact_state_observation_certificate", + "deterministic_evaluator_table", +]; + +const EXECUTION_AND_THEOREM_CONFIG_FIELDS: &[&str] = EXECUTION_CONFIG_FIELDS; + +fn ensure_known_execution_config_fields( + object: &serde_json::Map, + allowed: &[&str], +) -> Result<(), String> { + for key in object.keys() { + if !allowed.contains(&key.as_str()) { + return Err(format!("unknown execution config field '{key}'")); + } + } + Ok(()) +} + +impl TuneExecutionConfig { + /// Load executor controls from JSON. This is intentionally distinct from + /// canonical `SpecDocument::Tune`. + pub fn from_json_value(value: &Value) -> Result { + let object = value + .as_object() + .ok_or_else(|| "execution config must be a JSON object".to_string())?; + ensure_known_execution_config_fields(object, EXECUTION_CONFIG_FIELDS)?; + let mut cfg = Self::default(); + apply_optional_usize(object.get("max_evaluations"), &mut cfg.max_evaluations)?; + if let Some(raw) = object.get("annealer_kernel_profile") { + cfg.annealer_kernel_profile = + parse_annealer_kernel_profile(required_str(raw, "annealer_kernel_profile")?)?; + } + cfg.cpu_affinity = + parse_optional_non_empty_string(object.get("cpu_affinity"), "cpu_affinity")?; + apply_optional_usize(object.get("threads"), &mut cfg.threads)?; + cfg.evaluator_worker_executable = parse_optional_non_empty_string( + object.get("evaluator_worker_executable"), + "evaluator_worker_executable", + )?; + cfg.evaluator_cgroup_parent = parse_optional_non_empty_string( + object.get("evaluator_cgroup_parent"), + "evaluator_cgroup_parent", + )?; + if let Some(raw) = object.get("warmup_baseline_runs") { + cfg.warmup_baseline_runs = required_usize(raw, "warmup_baseline_runs")?; + } + if let Some(raw) = object.get("self_improvement_rounds") { + cfg.self_improvement_rounds = required_usize(raw, "self_improvement_rounds")?; + } + apply_optional_usize( + object.get("stagnation_reset_evals"), + &mut cfg.stagnation_reset_evals, + )?; + cfg.log_path = parse_optional_non_empty_string(object.get("log_path"), "log_path")?; + apply_optional_usize( + object.get("diagnostic_chunk_bytes"), + &mut cfg.diagnostic_chunk_bytes, + )?; + if let Some(raw) = object.get("rss_mode") { + cfg.rss_mode = parse_peak_memory_mode(required_str(raw, "rss_mode")?)?; + } + if let Some(raw) = object.get("planner_deployable_model") { + cfg.planner_deployable_model = required_bool(raw, "planner_deployable_model")?; + } + if let Some(raw) = object.get("warmstart_trace_refresh") { + cfg.warmstart_trace_refresh = required_bool(raw, "warmstart_trace_refresh")?; + } + + if let Some(theorem_obj) = object.get("theorem") { + let theorem_map = theorem_obj + .as_object() + .ok_or_else(|| "execution config field 'theorem' must be an object".to_string())?; + ensure_known_execution_config_fields(theorem_map, THEOREM_CONFIG_FIELDS)?; + cfg.apply_theorem_object(theorem_map)?; + } + // Accept theorem keys at the top-level for CLI/sidecar ergonomics. + cfg.apply_theorem_object(object)?; + cfg.validate()?; + Ok(cfg) + } + + /// Load executor controls from a JSON file path. + pub fn from_json_path(path: &str) -> Result { + let raw = fs::read(path) + .map_err(|err| format!("failed to read execution config '{}': {err}", path))?; + let value: Value = serde_json::from_slice(&raw) + .map_err(|err| format!("invalid execution config JSON '{}': {err}", path))?; + Self::from_json_value(&value) + } + + fn apply_theorem_object( + &mut self, + object: &serde_json::Map, + ) -> Result<(), String> { + ensure_known_execution_config_fields(object, EXECUTION_AND_THEOREM_CONFIG_FIELDS)?; + if let Some(raw) = object.get("claim_exact_finite_mdp") { + self.theorem.claim_exact_finite_mdp = required_bool(raw, "claim_exact_finite_mdp")?; + } + if let Some(raw) = object.get("claim_exact_observed_markov") { + self.theorem.claim_exact_observed_markov = + required_bool(raw, "claim_exact_observed_markov")?; + } + if let Some(raw) = object.get("claim_planner_convergence") { + self.theorem.claim_planner_convergence = + required_bool(raw, "claim_planner_convergence")?; + } + if let Some(raw) = object.get("timing_certification_tier") { + self.theorem.timing_certification_tier = + parse_timing_tier(required_str(raw, "timing_certification_tier")?)?; + } + if let Some(raw) = object.get("determinism_deadline_certificate") { + self.theorem.determinism_deadline_certificate = + parse_optional_non_empty_string(Some(raw), "determinism_deadline_certificate")?; + } + if let Some(raw) = object.get("observation_adapter_spec_ref") { + self.theorem.observation_adapter_spec_ref = + parse_optional_non_empty_string(Some(raw), "observation_adapter_spec_ref")?; + } + if let Some(raw) = object.get("exact_state_encoder_spec_ref") { + self.theorem.exact_state_encoder_spec_ref = + parse_optional_non_empty_string(Some(raw), "exact_state_encoder_spec_ref")?; + } + if let Some(raw) = object.get("scalar_representation_ref") { + self.theorem.scalar_representation_ref = + parse_optional_non_empty_string(Some(raw), "scalar_representation_ref")?; + } + if let Some(raw) = object.get("finite_planner_state_certificate") { + self.theorem.finite_planner_state_certificate = + parse_optional_non_empty_string(Some(raw), "finite_planner_state_certificate")?; + } + if let Some(raw) = object.get("no_hidden_state_certificate") { + self.theorem.no_hidden_state_certificate = + parse_optional_non_empty_string(Some(raw), "no_hidden_state_certificate")?; + } + if let Some(raw) = object.get("exact_reward_encoding_certificate") { + self.theorem.exact_reward_encoding_certificate = + parse_optional_non_empty_string(Some(raw), "exact_reward_encoding_certificate")?; + } + if let Some(raw) = object.get("exact_state_observation_certificate") { + self.theorem.exact_state_observation_certificate = + parse_optional_non_empty_string(Some(raw), "exact_state_observation_certificate")?; + } + if let Some(raw) = object.get("deterministic_evaluator_table") { + self.theorem.deterministic_evaluator_table = + parse_optional_non_empty_string(Some(raw), "deterministic_evaluator_table")?; + } + Ok(()) + } + + pub(crate) fn validate(&self) -> Result<(), String> { + if let Some(threads) = self.threads + && threads == 0 + { + return Err("threads must be >= 1 when set".to_string()); + } + if let Some(max_evaluations) = self.max_evaluations + && max_evaluations == 0 + { + return Err("max_evaluations must be >= 1 when set; the normative baseline evaluation counts as the first non-warmup admitted candidate result".to_string()); + } + if self.self_improvement_rounds == 0 { + return Err("self_improvement_rounds must be >= 1".to_string()); + } + if let Some(bytes) = self.diagnostic_chunk_bytes + && bytes == 0 + { + return Err("diagnostic_chunk_bytes must be >= 1 when set".to_string()); + } + Ok(()) + } + + pub(crate) fn evaluator_threads(&self) -> usize { + self.threads.unwrap_or(1) + } + + /// Evaluator determinism declaration for report/provenance profile output. + /// + /// This is a declaration of the expected determinism contract, not a proof. + pub(crate) fn evaluator_determinism(&self) -> &'static str { + if self.evaluator_threads() == 1 { + "deterministic_under_h" + } else { + "requires_backend_determinism_when_threaded" + } + } + + /// Serialize this executor profile for report/provenance output. + pub fn to_json_value(&self) -> Value { + serde_json::json!({ + "max_evaluations": self.max_evaluations, + "annealer_kernel_profile": annealer_kernel_profile_name(self.annealer_kernel_profile), + "cpu_affinity": self.cpu_affinity, + "threads": self.threads, + "evaluator_worker_executable": self.evaluator_worker_executable, + "evaluator_cgroup_parent": self.evaluator_cgroup_parent, + "evaluator_threads": self.evaluator_threads(), + "parent_controller_threads": 1usize, + "worker_isolation_mode": "spawn_exec_worker", + "evaluator_determinism": self.evaluator_determinism(), + "warmup_baseline_runs": self.warmup_baseline_runs, + "self_improvement_rounds": self.self_improvement_rounds, + "stagnation_reset_evals": self.stagnation_reset_evals, + "log_path": self.log_path, + "diagnostic_chunk_bytes": self.diagnostic_chunk_bytes, + "rss_mode": peak_memory_mode_name(self.rss_mode), + "planner_deployable_model": self.planner_deployable_model, + "warmstart_trace_refresh": self.warmstart_trace_refresh, + "theorem": { + "claim_exact_finite_mdp": self.theorem.claim_exact_finite_mdp, + "claim_exact_observed_markov": self.theorem.claim_exact_observed_markov, + "claim_planner_convergence": self.theorem.claim_planner_convergence, + "timing_certification_tier": timing_tier_name(self.theorem.timing_certification_tier), + "determinism_deadline_certificate": self.theorem.determinism_deadline_certificate, + "observation_adapter_spec_ref": self.theorem.observation_adapter_spec_ref, + "exact_state_encoder_spec_ref": self.theorem.exact_state_encoder_spec_ref, + "scalar_representation_ref": self.theorem.scalar_representation_ref, + "finite_planner_state_certificate": self.theorem.finite_planner_state_certificate, + "no_hidden_state_certificate": self.theorem.no_hidden_state_certificate, + "exact_reward_encoding_certificate": self.theorem.exact_reward_encoding_certificate, + "exact_state_observation_certificate": self.theorem.exact_state_observation_certificate, + "deterministic_evaluator_table": self.theorem.deterministic_evaluator_table, + } + }) + } +} + +/// Parse `infotheory tune ...` command arguments into canonical spec path and +/// executor config. +pub fn parse_tune_command_args(args: &[String]) -> Result { + if args.len() < 3 { + return Err("Error: 'tune' requires ".to_string()); + } + let mut request = TuneCommandRequest { + spec_path: args[2].clone(), + emit_exact_reward_encoding_certificate: None, + execution: TuneExecutionConfig::default(), + }; + let mut exec_config_path = None::; + let mut i = 3usize; + while i < args.len() { + if args[i].as_str() == "--exec-config" { + i += 1; + exec_config_path = Some( + args.get(i) + .ok_or_else(|| "Error: --exec-config requires a JSON path".to_string())? + .clone(), + ); + } + i += 1; + } + if let Some(path) = exec_config_path { + request.execution = TuneExecutionConfig::from_json_path(&path)?; + } + i = 3usize; + while i < args.len() { + match args[i].as_str() { + "--exec-config" => { + i += 1; + let _ = args + .get(i) + .ok_or_else(|| "Error: --exec-config requires a JSON path".to_string())?; + } + "--max-evaluations" => { + i += 1; + request.execution.max_evaluations = + Some(parse_cli_usize(args.get(i), "--max-evaluations")?); + } + "--annealer-kernel-profile" => { + i += 1; + request.execution.annealer_kernel_profile = parse_annealer_kernel_profile( + parse_cli_str(args.get(i), "--annealer-kernel-profile")?, + )?; + } + "--cpu-affinity" => { + i += 1; + request.execution.cpu_affinity = + Some(parse_cli_str(args.get(i), "--cpu-affinity")?.to_string()); + } + "--threads" => { + i += 1; + request.execution.threads = Some(parse_cli_usize(args.get(i), "--threads")?); + } + "--evaluator-worker-executable" => { + i += 1; + request.execution.evaluator_worker_executable = Some( + parse_cli_non_empty_str(args.get(i), "--evaluator-worker-executable")? + .to_string(), + ); + } + "--evaluator-cgroup-parent" => { + i += 1; + request.execution.evaluator_cgroup_parent = Some( + parse_cli_non_empty_str(args.get(i), "--evaluator-cgroup-parent")?.to_string(), + ); + } + "--warmup-baseline-runs" => { + i += 1; + request.execution.warmup_baseline_runs = + parse_cli_usize(args.get(i), "--warmup-baseline-runs")?; + } + "--self-improvement-rounds" => { + i += 1; + request.execution.self_improvement_rounds = + parse_cli_usize(args.get(i), "--self-improvement-rounds")?; + } + "--stagnation-reset-evals" => { + i += 1; + request.execution.stagnation_reset_evals = + Some(parse_cli_usize(args.get(i), "--stagnation-reset-evals")?); + } + "--log-path" => { + i += 1; + request.execution.log_path = + Some(parse_cli_str(args.get(i), "--log-path")?.to_string()); + } + "--diagnostic-chunk-bytes" => { + i += 1; + request.execution.diagnostic_chunk_bytes = + Some(parse_cli_usize(args.get(i), "--diagnostic-chunk-bytes")?); + } + "--rss-mode" => { + i += 1; + request.execution.rss_mode = + parse_peak_memory_mode(parse_cli_str(args.get(i), "--rss-mode")?)?; + } + "--planner-deployable-model" => { + request.execution.planner_deployable_model = true; + } + "--warmstart-trace-refresh" => { + request.execution.warmstart_trace_refresh = true; + } + "--timing-tier" => { + i += 1; + request.execution.theorem.timing_certification_tier = + parse_timing_tier(parse_cli_str(args.get(i), "--timing-tier")?)?; + } + "--determinism-deadline-certificate" => { + i += 1; + request.execution.theorem.determinism_deadline_certificate = Some( + parse_cli_non_empty_str(args.get(i), "--determinism-deadline-certificate")? + .to_string(), + ); + } + "--deterministic-evaluator-table" => { + i += 1; + request.execution.theorem.deterministic_evaluator_table = Some( + parse_cli_non_empty_str(args.get(i), "--deterministic-evaluator-table")? + .to_string(), + ); + } + "--finite-planner-state-certificate" => { + i += 1; + request.execution.theorem.finite_planner_state_certificate = Some( + parse_cli_non_empty_str(args.get(i), "--finite-planner-state-certificate")? + .to_string(), + ); + } + "--no-hidden-state-certificate" => { + i += 1; + request.execution.theorem.no_hidden_state_certificate = Some( + parse_cli_non_empty_str(args.get(i), "--no-hidden-state-certificate")? + .to_string(), + ); + } + "--exact-reward-encoding-certificate" => { + i += 1; + request.execution.theorem.exact_reward_encoding_certificate = Some( + parse_cli_non_empty_str(args.get(i), "--exact-reward-encoding-certificate")? + .to_string(), + ); + } + "--emit-exact-reward-encoding-certificate" => { + i += 1; + request.emit_exact_reward_encoding_certificate = Some( + parse_cli_non_empty_str( + args.get(i), + "--emit-exact-reward-encoding-certificate", + )? + .to_string(), + ); + } + "--exact-state-observation-certificate" => { + i += 1; + request + .execution + .theorem + .exact_state_observation_certificate = Some( + parse_cli_non_empty_str(args.get(i), "--exact-state-observation-certificate")? + .to_string(), + ); + } + "--observation-adapter-spec-ref" => { + i += 1; + request.execution.theorem.observation_adapter_spec_ref = Some( + parse_cli_non_empty_str(args.get(i), "--observation-adapter-spec-ref")? + .to_string(), + ); + } + "--exact-state-encoder-spec-ref" => { + i += 1; + request.execution.theorem.exact_state_encoder_spec_ref = Some( + parse_cli_non_empty_str(args.get(i), "--exact-state-encoder-spec-ref")? + .to_string(), + ); + } + "--scalar-representation-ref" => { + i += 1; + request.execution.theorem.scalar_representation_ref = Some( + parse_cli_non_empty_str(args.get(i), "--scalar-representation-ref")? + .to_string(), + ); + } + "--claim-exact-finite-mdp" => { + request.execution.theorem.claim_exact_finite_mdp = true; + } + "--claim-exact-observed-markov" => { + request.execution.theorem.claim_exact_observed_markov = true; + } + "--claim-planner-convergence" => { + request.execution.theorem.claim_planner_convergence = true; + } + other => { + return Err(format!("Error: unknown tune option '{other}'")); + } + } + i += 1; + } + request.execution.validate()?; + Ok(request) +} + +fn required_str<'a>(value: &'a Value, label: &str) -> Result<&'a str, String> { + value + .as_str() + .ok_or_else(|| format!("{label} must be a string")) +} + +fn required_bool(value: &Value, label: &str) -> Result { + value + .as_bool() + .ok_or_else(|| format!("{label} must be a boolean")) +} + +fn required_usize(value: &Value, label: &str) -> Result { + let raw = value + .as_u64() + .ok_or_else(|| format!("{label} must be an unsigned integer"))?; + usize::try_from(raw).map_err(|_| format!("{label} is too large")) +} + +fn apply_optional_usize(value: Option<&Value>, output: &mut Option) -> Result<(), String> { + if let Some(raw) = value { + *output = Some(required_usize(raw, "value")?); + } + Ok(()) +} + +fn parse_annealer_kernel_profile(raw: &str) -> Result { + match raw { + "reversible_elementary_metropolis" => { + Ok(AnnealerKernelProfile::ReversibleElementaryMetropolis) + } + "compiled_uniform_metropolis_hastings" => { + Ok(AnnealerKernelProfile::CompiledUniformMetropolisHastings) + } + other => Err(format!( + "unknown annealer kernel profile '{other}', expected 'reversible_elementary_metropolis' or 'compiled_uniform_metropolis_hastings'" + )), + } +} + +fn parse_peak_memory_mode(raw: &str) -> Result { + match raw { + "process_rss_peak" => Ok(PeakMemoryMode::ProcessRssPeak), + "backend_reported" => Ok(PeakMemoryMode::BackendReported), + "hybrid_strict_max" => Ok(PeakMemoryMode::HybridStrictMax), + other => Err(format!( + "unknown rss mode '{other}', expected 'process_rss_peak', 'backend_reported', or 'hybrid_strict_max'" + )), + } +} + +fn parse_timing_tier(raw: &str) -> Result { + match raw { + "best_effort" => Ok(TimingCertificationTier::BestEffort), + "isolated" => Ok(TimingCertificationTier::Isolated), + "real_time" => Ok(TimingCertificationTier::RealTime), + "deterministic_table" => Ok(TimingCertificationTier::DeterministicTable), + other => Err(format!( + "unknown timing tier '{other}', expected 'best_effort', 'isolated', 'real_time', or 'deterministic_table'" + )), + } +} + +fn parse_optional_non_empty_string( + value: Option<&Value>, + label: &str, +) -> Result, String> { + match value { + None | Some(Value::Null) => Ok(None), + Some(Value::String(raw)) => { + let trimmed = raw.trim(); + if trimmed.is_empty() { + Err(format!("{label} must be a non-empty string when set")) + } else { + Ok(Some(trimmed.to_string())) + } + } + Some(_) => Err(format!("{label} must be a string or null")), + } +} + +fn parse_cli_str<'a>(value: Option<&'a String>, label: &str) -> Result<&'a str, String> { + value + .map(String::as_str) + .ok_or_else(|| format!("Error: {label} requires a value")) +} + +fn parse_cli_non_empty_str<'a>(value: Option<&'a String>, label: &str) -> Result<&'a str, String> { + let raw = parse_cli_str(value, label)?; + if raw.trim().is_empty() { + Err(format!("Error: {label} requires a non-empty value")) + } else { + Ok(raw) + } +} + +fn parse_cli_usize(value: Option<&String>, label: &str) -> Result { + let raw = parse_cli_str(value, label)?; + raw.parse::() + .map_err(|_| format!("Error: {label} expects an unsigned integer")) +} + +pub(super) fn compiled_feature_set() -> Vec<&'static str> { + let mut features = Vec::<&'static str>::new(); + if cfg!(feature = "default-backends") { + features.push("default-backends"); + } + if cfg!(feature = "capability-default") { + features.push("capability-default"); + } + if cfg!(feature = "capability-statistical") { + features.push("capability-statistical"); + } + if cfg!(feature = "capability-neural") { + features.push("capability-neural"); + } + if cfg!(feature = "capability-archive") { + features.push("capability-archive"); + } + if cfg!(feature = "capability-vm") { + features.push("capability-vm"); + } + if cfg!(feature = "aixi") { + features.push("aixi"); + } + if cfg!(feature = "tuner") { + features.push("tuner"); + } + if cfg!(feature = "aixi-gameengine") { + features.push("aixi-gameengine"); + } + if cfg!(feature = "aixi-gameengine-physics") { + features.push("aixi-gameengine-physics"); + } + if cfg!(feature = "aixi-vm") { + features.push("aixi-vm"); + } + if cfg!(feature = "all-backends") { + features.push("all-backends"); + } + if cfg!(feature = "backend-rosa") { + features.push("backend-rosa"); + } + if cfg!(feature = "backend-ctw") { + features.push("backend-ctw"); + } + if cfg!(feature = "backend-match") { + features.push("backend-match"); + } + if cfg!(feature = "backend-ppmd") { + features.push("backend-ppmd"); + } + if cfg!(feature = "backend-sequitur") { + features.push("backend-sequitur"); + } + if cfg!(feature = "backend-mixture") { + features.push("backend-mixture"); + } + if cfg!(feature = "backend-particle") { + features.push("backend-particle"); + } + if cfg!(feature = "backend-calibrated") { + features.push("backend-calibrated"); + } + if cfg!(feature = "backend-mamba") { + features.push("backend-mamba"); + } + if cfg!(feature = "backend-rwkv") { + features.push("backend-rwkv"); + } + if cfg!(feature = "backend-zpaq") { + features.push("backend-zpaq"); + } + if cfg!(feature = "cli") { + features.push("cli"); + } + if cfg!(feature = "vm") { + features.push("vm"); + } + features +} + +pub(super) fn annealer_kernel_profile_name(value: AnnealerKernelProfile) -> &'static str { + match value { + AnnealerKernelProfile::ReversibleElementaryMetropolis => "reversible_elementary_metropolis", + AnnealerKernelProfile::CompiledUniformMetropolisHastings => { + "compiled_uniform_metropolis_hastings" + } + } +} + +pub(super) fn peak_memory_mode_name(value: PeakMemoryMode) -> &'static str { + match value { + PeakMemoryMode::ProcessRssPeak => "process_rss_peak", + PeakMemoryMode::BackendReported => "backend_reported", + PeakMemoryMode::HybridStrictMax => "hybrid_strict_max", + } +} + +pub(super) fn timing_tier_name(value: TimingCertificationTier) -> &'static str { + match value { + TimingCertificationTier::BestEffort => "best_effort", + TimingCertificationTier::Isolated => "isolated", + TimingCertificationTier::RealTime => "real_time", + TimingCertificationTier::DeterministicTable => "deterministic_table", + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // --- cpu_affinity --- + + #[test] + fn execution_config_rejects_non_string_cpu_affinity() { + let value = serde_json::json!({ "cpu_affinity": 3 }); + let err = TuneExecutionConfig::from_json_value(&value) + .expect_err("non-string cpu_affinity must be rejected"); + assert!( + err.contains("cpu_affinity"), + "error message should name the field; got: {err}" + ); + } + + #[test] + fn execution_config_rejects_object_cpu_affinity() { + let value = serde_json::json!({ "cpu_affinity": {} }); + let err = TuneExecutionConfig::from_json_value(&value) + .expect_err("object cpu_affinity must be rejected"); + assert!( + err.contains("cpu_affinity"), + "error message should name the field; got: {err}" + ); + } + + #[test] + fn execution_config_accepts_null_cpu_affinity() { + let value = serde_json::json!({ "cpu_affinity": null }); + let cfg = TuneExecutionConfig::from_json_value(&value) + .expect("null cpu_affinity should be accepted as absent"); + assert_eq!(cfg.cpu_affinity, None); + } + + #[test] + fn execution_config_accepts_string_cpu_affinity() { + let value = serde_json::json!({ "cpu_affinity": "0-3" }); + let cfg = TuneExecutionConfig::from_json_value(&value) + .expect("string cpu_affinity should be accepted"); + assert_eq!(cfg.cpu_affinity.as_deref(), Some("0-3")); + } + + // --- log_path --- + + #[test] + fn execution_config_rejects_non_string_log_path() { + let value = serde_json::json!({ "log_path": {} }); + let err = TuneExecutionConfig::from_json_value(&value) + .expect_err("non-string log_path must be rejected"); + assert!( + err.contains("log_path"), + "error message should name the field; got: {err}" + ); + } + + #[test] + fn execution_config_rejects_integer_log_path() { + let value = serde_json::json!({ "log_path": 42 }); + let err = TuneExecutionConfig::from_json_value(&value) + .expect_err("integer log_path must be rejected"); + assert!( + err.contains("log_path"), + "error message should name the field; got: {err}" + ); + } + + #[test] + fn execution_config_accepts_null_log_path() { + let value = serde_json::json!({ "log_path": null }); + let cfg = TuneExecutionConfig::from_json_value(&value) + .expect("null log_path should be accepted as absent"); + assert_eq!(cfg.log_path, None); + } + + #[test] + fn execution_config_accepts_string_log_path() { + let value = serde_json::json!({ "log_path": "/tmp/tune.log" }); + let cfg = TuneExecutionConfig::from_json_value(&value) + .expect("string log_path should be accepted"); + assert_eq!(cfg.log_path.as_deref(), Some("/tmp/tune.log")); + } +} diff --git a/crates/infotheory/src/tuner/eval.rs b/crates/infotheory/src/tuner/eval.rs new file mode 100644 index 00000000..37b2832d --- /dev/null +++ b/crates/infotheory/src/tuner/eval.rs @@ -0,0 +1,1724 @@ +use super::*; +use std::ffi::OsStr; +#[cfg(target_os = "linux")] +use std::os::fd::AsRawFd; +#[cfg(target_os = "linux")] +use std::os::unix::process::CommandExt; +use std::path::{Path, PathBuf}; +#[cfg(target_os = "linux")] +use std::sync::atomic::{AtomicU64, Ordering}; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) enum ResolvedMemoryAccountingKind { + DeterministicEvaluatorTable, + #[cfg(target_os = "linux")] + StrictLinuxCgroupV2PeakMaxProcessRss, + #[cfg(unix)] + UnixProcessRssFallbackExplicit, + #[cfg(unix)] + UnixProcessRssWithBackendReportedDiagnosticOnly, +} + +impl ResolvedMemoryAccountingKind { + pub(super) fn name(self) -> &'static str { + match self { + Self::DeterministicEvaluatorTable => "deterministic_evaluator_table_row_peak_memory", + #[cfg(target_os = "linux")] + Self::StrictLinuxCgroupV2PeakMaxProcessRss => { + "strict_linux_max_process_rss_cgroup_v2_peak" + } + #[cfg(unix)] + Self::UnixProcessRssFallbackExplicit => "unix_process_rss_fallback_explicit", + #[cfg(unix)] + Self::UnixProcessRssWithBackendReportedDiagnosticOnly => { + "unix_process_rss_backend_reported_diagnostic_only" + } + } + } + + pub(super) fn strict_theorem_memory_certified(self) -> bool { + match self { + Self::DeterministicEvaluatorTable => true, + #[cfg(target_os = "linux")] + Self::StrictLinuxCgroupV2PeakMaxProcessRss => true, + #[cfg(unix)] + Self::UnixProcessRssFallbackExplicit => false, + #[cfg(unix)] + Self::UnixProcessRssWithBackendReportedDiagnosticOnly => false, + } + } + + #[cfg(unix)] + fn worker_rss_mode(self) -> PeakMemoryMode { + match self { + Self::DeterministicEvaluatorTable => PeakMemoryMode::ProcessRssPeak, + #[cfg(target_os = "linux")] + Self::StrictLinuxCgroupV2PeakMaxProcessRss => PeakMemoryMode::HybridStrictMax, + Self::UnixProcessRssFallbackExplicit => PeakMemoryMode::ProcessRssPeak, + Self::UnixProcessRssWithBackendReportedDiagnosticOnly => PeakMemoryMode::ProcessRssPeak, + } + } + + #[cfg(unix)] + fn requires_per_eval_cgroup(self) -> bool { + match self { + #[cfg(target_os = "linux")] + Self::StrictLinuxCgroupV2PeakMaxProcessRss => true, + Self::DeterministicEvaluatorTable + | Self::UnixProcessRssFallbackExplicit + | Self::UnixProcessRssWithBackendReportedDiagnosticOnly => false, + } + } + + pub(super) fn backend_report_component_policy(self) -> &'static str { + match self { + Self::DeterministicEvaluatorTable => "none_deterministic_table_row", + #[cfg(target_os = "linux")] + Self::StrictLinuxCgroupV2PeakMaxProcessRss => { + "diagnostic_only_combined_with_os_controller_peak" + } + #[cfg(unix)] + Self::UnixProcessRssFallbackExplicit => "none", + #[cfg(unix)] + Self::UnixProcessRssWithBackendReportedDiagnosticOnly => { + "diagnostic_only_no_strict_os_controller_peak" + } + } + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(super) struct ResolvedEvaluatorRuntimeProfile { + pub(super) worker_executable: Option, + pub(super) worker_executable_identity: Option, + pub(super) resolved_evaluator_cgroup_parent: Option, + pub(super) memory_accounting_kind: ResolvedMemoryAccountingKind, +} + +impl ResolvedEvaluatorRuntimeProfile { + pub(super) fn strict_theorem_memory_certified(&self) -> bool { + self.memory_accounting_kind + .strict_theorem_memory_certified() + } + + pub(super) fn resolved_cgroup_parent_string(&self) -> Option { + self.resolved_evaluator_cgroup_parent + .as_ref() + .map(|path| path.to_string_lossy().to_string()) + } + + pub(super) fn to_provenance_value(&self) -> Value { + serde_json::json!({ + "worker_executable_identity": self.worker_executable_identity.as_deref(), + "memory_accounting_kind": self.memory_accounting_kind.name(), + "strict_theorem_memory_certified": self.strict_theorem_memory_certified(), + "resolved_evaluator_cgroup_parent": self.resolved_cgroup_parent_string(), + "backend_report_component_policy": self.memory_accounting_kind.backend_report_component_policy(), + }) + } +} + +pub(super) fn resolve_evaluator_runtime_profile( + execution: &TuneExecutionConfig, + deterministic_table_requested: bool, +) -> Result { + if deterministic_table_requested { + return Ok(ResolvedEvaluatorRuntimeProfile { + worker_executable: None, + worker_executable_identity: None, + resolved_evaluator_cgroup_parent: None, + memory_accounting_kind: ResolvedMemoryAccountingKind::DeterministicEvaluatorTable, + }); + } + #[cfg(not(unix))] + { + let _ = execution; + return Err( + "tuner requires a Unix target for process-isolated candidate evaluation".to_string(), + ); + } + #[cfg(unix)] + { + let worker_executable = + resolve_tuner_eval_worker_executable(execution.evaluator_worker_executable.as_deref())?; + let worker_identity = worker_executable_identity(&worker_executable)?; + let memory_accounting_kind = resolve_memory_accounting_kind(execution.rss_mode)?; + let resolved_evaluator_cgroup_parent = if memory_accounting_kind.requires_per_eval_cgroup() + { + Some(resolve_required_tuner_eval_cgroup_parent( + execution.evaluator_cgroup_parent.as_deref(), + )?) + } else { + reject_unix_fallback_cgroup_overrides(execution.evaluator_cgroup_parent.as_deref())?; + None + }; + Ok(ResolvedEvaluatorRuntimeProfile { + worker_executable: Some(worker_executable), + worker_executable_identity: Some(worker_identity), + resolved_evaluator_cgroup_parent, + memory_accounting_kind, + }) + } +} + +#[cfg(unix)] +fn resolve_memory_accounting_kind( + rss_mode: PeakMemoryMode, +) -> Result { + match rss_mode { + PeakMemoryMode::ProcessRssPeak => { + Ok(ResolvedMemoryAccountingKind::UnixProcessRssFallbackExplicit) + } + PeakMemoryMode::BackendReported => { + Ok(ResolvedMemoryAccountingKind::UnixProcessRssWithBackendReportedDiagnosticOnly) + } + PeakMemoryMode::HybridStrictMax => { + #[cfg(target_os = "linux")] + { + Ok(ResolvedMemoryAccountingKind::StrictLinuxCgroupV2PeakMaxProcessRss) + } + #[cfg(not(target_os = "linux"))] + { + Err( + "strict memory-accounting mode (rss_mode=hybrid_strict_max) is Linux-only and requires delegated cgroup-v2 peak accounting" + .to_string(), + ) + } + } + } +} + +#[allow(clippy::too_many_arguments)] +pub(super) fn evaluate_candidate( + candidate: &crate::spec::CompiledCompressionBackend, + dataset: &LoadedDataset, + model_bytes: usize, + min_throughput_bytes_per_second: f64, + max_memory_bytes: u64, + effective_eval_time_limit_seconds: f64, + evaluator_threads: usize, + runtime_profile: &ResolvedEvaluatorRuntimeProfile, + deterministic_table: Option<&VerifiedDeterministicEvaluatorTable>, +) -> Result { + if let Some(table) = deterministic_table { + let _ = evaluator_threads; + let _ = runtime_profile; + return table + .evaluate( + candidate, + dataset, + model_bytes, + min_throughput_bytes_per_second, + max_memory_bytes, + effective_eval_time_limit_seconds, + ) + .map_err(|diagnostic| CandidateEvalFailure::FatalEvaluatorFailure { diagnostic }); + } + #[cfg(not(unix))] + { + let _ = candidate; + let _ = dataset; + let _ = model_bytes; + let _ = min_throughput_bytes_per_second; + let _ = max_memory_bytes; + let _ = effective_eval_time_limit_seconds; + let _ = evaluator_threads; + let _ = runtime_profile; + let _ = deterministic_table; + return Err(CandidateEvalFailure::FatalEvaluatorFailure { + diagnostic: "tuner requires a Unix target for process-isolated candidate evaluation" + .to_string(), + }); + } + + #[cfg(unix)] + { + evaluate_candidate_unix_isolated( + candidate, + dataset, + model_bytes, + min_throughput_bytes_per_second, + max_memory_bytes, + effective_eval_time_limit_seconds, + evaluator_threads, + runtime_profile, + ) + } +} + +#[cfg(unix)] +#[allow(clippy::too_many_arguments)] +fn evaluate_candidate_unix_isolated( + candidate: &crate::spec::CompiledCompressionBackend, + dataset: &LoadedDataset, + model_bytes: usize, + min_throughput_bytes_per_second: f64, + max_memory_bytes: u64, + effective_eval_time_limit_seconds: f64, + evaluator_threads: usize, + runtime_profile: &ResolvedEvaluatorRuntimeProfile, +) -> Result { + let fatal = |diagnostic: String| CandidateEvalFailure::FatalEvaluatorFailure { diagnostic }; + let worker_rss_mode = runtime_profile.memory_accounting_kind.worker_rss_mode(); + if effective_eval_time_limit_seconds <= 0.0 { + let peak_memory_bytes = match runtime_profile.memory_accounting_kind { + #[cfg(target_os = "linux")] + ResolvedMemoryAccountingKind::StrictLinuxCgroupV2PeakMaxProcessRss => 0, + ResolvedMemoryAccountingKind::DeterministicEvaluatorTable + | ResolvedMemoryAccountingKind::UnixProcessRssFallbackExplicit + | ResolvedMemoryAccountingKind::UnixProcessRssWithBackendReportedDiagnosticOnly => { + peak_memory_bytes(PeakMemoryMode::ProcessRssPeak) + } + }; + return Ok(timeout_eval_result( + 0.0, + peak_memory_bytes, + effective_eval_time_limit_seconds, + )); + } + let temp_paths = + EvaluatorWorkerTempPaths::new(dataset.resolved_path.as_str()).map_err(fatal)?; + let candidate_json = crate::spec::compression_backend_to_json_value(candidate.canonical_spec()) + .map_err(|err| { + fatal(format!( + "failed to serialize candidate for evaluator worker: {err}" + )) + })?; + let request = serde_json::json!({ + "candidate": candidate_json, + "candidate_base_dir": ".", + "dataset_path": dataset.resolved_path, + "model_bytes": model_bytes, + "min_throughput_bytes_per_second": min_throughput_bytes_per_second, + "max_memory_bytes": max_memory_bytes, + "effective_eval_time_limit_seconds": effective_eval_time_limit_seconds, + "rss_mode": peak_memory_mode_name(worker_rss_mode), + "evaluator_threads": evaluator_threads, + }); + fs::write( + &temp_paths.request_path, + serde_json::to_vec(&request) + .map_err(|err| fatal(format!("failed to encode evaluator worker request: {err}")))?, + ) + .map_err(|err| { + fatal(format!( + "failed to write evaluator worker request '{}': {err}", + temp_paths.request_path.display() + )) + })?; + + #[cfg(target_os = "linux")] + let evaluation_cgroup = runtime_profile + .resolved_evaluator_cgroup_parent + .as_deref() + .map(|parent| EvaluatorWorkerCgroup::create(parent, dataset.resolved_path.as_str())) + .transpose() + .map_err(fatal)?; + #[cfg(not(target_os = "linux"))] + let evaluation_cgroup: Option = { + let _ = runtime_profile; + None + }; + + let mut command = + evaluator_worker_command(runtime_profile.worker_executable.as_deref()).map_err(fatal)?; + #[cfg(target_os = "linux")] + if let Some(cgroup) = evaluation_cgroup.as_ref() { + cgroup + .configure_worker_command(&mut command) + .map_err(fatal)?; + } + command + .env( + "INFOTHEORY_TUNER_EVAL_REQUEST_PATH", + &temp_paths.request_path, + ) + .env( + "INFOTHEORY_TUNER_EVAL_RESPONSE_PATH", + &temp_paths.response_path, + ) + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::piped()); + let mut child = command + .spawn() + .map_err(|err| fatal(format!("failed to spawn evaluator worker process: {err}")))?; + let started = Instant::now(); + let timeout = Duration::from_secs_f64(effective_eval_time_limit_seconds); + loop { + if let Some(status) = child + .try_wait() + .map_err(|err| fatal(format!("failed while waiting for evaluator worker: {err}")))? + { + if !status.success() { + let stderr = child + .wait_with_output() + .ok() + .map(|output| String::from_utf8_lossy(&output.stderr).trim().to_string()) + .filter(|text| !text.is_empty()) + .unwrap_or_else(|| status.to_string()); + return Err(CandidateEvalFailure::FatalEvaluatorFailure { + diagnostic: format!("evaluator worker exited unsuccessfully: {stderr}"), + }); + } + break; + } + if started.elapsed() >= timeout { + let peak_before_kill = peak_memory_bytes_for_live_worker( + child.id(), + runtime_profile.memory_accounting_kind, + evaluation_cgroup.as_ref(), + ); + #[cfg(target_os = "linux")] + if let Some(cgroup) = evaluation_cgroup.as_ref() { + let _ = cgroup.kill_all(); + } + let _ = child.kill(); + let _ = child.wait(); + return Ok(timeout_eval_result( + effective_eval_time_limit_seconds, + peak_before_kill.map_err(fatal)?, + effective_eval_time_limit_seconds, + )); + } + std::thread::sleep(Duration::from_millis(1)); + } + + let payload_bytes = fs::read(&temp_paths.response_path).map_err(|err| { + fatal(format!( + "failed to read evaluator worker response '{}': {err}", + temp_paths.response_path.display() + )) + })?; + if payload_bytes.is_empty() { + return Err(CandidateEvalFailure::FatalEvaluatorFailure { + diagnostic: "candidate evaluation worker returned no payload".to_string(), + }); + } + let payload: Value = serde_json::from_slice(&payload_bytes).map_err(|err| { + fatal(format!( + "invalid evaluator payload from child process: {err}" + )) + })?; + let mut result = parse_candidate_eval_payload(&payload)?; + apply_authoritative_worker_peak_memory( + &mut result, + model_bytes, + min_throughput_bytes_per_second, + max_memory_bytes, + runtime_profile.memory_accounting_kind, + evaluation_cgroup.as_ref(), + ) + .map_err(fatal)?; + Ok(result) +} + +#[cfg(unix)] +fn parse_candidate_eval_payload( + payload: &Value, +) -> Result { + let fatal = |diagnostic: String| CandidateEvalFailure::FatalEvaluatorFailure { diagnostic }; + let object = payload + .as_object() + .ok_or_else(|| fatal("invalid evaluator payload shape".to_string()))?; + let ok = object + .get("ok") + .and_then(Value::as_bool) + .ok_or_else(|| fatal("evaluator payload missing boolean 'ok' field".to_string()))?; + if !ok { + let diagnostic = object + .get("error") + .and_then(Value::as_str) + .unwrap_or("candidate evaluator failed without error message") + .to_string(); + return Err(CandidateEvalFailure::FatalEvaluatorFailure { diagnostic }); + } + let status = match object + .get("status") + .and_then(Value::as_str) + .ok_or_else(|| fatal("evaluator payload missing status".to_string()))? + { + "success" => CandidateEvalStatus::Success, + "timeout" => CandidateEvalStatus::Timeout, + "invalid" => CandidateEvalStatus::Invalid, + "error" => CandidateEvalStatus::Error, + other => { + return Err(CandidateEvalFailure::FatalEvaluatorFailure { + diagnostic: format!("unknown evaluator status '{other}'"), + }); + } + }; + let compressed_bytes = object + .get("compressed_bytes") + .and_then(Value::as_u64) + .ok_or_else(|| fatal("evaluator payload missing compressed_bytes".to_string()))?; + let compressed_bytes = usize::try_from(compressed_bytes) + .map_err(|_| fatal("evaluator payload compressed_bytes does not fit usize".to_string()))?; + let elapsed_seconds = object + .get("elapsed_seconds") + .and_then(Value::as_f64) + .ok_or_else(|| fatal("evaluator payload missing elapsed_seconds".to_string()))?; + let effective_eval_time_limit_seconds = object + .get("effective_eval_time_limit_seconds") + .and_then(Value::as_f64) + .ok_or_else(|| { + fatal("evaluator payload missing effective_eval_time_limit_seconds".to_string()) + })?; + if !effective_eval_time_limit_seconds.is_finite() || effective_eval_time_limit_seconds < 0.0 { + return Err(CandidateEvalFailure::FatalEvaluatorFailure { + diagnostic: "evaluator payload has invalid effective_eval_time_limit_seconds" + .to_string(), + }); + } + let throughput_bytes_per_second = object + .get("throughput_bytes_per_second") + .and_then(|v| { + if v.is_null() { + Some(f64::INFINITY) + } else { + v.as_f64() + } + }) + .ok_or_else(|| { + fatal("evaluator payload missing throughput_bytes_per_second".to_string()) + })?; + let peak_memory_bytes = object + .get("peak_memory_bytes") + .and_then(Value::as_u64) + .ok_or_else(|| fatal("evaluator payload missing peak_memory_bytes".to_string()))?; + let target_loss_bits = object + .get("target_loss_bits") + .and_then(|v| { + if v.is_null() { + Some(f64::INFINITY) + } else { + v.as_f64() + } + }) + .ok_or_else(|| fatal("evaluator payload missing target_loss_bits".to_string()))?; + let objective_bits = object + .get("objective_bits") + .and_then(|v| { + if v.is_null() { + Some(f64::INFINITY) + } else { + v.as_f64() + } + }) + .ok_or_else(|| fatal("evaluator payload missing objective_bits".to_string()))?; + let deployable = object + .get("deployable") + .and_then(Value::as_bool) + .ok_or_else(|| fatal("evaluator payload missing deployable".to_string()))?; + Ok(CandidateEvalResult { + status, + compressed_bytes, + elapsed_seconds, + effective_eval_time_limit_seconds, + throughput_bytes_per_second, + peak_memory_bytes, + target_loss_bits, + objective_bits, + deployable, + }) +} + +/// Run the process-isolated tuner evaluator worker described by environment. +/// +/// The parent executor sets `INFOTHEORY_TUNER_EVAL_REQUEST_PATH` to a JSON +/// request and `INFOTHEORY_TUNER_EVAL_RESPONSE_PATH` to the file where this +/// worker must write its JSON response. The worker performs exactly one +/// candidate evaluation, serializes either an `ok: true` result payload or an +/// `ok: false` error payload, and returns only after the response has been +/// written. This entrypoint is public so the CLI binary and libtest worker shim +/// can share the same evaluator contract; it is not a canonical tune-spec API. +pub fn run_tuner_eval_worker_from_env() -> Result<(), String> { + if std::env::var_os("INFOTHEORY_TUNER_EVAL_WORKER_PING").as_deref() == Some(OsStr::new("1")) { + return Ok(()); + } + let request_path = std::env::var_os("INFOTHEORY_TUNER_EVAL_REQUEST_PATH") + .ok_or_else(|| "missing INFOTHEORY_TUNER_EVAL_REQUEST_PATH".to_string())?; + let response_path = std::env::var_os("INFOTHEORY_TUNER_EVAL_RESPONSE_PATH") + .ok_or_else(|| "missing INFOTHEORY_TUNER_EVAL_RESPONSE_PATH".to_string())?; + let request_bytes = fs::read(&request_path).map_err(|err| { + format!( + "failed to read evaluator worker request '{}': {err}", + PathBuf::from(&request_path).display() + ) + })?; + let request: Value = serde_json::from_slice(&request_bytes) + .map_err(|err| format!("invalid evaluator worker request JSON: {err}"))?; + let payload = match run_tuner_eval_worker_request(&request) { + Ok(result) => candidate_eval_result_payload(&result), + Err(err) => serde_json::json!({ + "ok": false, + "error": err, + }), + }; + fs::write( + &response_path, + serde_json::to_vec(&payload) + .map_err(|err| format!("failed to encode evaluator worker response: {err}"))?, + ) + .map_err(|err| { + format!( + "failed to write evaluator worker response '{}': {err}", + PathBuf::from(&response_path).display() + ) + }) +} + +fn run_tuner_eval_worker_request(payload: &Value) -> Result { + let object = payload + .as_object() + .ok_or_else(|| "evaluator worker request must be a JSON object".to_string())?; + let candidate_value = object + .get("candidate") + .ok_or_else(|| "evaluator worker request missing candidate".to_string())?; + let candidate_base_dir = object + .get("candidate_base_dir") + .and_then(Value::as_str) + .unwrap_or("."); + let candidate = crate::spec::parse_compression_backend_json( + candidate_value, + Path::new(candidate_base_dir), + None, + crate::compression::FramingMode::Framed, + ) + .map_err(|err| format!("failed to parse evaluator worker candidate: {err}"))? + .compile_in(&SpecEnvironment::new(candidate_base_dir)) + .map_err(|err| format!("failed to compile evaluator worker candidate: {err}"))?; + let dataset_path = required_worker_str(object, "dataset_path")?; + let dataset = load_dataset(Path::new(dataset_path))?; + let model_bytes = required_worker_usize(object, "model_bytes")?; + let min_throughput_bytes_per_second = + required_worker_nonnegative_f64(object, "min_throughput_bytes_per_second")?; + let max_memory_bytes = required_worker_u64(object, "max_memory_bytes")?; + let effective_eval_time_limit_seconds = + required_worker_nonnegative_f64(object, "effective_eval_time_limit_seconds")?; + let rss_mode = parse_worker_peak_memory_mode(required_worker_str(object, "rss_mode")?)?; + let evaluator_threads = required_worker_usize(object, "evaluator_threads")?; + if evaluator_threads == 0 { + return Err("evaluator_threads must be >= 1".to_string()); + } + rayon::ThreadPoolBuilder::new() + .num_threads(evaluator_threads) + .build_global() + .map_err(|err| format!("failed to initialize evaluator worker thread pool: {err}"))?; + match evaluate_candidate_unbounded( + &candidate, + &dataset, + model_bytes, + min_throughput_bytes_per_second, + max_memory_bytes, + effective_eval_time_limit_seconds, + rss_mode, + ) { + Ok(result) => Ok(result), + Err(WorkerInnerEvalFailure::CandidateLocal { .. }) => Ok(error_eval_result( + 0.0, + peak_memory_bytes(rss_mode), + effective_eval_time_limit_seconds, + )), + Err(WorkerInnerEvalFailure::Fatal { diagnostic }) => Err(diagnostic), + } +} + +fn candidate_eval_result_payload(value: &CandidateEvalResult) -> Value { + serde_json::json!({ + "ok": true, + "status": value.status.name(), + "compressed_bytes": value.compressed_bytes, + "elapsed_seconds": value.elapsed_seconds, + "effective_eval_time_limit_seconds": value.effective_eval_time_limit_seconds, + "throughput_bytes_per_second": value.throughput_bytes_per_second, + "peak_memory_bytes": value.peak_memory_bytes, + "target_loss_bits": value.target_loss_bits, + "objective_bits": value.objective_bits, + "deployable": value.deployable, + }) +} + +fn required_worker_str<'a>( + object: &'a serde_json::Map, + field: &str, +) -> Result<&'a str, String> { + object + .get(field) + .and_then(Value::as_str) + .ok_or_else(|| format!("evaluator worker request field '{field}' must be a string")) +} + +fn required_worker_u64( + object: &serde_json::Map, + field: &str, +) -> Result { + object.get(field).and_then(Value::as_u64).ok_or_else(|| { + format!("evaluator worker request field '{field}' must be an unsigned integer") + }) +} + +fn required_worker_usize( + object: &serde_json::Map, + field: &str, +) -> Result { + usize::try_from(required_worker_u64(object, field)?) + .map_err(|_| format!("evaluator worker request field '{field}' is too large")) +} + +fn required_worker_nonnegative_f64( + object: &serde_json::Map, + field: &str, +) -> Result { + let value = object + .get(field) + .and_then(Value::as_f64) + .ok_or_else(|| format!("evaluator worker request field '{field}' must be a number"))?; + if value.is_finite() && value >= 0.0 { + Ok(value) + } else { + Err(format!( + "evaluator worker request field '{field}' must be finite and nonnegative" + )) + } +} + +fn parse_worker_peak_memory_mode(raw: &str) -> Result { + match raw { + "process_rss_peak" => Ok(PeakMemoryMode::ProcessRssPeak), + "backend_reported" => Ok(PeakMemoryMode::BackendReported), + "hybrid_strict_max" => Ok(PeakMemoryMode::HybridStrictMax), + other => Err(format!("unknown evaluator worker rss_mode '{other}'")), + } +} + +#[cfg(unix)] +struct EvaluatorWorkerTempPaths { + request_path: PathBuf, + response_path: PathBuf, +} + +#[cfg(unix)] +impl EvaluatorWorkerTempPaths { + fn new(label: &str) -> Result { + let nonce = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|duration| duration.as_nanos()) + .unwrap_or_else(|_| Instant::now().elapsed().as_nanos()); + let digest = crc32_hex(label.as_bytes()); + let base = std::env::temp_dir().join(format!( + "infotheory-tuner-eval-worker-{}-{nonce}-{digest}", + std::process::id() + )); + fs::create_dir_all(&base).map_err(|err| { + format!( + "failed to create evaluator worker temp directory '{}': {err}", + base.display() + ) + })?; + Ok(Self { + request_path: base.join("request.json"), + response_path: base.join("response.json"), + }) + } + + fn cleanup(&self) -> Result<(), String> { + let Some(parent) = self.request_path.parent() else { + return Ok(()); + }; + fs::remove_dir_all(parent).map_err(|err| { + format!( + "failed to remove evaluator worker temp directory '{}': {err}", + parent.display() + ) + }) + } +} + +#[cfg(unix)] +impl Drop for EvaluatorWorkerTempPaths { + fn drop(&mut self) { + let _ = self.cleanup(); + } +} + +#[cfg(all(unix, target_os = "linux"))] +fn resolve_required_tuner_eval_cgroup_parent(explicit: Option<&str>) -> Result { + let explicit_path = explicit.map(PathBuf::from); + let env_path = std::env::var_os("INFOTHEORY_TUNER_EVAL_CGROUP_PARENT").map(PathBuf::from); + let path = if let Some(path) = explicit_path { + path + } else if let Some(path) = env_path { + path + } else { + return Err( + "strict memory-accounting mode (rss_mode=hybrid_strict_max) requires a delegated cgroup-v2 parent via execution.evaluator_cgroup_parent, --evaluator-cgroup-parent, or INFOTHEORY_TUNER_EVAL_CGROUP_PARENT" + .to_string(), + ); + }; + let canonical = validate_evaluator_cgroup_parent(&path)?; + probe_evaluator_cgroup_parent(&canonical)?; + Ok(canonical) +} + +#[cfg(all(unix, not(target_os = "linux")))] +fn resolve_required_tuner_eval_cgroup_parent(explicit: Option<&str>) -> Result { + let _ = explicit; + Err( + "strict memory-accounting mode (rss_mode=hybrid_strict_max) requires Linux cgroup-v2 per-evaluation accounting" + .to_string(), + ) +} + +#[cfg(all(unix, not(target_os = "linux")))] +fn reject_unix_fallback_cgroup_overrides(explicit: Option<&str>) -> Result<(), String> { + if explicit.is_some() || std::env::var_os("INFOTHEORY_TUNER_EVAL_CGROUP_PARENT").is_some() { + return Err("evaluator_cgroup_parent requires Linux cgroup v2".to_string()); + } + Ok(()) +} + +#[cfg(all(unix, target_os = "linux"))] +fn reject_unix_fallback_cgroup_overrides(explicit: Option<&str>) -> Result<(), String> { + if explicit.is_some() || std::env::var_os("INFOTHEORY_TUNER_EVAL_CGROUP_PARENT").is_some() { + return Err( + "evaluator_cgroup_parent is only valid for strict Linux cgroup-v2 memory accounting mode (rss_mode=hybrid_strict_max)" + .to_string(), + ); + } + Ok(()) +} + +#[cfg(unix)] +pub(super) fn resolve_tuner_eval_worker_executable( + explicit: Option<&str>, +) -> Result { + let explicit_path = explicit.map(PathBuf::from); + let executable = evaluator_worker_executable(explicit_path.as_deref())?; + probe_evaluator_worker_executable(&executable)?; + Ok(executable) +} + +#[cfg(unix)] +fn evaluator_worker_command( + explicit_worker_executable: Option<&Path>, +) -> Result { + let executable = evaluator_worker_executable(explicit_worker_executable)?; + let mut command = std::process::Command::new(&executable); + append_evaluator_worker_entrypoint(&mut command, &executable); + Ok(command) +} + +#[cfg(unix)] +fn evaluator_worker_executable(explicit: Option<&Path>) -> Result { + if let Some(path) = explicit { + return ensure_file_path(path.to_path_buf(), "execution.evaluator_worker_executable"); + } + if let Some(path) = std::env::var_os("INFOTHEORY_TUNER_EVAL_WORKER_EXE") { + return ensure_file_path(PathBuf::from(path), "INFOTHEORY_TUNER_EVAL_WORKER_EXE"); + } + if let Some(path) = std::env::var_os("CARGO_BIN_EXE_infotheory") { + let executable = PathBuf::from(path); + if executable.is_file() { + return Ok(executable); + } + } + let current = std::env::current_exe().map_err(|err| { + format!("failed to resolve evaluator worker executable from current_exe: {err}") + })?; + ensure_file_path(current, "current_exe") +} + +#[cfg(unix)] +fn ensure_file_path(path: PathBuf, label: &str) -> Result { + if path.is_file() { + Ok(path) + } else { + Err(format!( + "{label} '{}' does not resolve to a file", + path.display() + )) + } +} + +#[cfg(unix)] +fn worker_executable_identity(executable: &Path) -> Result { + let raw = fs::read(executable).map_err(|err| { + format!( + "failed to read evaluator worker executable '{}' for cache identity: {err}", + executable.display() + ) + })?; + Ok(format!("crc32:{}:bytes:{}", crc32_hex(&raw), raw.len())) +} + +#[cfg(unix)] +fn append_evaluator_worker_entrypoint(command: &mut std::process::Command, executable: &Path) { + if evaluator_worker_executable_is_libtest(executable) { + command + .arg("__infotheory_tuner_eval_worker") + .arg("--ignored") + .arg("--nocapture"); + } else { + command.arg("__infotheory-tuner-eval-worker"); + } +} + +#[cfg(unix)] +fn probe_evaluator_worker_executable(executable: &Path) -> Result<(), String> { + let mut command = std::process::Command::new(executable); + append_evaluator_worker_entrypoint(&mut command, executable); + let output = command + .env("INFOTHEORY_TUNER_EVAL_WORKER_PING", "1") + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::piped()) + .output() + .map_err(|err| { + format!( + "failed to probe evaluator worker executable '{}': {err}", + executable.display() + ) + })?; + if output.status.success() { + return Ok(()); + } + let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); + let detail = if stderr.is_empty() { + output.status.to_string() + } else { + stderr + }; + Err(format!( + "evaluator worker executable '{}' is not compatible with tuner worker entrypoint: {detail}", + executable.display() + )) +} + +#[cfg(unix)] +fn evaluator_worker_executable_is_libtest(path: &Path) -> bool { + path.parent() + .and_then(Path::file_name) + .and_then(|name| name.to_str()) + == Some("deps") +} + +#[cfg(target_os = "linux")] +static EVALUATOR_CGROUP_SEQUENCE: AtomicU64 = AtomicU64::new(0); + +#[cfg(target_os = "linux")] +struct EvaluatorWorkerCgroup { + path: PathBuf, +} + +#[cfg(all(unix, not(target_os = "linux")))] +struct EvaluatorWorkerCgroup; + +#[cfg(target_os = "linux")] +impl EvaluatorWorkerCgroup { + fn create(parent: &Path, label: &str) -> Result { + let sequence = EVALUATOR_CGROUP_SEQUENCE.fetch_add(1, Ordering::Relaxed); + let nonce = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|duration| duration.as_nanos()) + .unwrap_or_else(|_| Instant::now().elapsed().as_nanos()); + let digest = crc32_hex(label.as_bytes()); + let path = parent.join(format!( + "infotheory-eval-{}-{sequence}-{nonce}-{digest}", + std::process::id() + )); + fs::create_dir(&path).map_err(|err| { + format!( + "failed to create per-evaluation cgroup '{}': {err}", + path.display() + ) + })?; + let cgroup = Self { path }; + cgroup.peak_memory_bytes().map_err(|err| { + format!( + "created cgroup '{}' but could not read cgroup-v2 memory.peak: {err}", + cgroup.path.display() + ) + })?; + Ok(cgroup) + } + + fn configure_worker_command(&self, command: &mut std::process::Command) -> Result<(), String> { + let cgroup_procs_path = self.path.join("cgroup.procs"); + let cgroup_procs = std::fs::OpenOptions::new() + .write(true) + .open(&cgroup_procs_path) + .map_err(|err| { + format!( + "failed to open per-evaluation cgroup procs file '{}': {err}", + cgroup_procs_path.display() + ) + })?; + // SAFETY: `pre_exec` runs in the forked child immediately before exec. + // The closure captures an already-open `cgroup.procs` file descriptor and + // performs only async-signal-safe operations: `getpid`, in-bounds pointer + // arithmetic over a stack buffer, and `write`. This moves the child into + // the dedicated cgroup before the evaluator worker binary is exec'd, so + // spec parsing, runtime construction, and compression are all accounted in + // the candidate-local memory peak without running them as root. + unsafe { + command.pre_exec(move || { + let pid = libc::getpid(); + let mut buffer = [0u8; 32]; + let (start, len) = decimal_pid_line(pid as u64, &mut buffer); + let written = libc::write( + cgroup_procs.as_raw_fd(), + buffer.as_ptr().add(start).cast::(), + len, + ); + if written == len as isize { + Ok(()) + } else { + Err(std::io::Error::last_os_error()) + } + }); + } + Ok(()) + } + + fn peak_memory_bytes(&self) -> Result { + read_u64_from_file(&self.path.join("memory.peak")) + } + + fn kill_all(&self) -> Result<(), String> { + let kill_path = self.path.join("cgroup.kill"); + if !kill_path.exists() { + return Ok(()); + } + fs::write(&kill_path, b"1\n").map_err(|err| { + format!( + "failed to kill evaluator worker cgroup '{}': {err}", + kill_path.display() + ) + }) + } +} + +#[cfg(target_os = "linux")] +impl Drop for EvaluatorWorkerCgroup { + fn drop(&mut self) { + let _ = fs::remove_dir(&self.path); + } +} + +#[cfg(target_os = "linux")] +fn decimal_pid_line(mut value: u64, buffer: &mut [u8; 32]) -> (usize, usize) { + let mut start = buffer.len() - 1; + buffer[start] = b'\n'; + if value == 0 { + start -= 1; + buffer[start] = b'0'; + } else { + while value > 0 { + start -= 1; + buffer[start] = b'0' + (value % 10) as u8; + value /= 10; + } + } + (start, buffer.len() - start) +} + +#[cfg(target_os = "linux")] +fn validate_evaluator_cgroup_parent(path: &Path) -> Result { + let cgroup_root = Path::new("/sys/fs/cgroup") + .canonicalize() + .map_err(|err| format!("failed to resolve /sys/fs/cgroup: {err}"))?; + let canonical = path.canonicalize().map_err(|err| { + format!( + "failed to resolve evaluator cgroup parent '{}': {err}", + path.display() + ) + })?; + if !canonical.starts_with(&cgroup_root) { + return Err(format!( + "evaluator_cgroup_parent '{}' must be under '{}'", + canonical.display(), + cgroup_root.display() + )); + } + if !canonical.is_dir() { + return Err(format!( + "evaluator_cgroup_parent '{}' is not a directory", + canonical.display() + )); + } + Ok(canonical) +} + +#[cfg(target_os = "linux")] +fn probe_evaluator_cgroup_parent(parent: &Path) -> Result<(), String> { + let probe = EvaluatorWorkerCgroup::create(parent, "probe")?; + probe.peak_memory_bytes()?; + Ok(()) +} + +#[cfg(target_os = "linux")] +fn peak_memory_bytes_for_live_worker( + pid: u32, + accounting: ResolvedMemoryAccountingKind, + cgroup: Option<&EvaluatorWorkerCgroup>, +) -> Result { + let process = peak_rss_bytes_for_pid(pid).unwrap_or(0); + let cgroup_peak = cgroup + .map(EvaluatorWorkerCgroup::peak_memory_bytes) + .transpose()?; + Ok(match accounting { + ResolvedMemoryAccountingKind::DeterministicEvaluatorTable => process, + ResolvedMemoryAccountingKind::StrictLinuxCgroupV2PeakMaxProcessRss => cgroup_peak + .ok_or_else(|| { + "strict cgroup-v2 accounting expected per-evaluation cgroup peak".to_string() + })? + .max(process), + ResolvedMemoryAccountingKind::UnixProcessRssFallbackExplicit + | ResolvedMemoryAccountingKind::UnixProcessRssWithBackendReportedDiagnosticOnly => process, + }) +} + +#[cfg(all(unix, not(target_os = "linux")))] +fn peak_memory_bytes_for_live_worker( + pid: u32, + accounting: ResolvedMemoryAccountingKind, + _cgroup: Option<&EvaluatorWorkerCgroup>, +) -> Result { + let mode = match accounting { + ResolvedMemoryAccountingKind::UnixProcessRssFallbackExplicit + | ResolvedMemoryAccountingKind::UnixProcessRssWithBackendReportedDiagnosticOnly + | ResolvedMemoryAccountingKind::DeterministicEvaluatorTable => { + PeakMemoryMode::ProcessRssPeak + } + }; + Ok(peak_memory_bytes_for_pid(pid, mode).unwrap_or(0)) +} + +#[cfg(unix)] +fn apply_authoritative_worker_peak_memory( + result: &mut CandidateEvalResult, + model_bytes: usize, + min_throughput_bytes_per_second: f64, + max_memory_bytes: u64, + accounting: ResolvedMemoryAccountingKind, + cgroup: Option<&EvaluatorWorkerCgroup>, +) -> Result<(), String> { + apply_authoritative_worker_peak_memory_inner( + result, + model_bytes, + min_throughput_bytes_per_second, + max_memory_bytes, + accounting, + cgroup, + ) +} + +#[cfg(target_os = "linux")] +fn apply_authoritative_worker_peak_memory_inner( + result: &mut CandidateEvalResult, + model_bytes: usize, + min_throughput_bytes_per_second: f64, + max_memory_bytes: u64, + accounting: ResolvedMemoryAccountingKind, + cgroup: Option<&EvaluatorWorkerCgroup>, +) -> Result<(), String> { + match accounting { + ResolvedMemoryAccountingKind::DeterministicEvaluatorTable + | ResolvedMemoryAccountingKind::UnixProcessRssFallbackExplicit + | ResolvedMemoryAccountingKind::UnixProcessRssWithBackendReportedDiagnosticOnly => {} + ResolvedMemoryAccountingKind::StrictLinuxCgroupV2PeakMaxProcessRss => { + let cgroup_peak = cgroup + .ok_or_else(|| { + "strict cgroup-v2 accounting expected per-evaluation cgroup handle".to_string() + })? + .peak_memory_bytes()?; + result.peak_memory_bytes = result.peak_memory_bytes.max(cgroup_peak); + } + }; + if result.status == CandidateEvalStatus::Success { + result.deployable = result.throughput_bytes_per_second >= min_throughput_bytes_per_second + && result.peak_memory_bytes <= max_memory_bytes; + result.objective_bits = if result.deployable { + ((model_bytes as f64) * 8.0) + result.target_loss_bits + } else { + f64::INFINITY + }; + } else { + result.deployable = false; + result.objective_bits = f64::INFINITY; + } + Ok(()) +} + +#[cfg(all(unix, not(target_os = "linux")))] +fn apply_authoritative_worker_peak_memory_inner( + result: &mut CandidateEvalResult, + _model_bytes: usize, + _min_throughput_bytes_per_second: f64, + _max_memory_bytes: u64, + _accounting: ResolvedMemoryAccountingKind, + _cgroup: Option<&EvaluatorWorkerCgroup>, +) -> Result<(), String> { + let _ = result; + Ok(()) +} + +#[derive(Clone, Debug, Eq, PartialEq)] +enum WorkerInnerEvalFailure { + CandidateLocal { diagnostic: String }, + Fatal { diagnostic: String }, +} + +impl WorkerInnerEvalFailure { + fn candidate_local(diagnostic: impl Into) -> Self { + Self::CandidateLocal { + diagnostic: diagnostic.into(), + } + } + + fn fatal(diagnostic: impl Into) -> Self { + Self::Fatal { + diagnostic: diagnostic.into(), + } + } +} + +fn evaluate_candidate_unbounded( + candidate: &crate::spec::CompiledCompressionBackend, + dataset: &LoadedDataset, + model_bytes: usize, + min_throughput_bytes_per_second: f64, + max_memory_bytes: u64, + effective_eval_time_limit_seconds: f64, + rss_mode: PeakMemoryMode, +) -> Result { + let before_peak = peak_memory_bytes(rss_mode); + let start = Instant::now(); + let deadline = start + Duration::from_secs_f64(effective_eval_time_limit_seconds); + let (compressed_bytes, target_loss_bits) = match dataset.kind { + DatasetKind::PassiveBytes => { + let mut runtime = + crate::runtime::build_compression_runtime(candidate).map_err(|err| { + WorkerInnerEvalFailure::candidate_local(format!( + "failed to build candidate runtime: {err}" + )) + })?; + let compressed_bytes_u64 = + runtime.compress_size(&dataset.raw_bytes).map_err(|err| { + WorkerInnerEvalFailure::candidate_local(format!( + "candidate evaluation failed: {err}" + )) + })?; + let compressed_bytes = usize::try_from(compressed_bytes_u64).map_err(|_| { + WorkerInnerEvalFailure::fatal("compressed size does not fit usize on this platform") + })?; + (compressed_bytes, (compressed_bytes as f64) * 8.0) + } + DatasetKind::InteractiveTrace | DatasetKind::CausalPrefixDataset => { + evaluate_candidate_causal_loss_typed(candidate, dataset, deadline)? + } + }; + let elapsed_seconds = start.elapsed().as_secs_f64(); + let after_peak = peak_memory_bytes(rss_mode); + let peak_memory_bytes = after_peak.max(before_peak); + if elapsed_seconds >= effective_eval_time_limit_seconds || target_loss_bits.is_infinite() { + return Ok(timeout_eval_result( + elapsed_seconds, + peak_memory_bytes, + effective_eval_time_limit_seconds, + )); + } + + let throughput_bytes_per_second = if elapsed_seconds <= 0.0 { + f64::INFINITY + } else { + dataset.dataset_units / elapsed_seconds + }; + let deployable = throughput_bytes_per_second >= min_throughput_bytes_per_second + && peak_memory_bytes <= max_memory_bytes; + let objective_bits = if deployable { + ((model_bytes as f64) * 8.0) + target_loss_bits + } else { + f64::INFINITY + }; + + Ok(CandidateEvalResult { + status: CandidateEvalStatus::Success, + compressed_bytes, + elapsed_seconds, + effective_eval_time_limit_seconds, + throughput_bytes_per_second, + peak_memory_bytes, + target_loss_bits, + objective_bits, + deployable, + }) +} + +pub(super) fn timeout_eval_result( + elapsed_seconds: f64, + peak_memory_bytes: u64, + effective_eval_time_limit_seconds: f64, +) -> CandidateEvalResult { + CandidateEvalResult { + status: CandidateEvalStatus::Timeout, + compressed_bytes: 0, + elapsed_seconds, + effective_eval_time_limit_seconds, + throughput_bytes_per_second: 0.0, + peak_memory_bytes, + target_loss_bits: f64::INFINITY, + objective_bits: f64::INFINITY, + deployable: false, + } +} + +pub(super) fn error_eval_result( + elapsed_seconds: f64, + peak_memory_bytes: u64, + effective_eval_time_limit_seconds: f64, +) -> CandidateEvalResult { + CandidateEvalResult { + status: CandidateEvalStatus::Error, + compressed_bytes: 0, + elapsed_seconds, + effective_eval_time_limit_seconds, + throughput_bytes_per_second: 0.0, + peak_memory_bytes, + target_loss_bits: f64::INFINITY, + objective_bits: f64::INFINITY, + deployable: false, + } +} + +#[cfg(test)] +pub(super) fn evaluate_candidate_causal_loss( + candidate: &crate::spec::CompiledCompressionBackend, + dataset: &LoadedDataset, + deadline: Instant, +) -> Result<(usize, f64), String> { + evaluate_candidate_causal_loss_typed(candidate, dataset, deadline).map_err( + |error| match error { + WorkerInnerEvalFailure::CandidateLocal { diagnostic } + | WorkerInnerEvalFailure::Fatal { diagnostic } => diagnostic, + }, + ) +} + +fn evaluate_candidate_causal_loss_typed( + candidate: &crate::spec::CompiledCompressionBackend, + dataset: &LoadedDataset, + deadline: Instant, +) -> Result<(usize, f64), WorkerInnerEvalFailure> { + if let crate::spec::core::CompressionBackendPlan::Rate { rate_backend, .. } = candidate.plan() { + let causal_profile = dataset.causal_profile.as_ref().ok_or_else(|| { + WorkerInnerEvalFailure::fatal( + "causal dataset evaluation requires a typed causal profile", + ) + })?; + let compiled_rate = crate::spec::core::compiled_rate_backend_from_plan( + rate_backend.clone(), + ) + .map_err(|err| { + WorkerInnerEvalFailure::candidate_local(format!( + "failed to compile causal evaluator rate backend: {err}" + )) + })?; + let mut prefix_parts = Vec::>::new(); + let mut target_loss_bits = 0.0f64; + for event in &dataset.events { + if Instant::now() >= deadline { + return Ok((0, f64::INFINITY)); + } + match event { + LoweredCausalEvent::Reset => prefix_parts.clear(), + LoweredCausalEvent::Context { channel, bytes } => { + prefix_parts.push(causal_event_conditioning_bytes( + "context", channel, None, bytes, + )); + } + LoweredCausalEvent::ObserveTargetNoScore { + channel, + domain, + bytes, + } => { + prefix_parts.push(causal_event_conditioning_bytes( + "observe_target_no_score", + channel, + Some(domain), + bytes, + )); + } + LoweredCausalEvent::Target { + channel, + domain, + bytes, + weight, + } => { + let support = causal_profile.domains.get(domain).ok_or_else(|| { + WorkerInnerEvalFailure::fatal(format!( + "target event references undeclared domain '{domain}'" + )) + })?; + let descriptor = causal_event_descriptor_bytes("target", channel, Some(domain)); + let refs = prefix_parts + .iter() + .map(Vec::as_slice) + .collect::>(); + let loss = causal_target_loss_bits( + &refs, + &descriptor, + bytes, + support, + &compiled_rate, + )?; + target_loss_bits += (*weight) * loss; + prefix_parts.push(causal_event_conditioning_bytes( + "target", + channel, + Some(domain), + bytes, + )); + } + } + } + let compressed_bytes = (target_loss_bits / 8.0).ceil().max(0.0) as usize; + Ok((compressed_bytes, target_loss_bits)) + } else { + Err(WorkerInnerEvalFailure::candidate_local( + "causal dataset evaluation requires a rate backend with conditional target-loss semantics", + )) + } +} + +fn causal_target_loss_bits( + prefix_parts: &[&[u8]], + descriptor: &[u8], + target: &[u8], + support: &CausalTargetDomain, + compiled_rate: &crate::spec::CompiledRateBackend, +) -> Result { + let mut descriptor_conditioned = Vec::<&[u8]>::with_capacity(prefix_parts.len() + 1); + descriptor_conditioned.extend_from_slice(prefix_parts); + descriptor_conditioned.push(descriptor); + match support { + CausalTargetDomain::ByteAlphabet => { + if target.len() != 1 { + return Err(WorkerInnerEvalFailure::fatal( + "byte_alphabet target payloads must be exactly one byte after lowering", + )); + } + crate::runtime::try_cross_entropy_conditional_chain_backend( + &descriptor_conditioned, + target, + compiled_rate, + ) + .map_err(|err| { + WorkerInnerEvalFailure::candidate_local(format!( + "causal byte-domain target evaluation failed: {err}" + )) + }) + } + CausalTargetDomain::EnumeratedPayloads { payloads } => { + if !payloads.iter().any(|payload| payload == target) { + return Err(WorkerInnerEvalFailure::fatal( + "target payload is outside enumerated target-domain support", + )); + } + let mut target_loss = None::; + let mut log2_terms = Vec::::with_capacity(payloads.len()); + for payload in payloads { + let loss = crate::runtime::try_cross_entropy_conditional_chain_backend( + &descriptor_conditioned, + payload, + compiled_rate, + ) + .map_err(|err| { + WorkerInnerEvalFailure::candidate_local(format!( + "causal enumerated-domain target evaluation failed: {err}" + )) + })?; + if payload == target { + target_loss = Some(loss); + } + log2_terms.push(-loss); + } + let log2_z = log2_sum_exp(&log2_terms); + let loss = target_loss.ok_or_else(|| { + WorkerInnerEvalFailure::fatal( + "target payload is outside enumerated target-domain support", + ) + })?; + Ok(loss + log2_z) + } + } +} + +fn log2_sum_exp(log2_terms: &[f64]) -> f64 { + let max_term = log2_terms.iter().copied().fold(f64::NEG_INFINITY, f64::max); + if !max_term.is_finite() { + return max_term; + } + let sum = log2_terms + .iter() + .map(|term| 2.0f64.powf(*term - max_term)) + .sum::(); + max_term + sum.log2() +} + +fn causal_event_conditioning_bytes( + kind: &str, + channel: &str, + domain: Option<&str>, + payload: &[u8], +) -> Vec { + let mut out = causal_event_descriptor_bytes(kind, channel, domain); + out.extend_from_slice(&(payload.len() as u64).to_le_bytes()); + out.extend_from_slice(payload); + out +} + +fn causal_event_descriptor_bytes(kind: &str, channel: &str, domain: Option<&str>) -> Vec { + let mut out = Vec::::new(); + out.extend_from_slice(b"infotheory:tuner:causal-event:v1\0"); + push_tag_component(&mut out, kind.as_bytes()); + push_tag_component(&mut out, channel.as_bytes()); + push_tag_component(&mut out, domain.unwrap_or("").as_bytes()); + out +} + +fn push_tag_component(out: &mut Vec, component: &[u8]) { + out.extend_from_slice(&(component.len() as u64).to_le_bytes()); + out.extend_from_slice(component); +} + +pub(super) fn cache_key_for_candidate( + candidate_bytes: &[u8], + evaluator_profile: &EvaluatorProfile, + dataset_hash: &str, +) -> Result { + let evaluator_profile_bytes = evaluator_profile.cache_identity_bytes()?; + Ok(CandidateCacheKey { + candidate_canonical_bytes: candidate_bytes.to_vec(), + evaluator_profile_bytes, + dataset_identity: dataset_hash.to_string(), + }) +} + +#[cfg(all(test, unix))] +mod tests { + use super::*; + use crate::api::{CompressionBackend, RateBackend}; + use crate::coders::CoderType; + use crate::compression::FramingMode; + use crate::spec::SpecEnvironment; + + fn compile_candidate(candidate: CompressionBackend) -> crate::spec::CompiledCompressionBackend { + candidate + .compile_in(&SpecEnvironment::new(".")) + .expect("compile test candidate") + } + + fn causal_dataset_without_profile() -> LoadedDataset { + LoadedDataset { + kind: DatasetKind::CausalPrefixDataset, + objective_target: ObjectiveTarget::InteractiveCausalAc, + lowering_version: "causal-prefix-events-v1", + codec_hash: "test-codec".to_string(), + event_grammar_hash: "test-grammar".to_string(), + target_domain_support_hash: "test-domain".to_string(), + causal_header_profile_hash: "test-causal-header".to_string(), + target_size_function: "causal-target-bits", + canonical_content_hash: "test-dataset".to_string(), + lowered_skeleton_hash: "test-skeleton".to_string(), + resolved_path: ".".to_string(), + source_size_bytes: 0, + raw_bytes: Vec::new(), + events: Vec::new(), + causal_profile: None, + dataset_units: 1.0, + target_events: 0, + } + } + + #[test] + fn parse_worker_payload_accepts_null_serialized_infinities() { + let payload = serde_json::json!({ + "ok": true, + "status": "success", + "compressed_bytes": 0, + "elapsed_seconds": 0.0, + "effective_eval_time_limit_seconds": 1.0, + "throughput_bytes_per_second": null, + "peak_memory_bytes": 0, + "target_loss_bits": 0.0, + "objective_bits": 8.0, + "deployable": true + }); + + let result = parse_candidate_eval_payload(&payload).expect("parse worker payload"); + + assert_eq!(result.status, CandidateEvalStatus::Success); + assert!(result.throughput_bytes_per_second.is_infinite()); + assert!(result.throughput_bytes_per_second.is_sign_positive()); + + let payload = serde_json::json!({ + "ok": true, + "status": "timeout", + "compressed_bytes": 0, + "elapsed_seconds": 1.0, + "effective_eval_time_limit_seconds": 1.0, + "throughput_bytes_per_second": 0.0, + "peak_memory_bytes": 0, + "target_loss_bits": null, + "objective_bits": null, + "deployable": false + }); + + let result = parse_candidate_eval_payload(&payload).expect("parse worker payload"); + + assert_eq!(result.status, CandidateEvalStatus::Timeout); + assert!(result.target_loss_bits.is_infinite()); + assert!(result.objective_bits.is_infinite()); + } + + #[test] + fn parse_worker_payload_ok_false_is_fatal_evaluator_failure() { + let payload = serde_json::json!({ + "ok": false, + "error": "worker dataset load failed", + }); + + let err = parse_candidate_eval_payload(&payload).expect_err("ok:false must be fatal"); + + match err { + CandidateEvalFailure::FatalEvaluatorFailure { diagnostic } => { + assert_eq!(diagnostic, "worker dataset load failed"); + } + } + } + + #[test] + fn evaluate_candidate_unbounded_dataset_invariant_error_is_fatal() { + let candidate = compile_candidate(CompressionBackend::Rate { + rate_backend: RateBackend::Ctw { depth: 2 }, + coder: CoderType::AC, + framing: FramingMode::Framed, + }); + let dataset = causal_dataset_without_profile(); + + let err = evaluate_candidate_unbounded( + &candidate, + &dataset, + 0, + 0.0, + u64::MAX, + 1.0, + PeakMemoryMode::ProcessRssPeak, + ) + .expect_err("missing causal profile must be fatal"); + + match err { + WorkerInnerEvalFailure::Fatal { diagnostic } => { + assert!(diagnostic.contains("typed causal profile"), "{diagnostic}"); + } + WorkerInnerEvalFailure::CandidateLocal { diagnostic } => { + panic!("expected fatal invariant error, got candidate-local: {diagnostic}"); + } + } + } + + #[test] + fn causal_profile_invariant_error_is_fatal() { + let candidate = compile_candidate(CompressionBackend::Rate { + rate_backend: RateBackend::Ctw { depth: 2 }, + coder: CoderType::AC, + framing: FramingMode::Framed, + }); + let dataset = causal_dataset_without_profile(); + let deadline = Instant::now() + Duration::from_millis(100); + + let err = evaluate_candidate_causal_loss_typed(&candidate, &dataset, deadline) + .expect_err("causal profile invariant failure must be fatal"); + + match err { + WorkerInnerEvalFailure::Fatal { diagnostic } => { + assert!(diagnostic.contains("typed causal profile"), "{diagnostic}"); + } + WorkerInnerEvalFailure::CandidateLocal { diagnostic } => { + panic!("expected fatal invariant error, got candidate-local: {diagnostic}"); + } + } + } + + #[cfg(feature = "backend-zpaq")] + #[test] + fn candidate_backend_rejection_is_recoverable_status_error() { + let candidate = compile_candidate(CompressionBackend::zpaq("3")); + let dataset = causal_dataset_without_profile(); + + let err = evaluate_candidate_unbounded( + &candidate, + &dataset, + 0, + 0.0, + u64::MAX, + 1.0, + PeakMemoryMode::ProcessRssPeak, + ) + .expect_err("candidate/dataset mismatch must be candidate-local"); + + match err { + WorkerInnerEvalFailure::CandidateLocal { diagnostic } => { + assert!( + diagnostic + .contains("requires a rate backend with conditional target-loss semantics"), + "{diagnostic}" + ); + } + WorkerInnerEvalFailure::Fatal { diagnostic } => { + panic!("expected candidate-local rejection, got fatal: {diagnostic}"); + } + } + } + + #[test] + fn worker_inner_fatal_error_emits_ok_false_or_fatal_response() { + let candidate = compile_candidate(CompressionBackend::Rate { + rate_backend: RateBackend::Ctw { depth: 2 }, + coder: CoderType::AC, + framing: FramingMode::Framed, + }); + let candidate_json = + crate::spec::compression_backend_to_json_value(candidate.canonical_spec()) + .expect("serialize candidate"); + let request = serde_json::json!({ + "candidate": candidate_json, + "candidate_base_dir": ".", + "dataset_path": "/path/that/does/not/exist.bin", + "model_bytes": 1, + "min_throughput_bytes_per_second": 1.0, + "max_memory_bytes": 1024, + "effective_eval_time_limit_seconds": 1.0, + "rss_mode": "process_rss_peak", + "evaluator_threads": 1 + }); + let payload = match run_tuner_eval_worker_request(&request) { + Ok(result) => candidate_eval_result_payload(&result), + Err(err) => serde_json::json!({ + "ok": false, + "error": err, + }), + }; + + assert_eq!(payload.get("ok").and_then(Value::as_bool), Some(false)); + assert!( + payload + .get("error") + .and_then(Value::as_str) + .is_some_and(|value| value.contains("failed to read")), + "expected read failure diagnostic in fatal worker response: {payload}" + ); + } +} diff --git a/crates/infotheory/src/tuner/planner_bridge.rs b/crates/infotheory/src/tuner/planner_bridge.rs new file mode 100644 index 00000000..7fd34b1c --- /dev/null +++ b/crates/infotheory/src/tuner/planner_bridge.rs @@ -0,0 +1,1305 @@ +use super::*; + +#[allow(clippy::too_many_arguments)] +pub(super) fn run_planner_family_controller( + compiled: &crate::spec::CompiledTuneSpec, + request: &TuneCommandRequest, + dataset: &LoadedDataset, + evaluator_profile: &EvaluatorProfile, + verified_theorem: &VerifiedTheoremInputs, + tune_started: Instant, + baseline_eval: CandidateEvalResult, + baseline_hash: String, + baseline_bytes: Vec, + baseline_key: CandidateCacheKey, + runtime_profile: &ResolvedEvaluatorRuntimeProfile, + cache: &mut HashMap, +) -> Result { + let contract = + planner_controller_contract(compiled.controller(), compiled, dataset, verified_theorem)?; + let reward_encoder = contract.reward_encoder(dataset, &baseline_eval, verified_theorem)?; + let actions = compile_planner_mutation_actions( + compiled.canonical_spec().baseline_candidate.clone(), + &compiled.canonical_spec().bounds, + )?; + validate_theorem_planner_mutation_domain(&actions, &request.execution.theorem)?; + let declared_actions = contract.interface.agent_actions.get(); + if actions.len() != declared_actions { + return Err(format!( + "action_alphabet_mismatch: compiled mutation alphabet has {} actions but controller.interface.agent_actions declares {}", + actions.len(), + declared_actions + )); + } + let planner_run = compile_tuner_planner_run_spec( + compiled.controller(), + &contract, + &reward_encoder, + compiled, + &SpecEnvironment::new(compiled.base_dir()), + )?; + if let Some(teacher) = &contract.teacher { + crate::aixi::warmstart::validate_warmstart_teacher_dataset_for_compiled_planner_run( + &planner_run, + &teacher.traces, + ) + .map_err(|err| err.to_string())?; + } + let mut agent_runtime = build_tuner_planner_agent_runtime( + compiled.controller(), + &planner_run, + &contract, + encode_tuner_planner_percept( + &contract.interface, + Some(&baseline_eval), + dataset.dataset_units, + 0, + "baseline_initial_state", + Some(&baseline_eval), + Some(&baseline_bytes), + Some(compiled.canonical_spec().eval_time_limit_seconds), + false, + )?, + )?; + let env = SpecEnvironment::new(compiled.base_dir()); + + let mut best_candidate = compiled.canonical_spec().baseline_candidate.clone(); + let mut best_eval = baseline_eval.clone(); + let mut best_hash = baseline_hash; + let mut best_bytes = baseline_bytes; + let mut best_key = baseline_key; + let mut current_candidate = best_candidate.clone(); + let mut current_eval = baseline_eval; + let mut candidate_result_counts = CandidateResultCounts::default(); + candidate_result_counts.record_admitted_result(¤t_eval); + let mut current_bytes = best_bytes.clone(); + let mut cache_hits: usize = 0; + let mut cache_misses: usize = 1; + let mut candidate_evaluations_executed: usize = 1; + let mut proposals_attempted: usize = 0; + let mut proposals_invalid: usize = 0; + let mut self_loop_proposals: usize = 0; + let mut invalid_reason_counts = InvalidReasonCounts::default(); + let mut successful_non_deployable: usize = 0; + let mut final_best_move_reward: f64 = 0.0; + let mut evaluations_seen: usize = 1; + let max_evaluations = request.execution.max_evaluations.unwrap_or(usize::MAX); + let mut stagnation_counter: usize = 0; + let mut decision_steps: usize = 0; + let mut warmstart_trace_refresh_merges: usize = 0; + let mut fatal_evaluator_failure: Option = None; + let total_rounds = if contract.warmstart_self_improvement { + request.execution.self_improvement_rounds.max(1) + } else { + 1 + }; + let mut realized_trace_counts_by_round = vec![0usize; total_rounds]; + let mut trace_refresh_merges_by_round = vec![0usize; total_rounds]; + let trace_refresh_enabled = contract.warmstart_self_improvement + && request.execution.warmstart_trace_refresh + && total_rounds > 1; + let mut refresh_teacher = contract + .teacher + .as_ref() + .map(|teacher| teacher.traces.clone()); + let planner_return_bins = planner_return_bins(&planner_run); + + 'rounds: for round in 0..total_rounds { + if round > 0 + && let Some(live_trace) = agent_runtime.same_task_live_trace() + { + let previous_round = round - 1; + realized_trace_counts_by_round[previous_round] = live_trace.transitions.len(); + if trace_refresh_enabled && let Some(teacher) = refresh_teacher.as_mut() { + let records_before = teacher_trace_record_count(teacher); + let inserted = merge_warmstart_trace_deterministic(teacher, live_trace)?; + let records_after = teacher_trace_record_count(teacher); + trace_refresh_merges_by_round[previous_round] = + records_after.saturating_sub(records_before); + if inserted { + crate::aixi::warmstart::validate_warmstart_teacher_dataset_for_compiled_planner_run( + &planner_run, + teacher, + ) + .map_err(|err| err.to_string())?; + agent_runtime.rebuild_warmstart_agent(&planner_run, teacher.clone())?; + warmstart_trace_refresh_merges = + warmstart_trace_refresh_merges.saturating_add(1); + } + } + } + let round_deadline_seconds = if contract.warmstart_self_improvement && total_rounds > 1 { + Some( + ((round + 1) as f64 / total_rounds as f64) + * compiled.canonical_spec().time_budget_seconds, + ) + } else { + None + }; + loop { + let elapsed_seconds = tune_started.elapsed().as_secs_f64(); + if evaluations_seen >= max_evaluations + || elapsed_seconds >= compiled.canonical_spec().time_budget_seconds + || round_deadline_seconds + .map(|deadline| elapsed_seconds >= deadline) + .unwrap_or(false) + { + break; + } + let action = agent_runtime.select_action(); + proposals_attempted = proposals_attempted.saturating_add(1); + let action_index = usize::try_from(action) + .map_err(|_| format!("planner action {action} does not fit usize"))?; + if action_index >= actions.len() { + proposals_invalid = proposals_invalid.saturating_add(1); + invalid_reason_counts.record(TuneInvalidReason::InvalidActionIndex); + let percept = encode_tuner_planner_percept( + &contract.interface, + Some(¤t_eval), + dataset.dataset_units, + 0, + TuneInvalidReason::InvalidActionIndex.as_str(), + None, + None, + None, + false, + )?; + agent_runtime.observe_transition(action, percept)?; + decision_steps = decision_steps.saturating_add(1); + continue; + } + let Some(proposed_candidate) = + apply_planner_mutation_action(¤t_candidate, &actions[action_index])? + else { + self_loop_proposals = self_loop_proposals.saturating_add(1); + invalid_reason_counts.record(TuneInvalidReason::InapplicableAction); + let percept = encode_tuner_planner_percept( + &contract.interface, + Some(¤t_eval), + dataset.dataset_units, + 0, + TuneInvalidReason::InapplicableAction.as_str(), + None, + None, + None, + false, + )?; + agent_runtime.observe_transition(action, percept)?; + decision_steps = decision_steps.saturating_add(1); + continue; + }; + if let Err(err) = reject_candidate_local_external_artifacts(&proposed_candidate) { + proposals_invalid = proposals_invalid.saturating_add(1); + invalid_reason_counts.record(err.reason); + let percept = encode_tuner_planner_percept( + &contract.interface, + Some(¤t_eval), + dataset.dataset_units, + 0, + err.reason.as_str(), + None, + None, + None, + false, + )?; + agent_runtime.observe_transition(action, percept)?; + decision_steps = decision_steps.saturating_add(1); + continue; + } + if validate_candidate_against_tune_bounds( + &proposed_candidate, + &compiled.canonical_spec().bounds, + ) + .is_err() + { + proposals_invalid = proposals_invalid.saturating_add(1); + invalid_reason_counts.record(TuneInvalidReason::CandidateOutOfBounds); + let percept = encode_tuner_planner_percept( + &contract.interface, + Some(¤t_eval), + dataset.dataset_units, + 0, + TuneInvalidReason::CandidateOutOfBounds.as_str(), + None, + None, + None, + false, + )?; + agent_runtime.observe_transition(action, percept)?; + decision_steps = decision_steps.saturating_add(1); + continue; + } + let compiled_candidate = match proposed_candidate.compile_in(&env) { + Ok(value) => value, + Err(_) => { + proposals_invalid = proposals_invalid.saturating_add(1); + invalid_reason_counts.record(TuneInvalidReason::CandidateCompileError); + let percept = encode_tuner_planner_percept( + &contract.interface, + Some(¤t_eval), + dataset.dataset_units, + 0, + TuneInvalidReason::CandidateCompileError.as_str(), + None, + None, + None, + false, + )?; + agent_runtime.observe_transition(action, percept)?; + decision_steps = decision_steps.saturating_add(1); + continue; + } + }; + let candidate_bytes = compiled_candidate.canonical_bytes().as_slice().to_vec(); + let candidate_hash = crc32_hex(&candidate_bytes); + let effective_limit = effective_eval_limit_seconds( + compiled, + tune_started, + Some(compiled.canonical_spec().eval_time_limit_seconds), + round_deadline_seconds, + ); + if effective_limit <= 0.0 { + break; + } + let candidate_profile = evaluator_profile.with_eval_time_limit(effective_limit); + if candidate_bytes == current_bytes { + self_loop_proposals = self_loop_proposals.saturating_add(1); + let percept = encode_tuner_planner_percept( + &contract.interface, + Some(¤t_eval), + dataset.dataset_units, + 0, + "self_loop_proposal", + None, + Some(&candidate_bytes), + None, + false, + )?; + agent_runtime.observe_transition(action, percept)?; + decision_steps = decision_steps.saturating_add(1); + continue; + } + let cache_key = cache_key_for_candidate( + compiled_candidate.canonical_bytes().as_slice(), + &candidate_profile, + &dataset.canonical_content_hash, + )?; + let candidate_eval = if let Some(cached) = cache.get(&cache_key) { + cache_hits = cache_hits.saturating_add(1); + cached.clone() + } else { + let evaluated = match evaluate_candidate( + &compiled_candidate, + dataset, + compiled_candidate.canonical_bytes().len(), + compiled.canonical_spec().min_throughput_bytes_per_second, + compiled.canonical_spec().max_memory_bytes, + effective_limit, + request.execution.evaluator_threads(), + runtime_profile, + verified_theorem.deterministic_table.as_ref(), + ) { + Ok(value) => value, + Err(CandidateEvalFailure::FatalEvaluatorFailure { diagnostic }) => { + fatal_evaluator_failure = Some(diagnostic); + break 'rounds; + } + }; + cache.insert(cache_key.clone(), evaluated.clone()); + cache_misses = cache_misses.saturating_add(1); + candidate_evaluations_executed = candidate_evaluations_executed.saturating_add(1); + evaluated + }; + evaluations_seen = evaluations_seen.saturating_add(1); + candidate_result_counts.record_admitted_result(&candidate_eval); + let incumbent_eval_before_step = current_eval.clone(); + let mut raw_improvement = 0.0f64; + if candidate_eval.status == CandidateEvalStatus::Success && !candidate_eval.deployable { + successful_non_deployable = successful_non_deployable.saturating_add(1); + } + if candidate_eval.deployable { + if key_less( + &candidate_eval, + &candidate_bytes, + ¤t_eval, + ¤t_bytes, + ) { + raw_improvement = + (current_eval.objective_bits - candidate_eval.objective_bits).max(0.0); + current_candidate = proposed_candidate.clone(); + current_eval = candidate_eval.clone(); + current_bytes = candidate_bytes.clone(); + } + if key_less(&candidate_eval, &candidate_bytes, &best_eval, &best_bytes) { + final_best_move_reward = raw_improvement; + best_candidate = proposed_candidate; + best_eval = candidate_eval.clone(); + best_hash = candidate_hash; + best_bytes = candidate_bytes.clone(); + best_key = cache_key; + stagnation_counter = 0; + } else { + stagnation_counter = stagnation_counter.saturating_add(1); + } + if let Some(reset_after) = request.execution.stagnation_reset_evals + && stagnation_counter >= reset_after + { + current_candidate = best_candidate.clone(); + current_eval = best_eval.clone(); + current_bytes = best_bytes.clone(); + stagnation_counter = 0; + } + } + let reward = reward_encoder.encode(raw_improvement)?; + let diagnostic_token = if !candidate_eval.deployable { + match candidate_eval.status { + CandidateEvalStatus::Timeout => "evaluator_timeout", + CandidateEvalStatus::Invalid => "evaluator_invalid", + CandidateEvalStatus::Error => "evaluator_error", + CandidateEvalStatus::Success => "nondeployable_candidate", + } + } else { + "deployable_success" + }; + let percept = encode_tuner_planner_percept( + &contract.interface, + Some(&incumbent_eval_before_step), + dataset.dataset_units, + reward, + diagnostic_token, + Some(&candidate_eval), + Some(&candidate_bytes), + Some(effective_limit), + false, + )?; + agent_runtime.observe_transition(action, percept)?; + decision_steps = decision_steps.saturating_add(1); + } + } + + if contract.warmstart_self_improvement + && total_rounds > 0 + && let Some(live_trace) = agent_runtime.same_task_live_trace() + { + realized_trace_counts_by_round[total_rounds - 1] = live_trace.transitions.len(); + } + + let status = if fatal_evaluator_failure.is_some() { + "terminated_unrecoverable_evaluator_failure" + } else { + planner_completed_status(compiled.controller()) + }; + let warning = fatal_evaluator_failure + .as_ref() + .map(|_| "terminated due to unrecoverable evaluator failure".to_string()); + + Ok(SearchSummary { + status, + warning, + fatal_evaluator_failure: fatal_evaluator_failure.clone(), + fatal_evaluator_failures: usize::from(fatal_evaluator_failure.is_some()), + best_candidate, + best_candidate_crc32: best_hash, + best_eval, + cache_key_digest: best_key.digest_crc32(), + cache_hits, + cache_misses, + candidate_evaluations_executed, + non_warmup_candidate_results_seen: evaluations_seen, + post_baseline_candidate_results_seen: evaluations_seen.saturating_sub(1), + proposals_attempted, + proposals_invalid, + self_loop_proposals, + invalid_reason_counts, + successful_non_deployable, + candidate_result_counts, + final_best_move_reward, + realized_trace_counts_by_round: if contract.warmstart_self_improvement { + Some(realized_trace_counts_by_round.clone()) + } else { + None + }, + trace_refresh_merges_by_round: if contract.warmstart_self_improvement { + Some(trace_refresh_merges_by_round.clone()) + } else { + None + }, + controller_report: serde_json::json!({ + "kind": controller_kind_name(compiled.controller()), + "runtime_path": planner_runtime_path_name(compiled.controller()), + "planner_simulations_per_step": contract.planner_simulations_per_step, + "simulations_per_decision_step": contract.planner_simulations_per_step, + "decision_steps": decision_steps, + "return_horizon": contract.return_horizon, + "return_bins": planner_return_bins, + "label_phase_period": contract.label_phase_period, + "reward_semantics": contract.reward_semantics.name(), + "reward_encoding": tuner_reward_encoding_name(&reward_encoder), + "discount_factor": contract.discount_factor, + "planner_run_controller_kind": planner_run.controller().kind_str(), + "agent_runtime": planner_agent_runtime_name(compiled.controller()), + "compiled_action_count": actions.len(), + "compiled_action_paths": planner_action_paths(&actions), + "declared_agent_actions": declared_actions, + "observation_adapter": OBSERVATION_ADAPTER_DECLARATION, + "scalar_representation": SCALAR_REPRESENTATION_DECLARATION, + "warmstart_teacher_dataset": contract.teacher.as_ref().map(|value| serde_json::json!({ + "asset_id": value.asset_id.clone(), + "resolved_path": value.resolved_path.clone(), + "content_crc32": value.content_hash.clone(), + "records": value.records, + })), + "warmstart_self_improvement_update": if contract.warmstart_self_improvement { + if trace_refresh_enabled { + Some("same_task_trace_refresh_rebuild") + } else { + Some("online_exact_h_step_delayed_label_update") + } + } else { + None::<&str> + }, + "warmstart_trace_refresh_merges": warmstart_trace_refresh_merges, + "warmstart_trace_refresh_merged_records_total": trace_refresh_merges_by_round + .iter() + .copied() + .sum::(), + }), + }) +} + +pub(super) fn planner_controller_contract( + controller: &crate::spec::CompiledTuneController, + compiled: &crate::spec::CompiledTuneSpec, + _dataset: &LoadedDataset, + verified_theorem: &VerifiedTheoremInputs, +) -> Result { + match controller { + crate::spec::CompiledTuneController::McAixiFacCtw(inner) => Ok(PlannerControllerContract { + interface: inner.interface.clone(), + planner_simulations_per_step: inner.planner_simulations_per_step, + return_horizon: None, + label_phase_period: None, + discount_factor: 1.0, + reward_semantics: PlannerRewardSemantics::ExactObjectiveDifference, + clipping_interval: None, + teacher: None, + warmstart_self_improvement: false, + }), + crate::spec::CompiledTuneController::AiqiDiscounted(inner) => { + if inner.max_improvement <= inner.min_improvement { + return Err( + "normalized clipped reward contract requires max_improvement > min_improvement" + .to_string(), + ); + } + Ok(PlannerControllerContract { + interface: inner.interface.clone(), + planner_simulations_per_step: inner.planner_simulations_per_step, + return_horizon: Some(inner.return_horizon), + label_phase_period: None, + discount_factor: inner.discount_factor, + reward_semantics: PlannerRewardSemantics::NormalizedClippedImprovement, + clipping_interval: Some((inner.min_improvement, inner.max_improvement)), + teacher: None, + warmstart_self_improvement: false, + }) + } + crate::spec::CompiledTuneController::AiqiWarmstartExactJh(inner) => { + let reward_certificate = verified_theorem.exact_reward_encoding.as_ref().ok_or_else(|| { + "reward_encoding_unsafe: warm-start exact-J_H requires a verified exact_reward_encoding_certificate" + .to_string() + })?; + if !reward_certificate.is_identity_or_interval_encoding() { + return Err( + "reward_encoding_unsafe: warm-start exact-J_H cannot use a non-identity finite_reward_map until teacher/live traces carry objective-difference labels or a verified decoder-based return encoder" + .to_string(), + ); + } + let teacher = load_warmstart_teacher_dataset( + compiled, + verified_theorem, + &inner.warmstart_teacher_dataset_asset, + &inner.interface, + inner.return_horizon, + inner.label_phase_period, + )?; + Ok(PlannerControllerContract { + interface: inner.interface.clone(), + planner_simulations_per_step: inner.planner_simulations_per_step, + return_horizon: Some(inner.return_horizon), + label_phase_period: Some(inner.label_phase_period), + discount_factor: 1.0, + reward_semantics: PlannerRewardSemantics::ExactObjectiveDifference, + clipping_interval: None, + teacher: Some(teacher), + warmstart_self_improvement: true, + }) + } + crate::spec::CompiledTuneController::AnnealedHillClimbing(_) => { + Err("annealed controller does not use planner-family contract".to_string()) + } + } +} + +fn load_warmstart_teacher_dataset( + compiled: &crate::spec::CompiledTuneSpec, + verified_theorem: &VerifiedTheoremInputs, + asset_id: &str, + interface: &crate::spec::TunePlannerInterfaceSpec, + return_horizon: usize, + label_phase_period: usize, +) -> Result { + if asset_id == compiled.canonical_spec().input_asset { + return Err( + "warmstart_teacher_dataset_asset must be distinct from input_asset".to_string(), + ); + } + let binding = compiled + .resolved_assets() + .iter() + .find(|entry| entry.id == asset_id) + .ok_or_else(|| format!("unknown warmstart teacher dataset asset '{asset_id}'"))?; + let AssetRef::Filesystem(path) = &binding.asset; + let bytes = fs::read(path).map_err(|err| { + format!( + "failed to read warmstart_teacher_dataset_asset '{}': {err}", + path.display() + ) + })?; + let hash = crc32_hex(&bytes); + let traces = WarmStartExactJhTeacherDataset::from_json_slice(&bytes) + .map_err(|err| format!("invalid warmstart_teacher_dataset_asset: {err}"))?; + validate_warmstart_teacher_contract( + verified_theorem, + interface, + return_horizon, + label_phase_period, + &traces, + )?; + let records = traces + .traces + .iter() + .map(|trace| trace.transitions.len()) + .sum(); + Ok(WarmstartTeacherDataset { + asset_id: asset_id.to_string(), + resolved_path: path.to_string_lossy().to_string(), + content_hash: hash, + records, + traces, + }) +} + +fn teacher_trace_record_count(dataset: &WarmStartExactJhTeacherDataset) -> usize { + dataset + .traces + .iter() + .map(|trace| trace.transitions.len()) + .sum() +} + +pub(super) fn validate_warmstart_teacher_contract( + verified_theorem: &VerifiedTheoremInputs, + interface: &crate::spec::TunePlannerInterfaceSpec, + return_horizon: usize, + label_phase_period: usize, + teacher: &WarmStartExactJhTeacherDataset, +) -> Result<(), String> { + let contract = &teacher.contract; + if contract.schema_version != 1 { + return Err("warmstart teacher schema_version must be 1".to_string()); + } + if contract.action_alphabet_size != interface.agent_actions.get() { + return Err(format!( + "warmstart teacher action_alphabet_size {} does not match configured {}", + contract.action_alphabet_size, + interface.agent_actions.get() + )); + } + if contract.observation_bits != interface.observation_bits { + return Err(format!( + "warmstart teacher observation_bits {} does not match configured {}", + contract.observation_bits, interface.observation_bits + )); + } + if contract.observation_stream_len != interface.observation_stream_len.max(1) { + return Err(format!( + "warmstart teacher observation_stream_len {} does not match configured {}", + contract.observation_stream_len, + interface.observation_stream_len.max(1) + )); + } + if contract.observation_key_mode != observation_key_mode_name(interface.observation_key_mode) { + return Err(format!( + "warmstart teacher observation_key_mode '{}' does not match configured '{}'", + contract.observation_key_mode, + observation_key_mode_name(interface.observation_key_mode) + )); + } + if contract.reward_bits != interface.reward_bits { + return Err(format!( + "warmstart teacher reward_bits {} does not match configured {}", + contract.reward_bits, interface.reward_bits + )); + } + if contract.return_horizon != return_horizon { + return Err(format!( + "warmstart teacher return_horizon {} does not match configured {}", + contract.return_horizon, return_horizon + )); + } + if contract.label_phase_period != label_phase_period { + return Err(format!( + "warmstart teacher label_phase_period {} does not match configured {}", + contract.label_phase_period, label_phase_period + )); + } + let expected_adapter_ref = OBSERVATION_ADAPTER_DECLARATION; + let expected_adapter_hash = observation_adapter_content_hash()?; + if contract.observation_adapter_spec_ref != expected_adapter_ref { + return Err(format!( + "warmstart teacher observation adapter fingerprint does not match current tuner observation adapter: observation_adapter_spec_ref '{}' does not match expected '{}'", + contract.observation_adapter_spec_ref, expected_adapter_ref + )); + } + if contract.observation_adapter_content_crc32 != expected_adapter_hash { + return Err(format!( + "warmstart teacher observation adapter fingerprint does not match current tuner observation adapter: observation_adapter_content_crc32 '{}' does not match expected '{}'", + contract.observation_adapter_content_crc32, expected_adapter_hash + )); + } + let reward_cert = verified_theorem + .exact_reward_encoding + .as_ref() + .ok_or_else(|| { + "warmstart exact-J_H requires a verified exact_reward_encoding_certificate".to_string() + })?; + if contract.scalar_representation != reward_cert.scalar_representation { + return Err(format!( + "warmstart teacher scalar_representation '{}' does not match verified encoder '{}'", + contract.scalar_representation, reward_cert.scalar_representation + )); + } + if contract.exact_reward_encoding_certificate != reward_cert.base.content_hash { + return Err(format!( + "warmstart teacher exact_reward_encoding_certificate '{}' does not match verified encoder '{}'", + contract.exact_reward_encoding_certificate, reward_cert.base.content_hash + )); + } + Ok(()) +} + +fn compile_planner_mutation_actions( + baseline: crate::api::CompressionBackend, + bounds: &crate::spec::TuneBoundsSpec, +) -> Result, String> { + let json = crate::spec::compression_backend_to_json_value(&baseline) + .map_err(|err| format!("failed to serialize baseline for action compilation: {err}"))?; + let range_map = bounds + .parameter_ranges + .iter() + .map(|range| (range.parameter.clone(), (range.min, range.max))) + .collect::>(); + let mut leaves = collect_numeric_leaves(&json); + if !range_map.is_empty() { + leaves.retain(|leaf| range_map.contains_key(&leaf.path)); + } + let mut actions = Vec::::new(); + for leaf in leaves { + let deltas = match leaf.kind { + NumericKind::Unsigned | NumericKind::Signed => [-1.0, 1.0], + NumericKind::Float => [-0.05, 0.05], + }; + for delta in deltas { + actions.push(PlannerMutationAction::NumericStep { + path: leaf.path.clone(), + pointer: leaf.pointer.clone(), + kind: leaf.kind, + delta, + range: range_map.get(&leaf.path).copied(), + }); + } + } + if actions.is_empty() { + actions.push(PlannerMutationAction::Noop); + } + Ok(actions) +} + +pub(super) fn apply_planner_mutation_action( + candidate: &crate::api::CompressionBackend, + action: &PlannerMutationAction, +) -> Result, String> { + let PlannerMutationAction::NumericStep { + path: _, + pointer, + kind, + delta, + range, + } = action + else { + return Ok(None); + }; + let mut json = crate::spec::compression_backend_to_json_value(candidate) + .map_err(|err| format!("failed to serialize candidate for planner action: {err}"))?; + let Some(slot) = json.pointer_mut(pointer) else { + return Ok(None); + }; + if !apply_numeric_delta(slot, *kind, *delta, *range) { + return Ok(None); + } + let Ok(mutated) = crate::spec::parse_compression_backend_json( + &json, + Path::new("."), + None, + crate::compression::FramingMode::Framed, + ) else { + // Planner-family mutation decoding is totalized: a syntactically + // invalid edit is an inapplicable action rather than a fatal run + // abort. + return Ok(None); + }; + Ok(Some(mutated)) +} + +fn apply_numeric_delta( + slot: &mut Value, + kind: NumericKind, + delta: f64, + range: Option<(f64, f64)>, +) -> bool { + match kind { + NumericKind::Unsigned => { + let Some(current) = slot.as_u64() else { + return false; + }; + let step = delta.abs().ceil() as u64; + let next = if delta >= 0.0 { + current.checked_add(step) + } else { + Some(current.saturating_sub(step)) + }; + let Some(next) = next else { + return false; + }; + if let Some((min, max)) = range { + let min = min.ceil(); + let max = max.floor(); + if !min.is_finite() || !max.is_finite() { + return false; + } + let next_f64 = next as f64; + if next_f64 < min || next_f64 > max { + return false; + } + }; + if next == current { + return false; + } + *slot = Value::Number(serde_json::Number::from(next)); + true + } + NumericKind::Signed => { + let Some(current) = slot.as_i64() else { + return false; + }; + let step = delta.abs().ceil() as i64; + let next = if delta >= 0.0 { + current.checked_add(step) + } else { + current.checked_sub(step) + }; + let Some(next) = next else { + return false; + }; + if let Some((min, max)) = range { + let min = min.ceil(); + let max = max.floor(); + if !min.is_finite() || !max.is_finite() { + return false; + } + let next_f64 = next as f64; + if next_f64 < min || next_f64 > max { + return false; + } + }; + if next == current { + return false; + } + *slot = Value::Number(serde_json::Number::from(next)); + true + } + NumericKind::Float => { + let Some(current) = slot.as_f64() else { + return false; + }; + let scale = match range { + Some((min, max)) => { + let span = max - min; + if !span.is_finite() || span <= 0.0 { + return false; + } + current.abs().max(span) + } + None => current.abs().max(1.0), + }; + let next = current + scale * delta; + if !next.is_finite() || (next - current).abs() <= f64::EPSILON { + return false; + } + if let Some((min, max)) = range + && (next < min || next > max) + { + return false; + } + if let Some(number) = serde_json::Number::from_f64(next) { + *slot = Value::Number(number); + true + } else { + false + } + } + } +} + +fn planner_action_paths(actions: &[PlannerMutationAction]) -> Vec { + actions + .iter() + .map(|action| match action { + PlannerMutationAction::NumericStep { path, delta, .. } => { + format!("{path}:{delta:+}") + } + PlannerMutationAction::Noop => "noop".to_string(), + }) + .collect() +} + +pub(super) fn validate_theorem_planner_mutation_domain( + actions: &[PlannerMutationAction], + theorem: &TuneTheoremConfig, +) -> Result<(), String> { + if !theorem_requests_exact_claims(theorem) { + return Ok(()); + } + if let Some(path) = actions.iter().find_map(|action| match action { + PlannerMutationAction::NumericStep { + path, + kind: NumericKind::Float, + .. + } => Some(path.as_str()), + _ => None, + }) { + return Err(format!( + "theorem_finite_state_unsafe: planner mutation action '{path}' targets a floating-point leaf; exact theorem claims require an integer finite mutation grammar or a future certificate-enumerated finite float domain" + )); + } + Ok(()) +} + +fn theorem_requests_exact_claims(theorem: &TuneTheoremConfig) -> bool { + theorem.claim_exact_finite_mdp + || theorem.claim_exact_observed_markov + || theorem.claim_planner_convergence +} + +pub(super) fn compile_tuner_planner_run_spec( + controller: &crate::spec::CompiledTuneController, + contract: &PlannerControllerContract, + reward_encoder: &TunerRewardEncoder, + compiled: &crate::spec::CompiledTuneSpec, + env: &SpecEnvironment, +) -> Result { + let interface = PlannerInterfaceSpec { + observation_bits: contract.interface.observation_bits, + observation_stream_len: contract.interface.observation_stream_len, + observation_key_mode: contract.interface.observation_key_mode, + reward_bits: contract.interface.reward_bits, + agent_actions: contract.interface.agent_actions, + }; + let percept_bits = interface + .observation_bits + .saturating_mul(interface.observation_stream_len.max(1)) + .saturating_add(interface.reward_bits) + .max(1); + let controller_spec = match controller { + crate::spec::CompiledTuneController::McAixiFacCtw(inner) => { + ControllerSpec::McAixi(McAixiControllerSpec { + predictor: RateBackend::FacCtw { + base_depth: TUNER_MCAIXI_FAC_CTW_BASE_DEPTH, + num_percept_bits: percept_bits, + encoding_bits: 1, + msb_first: None, + }, + bit_stream_semantics: crate::api::BitStreamSemantics::BinaryTokens, + agent_horizon: TUNER_MCAIXI_HORIZON, + num_simulations: inner.planner_simulations_per_step, + mcts_strategy: MctsStrategy::RhoUct, + exploration_exploitation_ratio: 1.4, + discount_gamma: 1.0, + }) + } + crate::spec::CompiledTuneController::AiqiDiscounted(inner) => { + ControllerSpec::AiqiDiscounted(AiqiDiscountedControllerSpec { + predictor: RateBackend::Ctw { depth: 8 }, + bit_stream_semantics: crate::api::BitStreamSemantics::BinaryTokens, + discount_gamma: inner.discount_factor, + return_horizon: inner.return_horizon, + return_bins: inner.return_bins, + augmentation_period: inner.return_horizon, + history_prune_keep_steps: None, + baseline_exploration: 1.0e-12, + }) + } + crate::spec::CompiledTuneController::AiqiWarmstartExactJh(inner) => { + let return_bins = + warmstart_return_bins(reward_encoder.max_reward(), inner.return_horizon)?; + ControllerSpec::AiqiWarmstartExactJh(WarmStartExactJhControllerSpec { + predictor: RateBackend::Ctw { depth: 8 }, + bit_stream_semantics: crate::api::BitStreamSemantics::BinaryTokens, + return_horizon: inner.return_horizon, + return_bins, + label_phase_period: inner.label_phase_period, + teacher_dataset_asset: inner.warmstart_teacher_dataset_asset.clone(), + planner_simulations_per_step: inner.planner_simulations_per_step, + }) + } + crate::spec::CompiledTuneController::AnnealedHillClimbing(_) => { + return Err("annealed controller does not compile to planner-run agent".to_string()); + } + }; + PlannerRunSpec { + assets: compiled.canonical_spec().assets.clone(), + environment: EnvironmentSpec::Builtin { + builtin: BuiltinEnvironmentSpec::TunerBridge, + }, + interface, + controller: controller_spec, + runtime: PlannerRuntimeSpec { + random_seed: Some(compiled.canonical_spec().seed), + learn_cycles: None, + eval_cycles: None, + terminate_lifetime: 1, + log_every: 1, + perf: false, + vm_perf_only: false, + explore_epsilon: 0.0, + explore_gamma: 1.0, + }, + } + .compile_in(env) + .map_err(|err| format!("failed to compile tuner planner-run bridge: {err}")) +} + +fn build_tuner_planner_agent_runtime( + controller: &crate::spec::CompiledTuneController, + planner_run: &CompiledPlannerRunSpec, + contract: &PlannerControllerContract, + initial_percept: PlannerEncodedPercept, +) -> Result { + match controller { + crate::spec::CompiledTuneController::McAixiFacCtw(_) => { + Ok(TunerPlannerAgentRuntime::McAixi { + agent: Agent::from_compiled_planner_run(planner_run) + .map_err(|err| err.to_string())?, + prev_action: 0, + prev_percept: initial_percept, + }) + } + crate::spec::CompiledTuneController::AiqiDiscounted(_) => { + Ok(TunerPlannerAgentRuntime::AiqiDiscounted { + agent: AiqiAgent::from_compiled_planner_run(planner_run) + .map_err(|err| err.to_string())?, + }) + } + crate::spec::CompiledTuneController::AiqiWarmstartExactJh(_) => { + let teacher = contract + .teacher + .as_ref() + .ok_or_else(|| "warm-start controller missing teacher dataset".to_string())?; + Ok(TunerPlannerAgentRuntime::WarmStartExactJh { + agent: WarmStartExactJhAgent::from_compiled_planner_run( + planner_run, + teacher.traces.clone(), + ) + .map_err(|err| err.to_string())?, + }) + } + crate::spec::CompiledTuneController::AnnealedHillClimbing(_) => { + Err("annealed controller does not instantiate planner-family agents".to_string()) + } + } +} + +pub(super) fn merge_warmstart_trace_deterministic( + teacher: &mut WarmStartExactJhTeacherDataset, + trace: WarmStartExactJhTeacherTrace, +) -> Result { + Ok(merge_warmstart_teacher_trace_deterministic( + &mut teacher.traces, + trace, + )) +} + +#[allow(clippy::too_many_arguments)] +pub(super) fn encode_tuner_planner_percept( + interface: &crate::spec::TunePlannerInterfaceSpec, + incumbent_eval: Option<&CandidateEvalResult>, + dataset_units: f64, + reward: Reward, + diagnostic_token: &str, + candidate_eval: Option<&CandidateEvalResult>, + candidate_bytes: Option<&[u8]>, + effective_eval_limit_seconds: Option, + terminal: bool, +) -> Result { + let raw = TunerRawObservation::from_runtime_step( + incumbent_eval, + dataset_units, + candidate_eval, + candidate_bytes, + effective_eval_limit_seconds, + diagnostic_token, + terminal, + ); + let stream_len = interface.observation_stream_len.max(1); + let mut observations = Vec::with_capacity(stream_len); + for index in 0..stream_len { + observations.push(packed_observation_symbol( + interface.observation_bits, + index, + raw.encoded_bytes(), + )); + } + Ok(PlannerEncodedPercept { + observations, + reward, + }) +} + +pub(super) struct TunerRawObservation { + encoded: Vec, +} + +impl TunerRawObservation { + #[allow(clippy::too_many_arguments)] + pub(super) fn from_runtime_step( + incumbent_eval: Option<&CandidateEvalResult>, + dataset_units: f64, + candidate_eval: Option<&CandidateEvalResult>, + candidate_bytes: Option<&[u8]>, + effective_eval_limit_seconds: Option, + diagnostic_token: &str, + terminal: bool, + ) -> Self { + let units = if dataset_units.is_finite() && dataset_units > 0.0 { + dataset_units + } else { + 1.0 + }; + let fail_flag = candidate_eval + .map(|value| value.status != CandidateEvalStatus::Success || !value.deployable) + .unwrap_or(true); + let successful_eval = + candidate_eval.filter(|value| value.status == CandidateEvalStatus::Success); + let normalized_physical_size = + successful_eval.map(|value| (value.compressed_bytes as f64) / units); + let normalized_target_loss = successful_eval.and_then(|value| { + if value.target_loss_bits.is_finite() { + Some((value.target_loss_bits / units).max(0.0)) + } else { + None + } + }); + let normalized_eval_time = match candidate_eval { + Some(value) if value.status == CandidateEvalStatus::Timeout => Some(1.0), + Some(value) if value.status == CandidateEvalStatus::Success => Some( + normalize_eval_time(value.elapsed_seconds, effective_eval_limit_seconds), + ), + _ => None, + }; + let physical_size_delta = match (incumbent_eval, successful_eval) { + (Some(incumbent), Some(value)) + if incumbent.status == CandidateEvalStatus::Success + && incumbent.compressed_bytes > 0 => + { + Some( + (incumbent.compressed_bytes as f64 - value.compressed_bytes as f64) + / incumbent.compressed_bytes as f64, + ) + } + _ => None, + }; + let eval_time_delta = match (incumbent_eval, successful_eval) { + (Some(incumbent), Some(value)) + if incumbent.status == CandidateEvalStatus::Success + && incumbent.elapsed_seconds.is_finite() + && value.elapsed_seconds.is_finite() => + { + let denom = incumbent.elapsed_seconds.max(1.0e-9); + Some((incumbent.elapsed_seconds - value.elapsed_seconds) / denom) + } + _ => None, + }; + let signature_source = candidate_bytes.unwrap_or(diagnostic_token.as_bytes()); + let candidate_signature_crc32 = crc32_u32(signature_source); + let mut encoded = Vec::::with_capacity(64); + encoded.push(u8::from(fail_flag)); + push_optional_f64(&mut encoded, normalized_physical_size); + push_optional_f64(&mut encoded, normalized_target_loss); + push_optional_f64(&mut encoded, normalized_eval_time); + push_optional_f64(&mut encoded, physical_size_delta); + push_optional_f64(&mut encoded, eval_time_delta); + encoded.extend_from_slice(&candidate_signature_crc32.to_le_bytes()); + encoded.push(u8::from(terminal)); + Self { encoded } + } + + pub(super) fn encoded_bytes(&self) -> &[u8] { + &self.encoded + } +} + +fn normalize_eval_time(elapsed_seconds: f64, effective_eval_limit_seconds: Option) -> f64 { + if let Some(limit) = effective_eval_limit_seconds + && limit > 0.0 + { + return (elapsed_seconds / limit).clamp(0.0, 1.0); + } + if elapsed_seconds.is_finite() { + elapsed_seconds.max(0.0) + } else { + 1.0 + } +} + +fn push_optional_f64(out: &mut Vec, value: Option) { + match value { + Some(number) => { + out.push(1); + out.extend_from_slice(&number.to_bits().to_le_bytes()); + } + None => out.push(0), + } +} + +fn packed_observation_symbol( + observation_bits: usize, + index: usize, + raw_payload: &[u8], +) -> PerceptVal { + if observation_bits == 0 { + return 0; + } + let offset = index.saturating_mul(std::mem::size_of::()); + let mut bytes = [0_u8; 8]; + if offset < raw_payload.len() { + let available = (raw_payload.len() - offset).min(bytes.len()); + bytes[..available].copy_from_slice(&raw_payload[offset..offset + available]); + } else { + bytes[0] = 0xff; + } + let value = u64::from_le_bytes(bytes); + if observation_bits >= 64 { + value + } else { + value & ((1u64 << observation_bits) - 1) + } +} + +fn crc32_u32(bytes: &[u8]) -> u32 { + let mut hasher = Hasher::new(); + hasher.update(bytes); + hasher.finalize() +} + +pub(super) fn exact_nonnegative_i64_from_f64(value: f64, label: &str) -> Result { + if !value.is_finite() { + return Err(format!("{label} must be finite")); + } + if value < 0.0 { + return Err(format!("{label} must be nonnegative")); + } + let rounded = value.round(); + if (rounded - value).abs() > f64::EPSILON { + return Err(format!( + "reward_encoding_unsafe: {label} must be exactly representable as a finite nonnegative integer" + )); + } + if rounded > (Reward::MAX as f64) { + return Err(format!( + "reward_encoding_unsafe: {label} exceeds maximum representable reward" + )); + } + Ok(rounded as Reward) +} + +fn warmstart_return_bins(max_reward: Reward, return_horizon: usize) -> Result { + let max_total = (max_reward as u128) + .checked_mul(return_horizon as u128) + .and_then(|value| value.checked_add(1)) + .ok_or_else(|| "warm-start exact-J_H return label range overflowed".to_string())?; + usize::try_from(max_total) + .map_err(|_| "warm-start exact-J_H return label range does not fit usize".to_string()) +} + +fn planner_return_bins(planner_run: &CompiledPlannerRunSpec) -> Option { + match planner_run.controller() { + crate::spec::CompiledPlannerController::AiqiDiscounted { return_bins, .. } + | crate::spec::CompiledPlannerController::AiqiWarmstartExactJh { return_bins, .. } => { + Some(*return_bins) + } + crate::spec::CompiledPlannerController::McAixi { .. } => None, + } +} + +fn planner_agent_runtime_name(controller: &crate::spec::CompiledTuneController) -> &'static str { + match controller { + crate::spec::CompiledTuneController::McAixiFacCtw(_) => "aixi::agent::Agent", + crate::spec::CompiledTuneController::AiqiDiscounted(_) => "aixi::aiqi::AiqiAgent", + crate::spec::CompiledTuneController::AiqiWarmstartExactJh(_) => { + "aixi::warmstart::WarmStartExactJhAgent" + } + crate::spec::CompiledTuneController::AnnealedHillClimbing(_) => "annealer", + } +} + +fn tuner_reward_encoding_name(encoder: &TunerRewardEncoder) -> &'static str { + match encoder { + TunerRewardEncoder::ExactIntegerObjectiveDifference { + objective_difference_to_symbol: Some(_), + .. + } => "exact_finite_reward_symbol_map", + TunerRewardEncoder::ExactIntegerObjectiveDifference { .. } => { + "exact_integer_objective_difference_interval" + } + TunerRewardEncoder::NormalizedClipped { .. } => "rounded_normalized_clipped_scalar", + } +} + +pub(super) fn normalized_clipped_improvement( + raw_improvement: f64, + min_improvement: f64, + max_improvement: f64, +) -> Result { + if !(min_improvement.is_finite() + && max_improvement.is_finite() + && max_improvement > min_improvement) + { + return Err( + "normalized clipped improvement requires finite max_improvement > min_improvement" + .to_string(), + ); + } + Ok(((raw_improvement - min_improvement) / (max_improvement - min_improvement)).clamp(0.0, 1.0)) +} + +pub(super) fn key_less( + candidate_eval: &CandidateEvalResult, + candidate_bytes: &[u8], + incumbent_eval: &CandidateEvalResult, + incumbent_bytes: &[u8], +) -> bool { + candidate_eval.objective_bits < incumbent_eval.objective_bits + || (candidate_eval.objective_bits == incumbent_eval.objective_bits + && candidate_bytes < incumbent_bytes) +} diff --git a/crates/infotheory/src/tuner/report.rs b/crates/infotheory/src/tuner/report.rs new file mode 100644 index 00000000..4fb18277 --- /dev/null +++ b/crates/infotheory/src/tuner/report.rs @@ -0,0 +1,488 @@ +use super::*; + +pub(super) use crate::aixi::warmstart_contract::observation_key_mode_name; + +pub(super) fn executor_controls_report( + config: &TuneExecutionConfig, + runtime_profile: &ResolvedEvaluatorRuntimeProfile, +) -> Value { + let container_peak_available = cgroup_peak_memory_bytes().is_some(); + let per_eval_cgroup_peak_available = runtime_profile.resolved_evaluator_cgroup_parent.is_some(); + serde_json::json!({ + "cpu_affinity": { + "requested": config.cpu_affinity, + "applied_to_current_process": config.cpu_affinity.is_some(), + }, + "threads": { + "requested": config.threads, + "parent_controller_threads": 1usize, + "evaluator_threads": config.evaluator_threads(), + "rayon_global_pool_configured_in_parent": false, + "worker_isolation_mode": "spawn_exec_worker", + "evaluator_worker_executable": config.evaluator_worker_executable, + "evaluator_worker_executable_identity": runtime_profile + .worker_executable_identity + .as_deref(), + "evaluator_cgroup_parent_requested": config.evaluator_cgroup_parent, + "evaluator_cgroup_parent_resolved": runtime_profile.resolved_cgroup_parent_string(), + "evaluator_determinism": config.evaluator_determinism(), + }, + "log_path": config.log_path, + "diagnostic_chunk_bytes": config.diagnostic_chunk_bytes, + "rss_mode": { + "requested": peak_memory_mode_name(config.rss_mode), + "effective_measurement": runtime_profile.memory_accounting_kind.name(), + "strict_theorem_memory_certified": runtime_profile.strict_theorem_memory_certified(), + "per_eval_cgroup_peak_memory_available": per_eval_cgroup_peak_available, + "container_peak_memory_visible": container_peak_available, + "backend_report_component_policy": runtime_profile + .memory_accounting_kind + .backend_report_component_policy(), + }, + }) +} + +pub(super) fn diagnostic_chunking_report( + dataset: &LoadedDataset, + chunk_bytes: Option, +) -> Value { + let charged_payload_bytes = dataset.raw_bytes.len(); + let Some(chunk_bytes) = chunk_bytes else { + return serde_json::json!({ + "enabled": false, + "chunk_bytes": null, + "charged_payload_bytes": charged_payload_bytes, + "chunk_count": 0usize, + "last_chunk_bytes": 0usize, + "affects_objective": false, + "affects_canonical_candidate_identity": false, + }); + }; + let chunk_count = if charged_payload_bytes == 0 { + 0usize + } else { + (charged_payload_bytes / chunk_bytes) + + usize::from(!charged_payload_bytes.is_multiple_of(chunk_bytes)) + }; + let last_chunk_bytes = if charged_payload_bytes == 0 { + 0usize + } else { + let remainder = charged_payload_bytes % chunk_bytes; + if remainder == 0 { + chunk_bytes + } else { + remainder + } + }; + serde_json::json!({ + "enabled": true, + "chunk_bytes": chunk_bytes, + "charged_payload_bytes": charged_payload_bytes, + "chunk_count": chunk_count, + "last_chunk_bytes": last_chunk_bytes, + "affects_objective": false, + "affects_canonical_candidate_identity": false, + }) +} + +pub(super) fn causal_profile_report(dataset: &LoadedDataset) -> Value { + let Some(profile) = &dataset.causal_profile else { + return serde_json::json!(null); + }; + let domains = profile + .domains + .iter() + .map(|(domain, support)| match support { + CausalTargetDomain::ByteAlphabet => serde_json::json!({ + "domain": domain, + "kind": "byte_alphabet", + "normalization": "per_byte_symbol", + "symbol_width_bytes": profile.byte_alphabet_symbol_width, + "symbols": 256usize, + }), + CausalTargetDomain::EnumeratedPayloads { payloads } => serde_json::json!({ + "domain": domain, + "kind": "enumerated_payloads", + "normalization": "finite_payload_set", + "payloads": payloads.len(), + }), + }) + .collect::>(); + serde_json::json!({ + "channel_set": profile.channel_set.iter().collect::>(), + "domain_support_crc32": profile.domain_support_hash, + "header_profile_crc32": profile.header_profile_hash, + "collection_policy": profile.collection_policy, + "action_alphabet_size": profile.action_alphabet_size, + "percept_schema_channels": profile + .percept_channels + .iter() + .map(|pair| { + serde_json::json!({"channel": pair.channel.as_str(), "domain": pair.domain.as_str()}) + }) + .collect::>(), + "reward_encoding": serde_json::json!({ + "channel": profile.reward_channel.channel.as_str(), + "domain": profile.reward_channel.domain.as_str(), + }), + "terminal_encoding": serde_json::json!({ + "channel": profile.terminal_channel.channel.as_str(), + "domain": profile.terminal_channel.domain.as_str(), + }), + "event_grammar": serde_json::json!({ + "context_channels": profile.event_grammar.context_channels.iter().collect::>(), + "observe_target_no_score": profile + .event_grammar + .observe_target_no_score + .iter() + .map(|pair| { + serde_json::json!({"channel": pair.channel.as_str(), "domain": pair.domain.as_str()}) + }) + .collect::>(), + "target": profile + .event_grammar + .target + .iter() + .map(|pair| { + serde_json::json!({"channel": pair.channel.as_str(), "domain": pair.domain.as_str()}) + }) + .collect::>(), + }), + "domains": domains, + "structural_conditioning": "zero_cost_event_descriptor_tags_v1", + "byte_alphabet_expansion_policy": "multi_byte_targets_expand_to_single_byte_events", + }) +} + +pub(super) fn evaluator_execution_model( + deterministic_table: Option<&VerifiedDeterministicEvaluatorTable>, +) -> &'static str { + if deterministic_table.is_some() { + "deterministic_table" + } else if cfg!(unix) { + "spawn_exec_worker_process_isolated_operational" + } else { + "unsupported_non_unix" + } +} + +pub(super) fn theorem_timing_basis( + theorem: &TuneTheoremConfig, + verified: &VerifiedTheoremInputs, +) -> &'static str { + match theorem.timing_certification_tier { + TimingCertificationTier::DeterministicTable if verified.deterministic_table.is_some() => { + "verified_deterministic_evaluator_table" + } + TimingCertificationTier::RealTime if verified.determinism_deadline.is_some() => { + "verified_real_time_deadline_certificate" + } + _ => "operational_only_uncertified", + } +} + +pub(super) fn dataset_kind_name(value: DatasetKind) -> &'static str { + match value { + DatasetKind::PassiveBytes => "passive_bytes", + DatasetKind::InteractiveTrace => "interactive_trace", + DatasetKind::CausalPrefixDataset => "causal_prefix_dataset", + } +} + +pub(super) fn objective_target_name(value: ObjectiveTarget) -> &'static str { + match value { + ObjectiveTarget::PassiveAc => "passive_ac", + ObjectiveTarget::InteractiveCausalAc => "interactive_causal_ac", + ObjectiveTarget::PlannerDeployableModel => "planner_deployable_model", + } +} + +pub(super) fn planner_deployability_report( + enabled: bool, + model_state_bytes: usize, + eval_latency_seconds: f64, + deployable_under_executor_limits: bool, +) -> Value { + serde_json::json!({ + "enabled": enabled, + "primary_score": "8L_B(z)+ell_D(z)", + "diagnostics_are_secondary": true, + "model_state_bytes": model_state_bytes, + "snapshot_bytes": model_state_bytes, + "clone_latency_seconds": 0.0, + "update_latency_seconds": eval_latency_seconds.max(0.0), + "restore_latency_seconds": 0.0, + "sampling_support": true, + "exact_log_probability_support": true, + "deployable_under_executor_limits": deployable_under_executor_limits, + }) +} + +pub(super) fn controller_kind_name( + controller: &crate::spec::CompiledTuneController, +) -> &'static str { + match controller { + crate::spec::CompiledTuneController::AnnealedHillClimbing(_) => "annealed_hill_climbing", + crate::spec::CompiledTuneController::McAixiFacCtw(_) => "mc_aixi_fac_ctw", + crate::spec::CompiledTuneController::AiqiDiscounted(_) => "aiqi_discounted", + crate::spec::CompiledTuneController::AiqiWarmstartExactJh(_) => "aiqi_warmstart_exact_jh", + } +} + +pub(super) fn planner_completed_status( + controller: &crate::spec::CompiledTuneController, +) -> &'static str { + match controller { + crate::spec::CompiledTuneController::McAixiFacCtw(_) => "completed_mc_aixi_fac_ctw", + crate::spec::CompiledTuneController::AiqiDiscounted(_) => "completed_aiqi_discounted", + crate::spec::CompiledTuneController::AiqiWarmstartExactJh(_) => { + "completed_aiqi_warmstart_exact_jh" + } + crate::spec::CompiledTuneController::AnnealedHillClimbing(_) => "completed_annealed", + } +} + +pub(super) fn planner_runtime_path_name( + controller: &crate::spec::CompiledTuneController, +) -> &'static str { + match controller { + crate::spec::CompiledTuneController::McAixiFacCtw(_) => { + "finite_mutation_agent_bridge_mcaixi_fac_ctw" + } + crate::spec::CompiledTuneController::AiqiDiscounted(_) => { + "finite_mutation_agent_bridge_aiqi_discounted" + } + crate::spec::CompiledTuneController::AiqiWarmstartExactJh(_) => { + "finite_mutation_agent_bridge_aiqi_warmstart_exact_jh" + } + crate::spec::CompiledTuneController::AnnealedHillClimbing(_) => { + "reversible_elementary_metropolis" + } + } +} + +pub(super) fn theorem_claims_report( + theorem: &TuneTheoremConfig, + verified: &VerifiedTheoremInputs, + controller: &crate::spec::CompiledTuneController, + dataset: &LoadedDataset, + search: &SearchSummary, + strict_theorem_memory_certified: bool, +) -> Value { + serde_json::json!({ + "exact_finite_mdp": theorem_claim_status( + theorem.claim_exact_finite_mdp, + exact_finite_mdp_missing_prereqs( + theorem, + verified, + controller, + search, + strict_theorem_memory_certified, + ), + &[ + "Assumptions finite-Z/no-hidden-state are represented by finite compiled mutation alphabet", + "Timing tier is theorem-admissible", + "Verified determinism/deadline certificate is present unless verified deterministic_table is used", + "Strict theorem-facing memory accounting is certified by resolved evaluator profile", + ], + ), + "exact_observed_markov": theorem_claim_status( + theorem.claim_exact_observed_markov, + exact_observed_markov_missing_prereqs( + theorem, + verified, + controller, + search, + strict_theorem_memory_certified, + ), + &[ + "All exact finite-MDP prerequisites hold", + "Exact-state observation encoder reference is present", + "Verified exact-state observation certificate is present", + ], + ), + "planner_convergence": theorem_claim_status( + theorem.claim_planner_convergence, + planner_convergence_missing_prereqs( + theorem, + verified, + controller, + search, + strict_theorem_memory_certified, + ), + &[ + "Controller is MC-AIXI(FAC-CTW)", + "Exact finite-MDP prerequisites hold", + "Exact objective-difference reward semantics are active", + ], + ), + "refs": { + "proof_boundary": theorem_proof_boundary_report(verified), + "timing_certification_tier": timing_tier_name(theorem.timing_certification_tier), + "determinism_deadline_certificate": theorem.determinism_deadline_certificate, + "observation_adapter_spec_ref": theorem.observation_adapter_spec_ref.as_deref().unwrap_or(OBSERVATION_ADAPTER_DECLARATION), + "exact_state_encoder_spec_ref": theorem.exact_state_encoder_spec_ref, + "exact_state_observation_basis": { + "verified_certificate": verified.exact_state_observation.is_some(), + "certificate": verified.exact_state_observation.as_ref().map(VerifiedExactStateObservationCertificate::to_json_value), + }, + "scalar_representation_ref": theorem.scalar_representation_ref.as_deref().unwrap_or(SCALAR_REPRESENTATION_DECLARATION), + "dataset_kind": dataset_kind_name(dataset.kind), + "dataset_lowering_version": dataset.lowering_version, + "target_domain_support_hash": dataset.target_domain_support_hash, + "causal_header_profile_hash": dataset.causal_header_profile_hash, + "verified": verified.to_json_value(), + } + }) +} + +fn theorem_proof_boundary_report(verified: &VerifiedTheoremInputs) -> Value { + serde_json::json!({ + "finite_planner_state_certificate": certificate_boundary_kind(verified.finite_planner_state.as_ref()), + "no_hidden_state_certificate": certificate_boundary_kind(verified.no_hidden_state.as_ref()), + "determinism_deadline_certificate": certificate_boundary_kind(verified.determinism_deadline.as_ref()), + "exact_reward_encoding_certificate": if verified.exact_reward_encoding.is_some() { + "certified_by_checked_artifact" + } else { + "operational_only_uncertified" + }, + "exact_state_observation_certificate": if verified.exact_state_observation.is_some() { + "certified_by_checked_artifact" + } else { + "operational_only_uncertified" + }, + "deterministic_evaluator_table": if verified.deterministic_table.is_some() { + "certified_by_checked_artifact" + } else { + "operational_only_uncertified" + }, + "generic_certificate_semantics": "content_hash_and_domain_context_checked_external_certificate", + }) +} + +fn certificate_boundary_kind(value: Option<&VerifiedCertificate>) -> &'static str { + if value.is_some() { + "certified_by_external_certificate" + } else { + "operational_only_uncertified" + } +} + +fn theorem_claim_status( + requested: bool, + missing_prereqs: Vec<&'static str>, + certified_basis: &[&'static str], +) -> Value { + if !requested { + serde_json::json!({ + "requested": false, + "status": "disabled", + "missing_prerequisites": [], + "certified_basis": [], + }) + } else if missing_prereqs.is_empty() { + serde_json::json!({ + "requested": true, + "status": "certified", + "missing_prerequisites": [], + "certified_basis": certified_basis, + }) + } else { + serde_json::json!({ + "requested": true, + "status": "uncertified", + "missing_prerequisites": missing_prereqs, + "certified_basis": [], + }) + } +} + +pub(super) fn exact_finite_mdp_missing_prereqs( + theorem: &TuneTheoremConfig, + verified: &VerifiedTheoremInputs, + controller: &crate::spec::CompiledTuneController, + search: &SearchSummary, + strict_theorem_memory_certified: bool, +) -> Vec<&'static str> { + let mut missing = Vec::new(); + match controller { + crate::spec::CompiledTuneController::AnnealedHillClimbing(_) => { + missing.push("planner_family_controller"); + } + crate::spec::CompiledTuneController::AiqiDiscounted(_) => { + missing.push("exact_objective_difference_controller"); + } + crate::spec::CompiledTuneController::McAixiFacCtw(_) + | crate::spec::CompiledTuneController::AiqiWarmstartExactJh(_) => {} + } + if verified.finite_planner_state.is_none() { + missing.push("verified_finite_planner_state_certificate"); + } + if verified.no_hidden_state.is_none() { + missing.push("verified_no_hidden_state_or_inert_state_certificate"); + } + if verified.exact_reward_encoding.is_none() { + missing.push("verified_exact_reward_encoding_certificate"); + } + if !verified.timing_certified(theorem) { + missing.push("theorem_certified_timing_or_deterministic_table"); + } + if !strict_theorem_memory_certified { + missing.push("strict_theorem_facing_memory_accounting"); + } + if theorem.scalar_representation_ref.is_none() { + missing.push("scalar_representation_ref"); + } + if search.best_eval.objective_bits.is_finite() { + missing + } else { + missing.push("finite_deployable_objective"); + missing + } +} + +fn exact_observed_markov_missing_prereqs( + theorem: &TuneTheoremConfig, + verified: &VerifiedTheoremInputs, + controller: &crate::spec::CompiledTuneController, + search: &SearchSummary, + strict_theorem_memory_certified: bool, +) -> Vec<&'static str> { + let mut missing = exact_finite_mdp_missing_prereqs( + theorem, + verified, + controller, + search, + strict_theorem_memory_certified, + ); + if theorem.exact_state_encoder_spec_ref.is_none() { + missing.push("exact_state_encoder_spec_ref"); + } + if verified.exact_state_observation.is_none() { + missing.push("verified_exact_state_observation_certificate"); + } + missing +} + +fn planner_convergence_missing_prereqs( + theorem: &TuneTheoremConfig, + verified: &VerifiedTheoremInputs, + controller: &crate::spec::CompiledTuneController, + search: &SearchSummary, + strict_theorem_memory_certified: bool, +) -> Vec<&'static str> { + let mut missing = exact_finite_mdp_missing_prereqs( + theorem, + verified, + controller, + search, + strict_theorem_memory_certified, + ); + if !matches!( + controller, + crate::spec::CompiledTuneController::McAixiFacCtw(_) + ) { + missing.push("mc_aixi_fac_ctw_controller"); + } + missing +} diff --git a/crates/infotheory/src/tuner/tests.rs b/crates/infotheory/src/tuner/tests.rs new file mode 100644 index 00000000..57948b08 --- /dev/null +++ b/crates/infotheory/src/tuner/tests.rs @@ -0,0 +1,4465 @@ +use super::*; +#[cfg(feature = "backend-ctw")] +use crate::aixi::common::{ActionAlphabet, ObservationKeyMode}; +use crate::aixi::warmstart::{ + WarmStartExactJhTeacherContract, WarmStartExactJhTeacherDataset, WarmStartExactJhTeacherTrace, + WarmStartExactJhTransition, +}; +use crate::aixi::warmstart_contract::TaskFingerprint; +use crate::api::CompressionBackend; +#[cfg(feature = "backend-ctw")] +use crate::api::RateBackend; +#[cfg(feature = "backend-ctw")] +use crate::compression::FramingMode; +#[cfg(feature = "backend-ctw")] +use crate::spec::{ + AiqiDiscountedTuneControllerSpec, AnnealedHillClimbingTuneControllerSpec, AssetBinding, + McAixiFacCtwTuneControllerSpec, SpecDocument, TuneBoundsSpec, TuneControllerSpec, + TunePlannerInterfaceSpec, TuneSpec, WarmStartExactJhTuneControllerSpec, +}; +use crate::tuner::eval::ResolvedMemoryAccountingKind; +#[cfg(all(feature = "backend-ctw", feature = "backend-mixture"))] +use crate::tuner::planner_bridge::apply_planner_mutation_action; +#[cfg(feature = "backend-ctw")] +use std::time::{SystemTime, UNIX_EPOCH}; + +fn strict_mode_test_accounting_kind() -> ResolvedMemoryAccountingKind { + #[cfg(target_os = "linux")] + { + ResolvedMemoryAccountingKind::StrictLinuxCgroupV2PeakMaxProcessRss + } + #[cfg(all(unix, not(target_os = "linux")))] + { + // Non-Linux targets cannot resolve strict cgroup-v2 accounting. + ResolvedMemoryAccountingKind::UnixProcessRssFallbackExplicit + } + #[cfg(not(unix))] + { + ResolvedMemoryAccountingKind::DeterministicEvaluatorTable + } +} + +#[cfg(feature = "backend-ctw")] +fn temp_path(prefix: &str, suffix: &str) -> std::path::PathBuf { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("clock") + .as_nanos(); + std::env::temp_dir().join(format!("infotheory-tuner-{prefix}-{nanos}{suffix}")) +} + +#[cfg(unix)] +#[test] +#[ignore = "libtest entrypoint for spawned tuner evaluator workers"] +fn __infotheory_tuner_eval_worker() { + if std::env::var_os("INFOTHEORY_TUNER_EVAL_REQUEST_PATH").is_none() + || std::env::var_os("INFOTHEORY_TUNER_EVAL_RESPONSE_PATH").is_none() + { + return; + } + run_tuner_eval_worker_from_env().expect("run tuner evaluator worker from env"); +} + +#[cfg(feature = "backend-ctw")] +fn sample_tune_spec(dataset_path: &str, output_path: &str, report_path: &str) -> TuneSpec { + TuneSpec { + assets: vec![AssetBinding { + id: "dataset".to_string(), + path: dataset_path.to_string(), + }], + input_asset: "dataset".to_string(), + baseline_candidate: CompressionBackend::Rate { + rate_backend: RateBackend::Ctw { depth: 8 }, + coder: crate::coders::CoderType::AC, + framing: FramingMode::Framed, + }, + controller: TuneControllerSpec::AnnealedHillClimbing( + AnnealedHillClimbingTuneControllerSpec { + max_mutation_radius: 1, + }, + ), + bounds: TuneBoundsSpec { + allowed_backends: vec!["ctw".to_string()], + forbidden_backends: Vec::new(), + parameter_ranges: Vec::new(), + max_experts: 2, + max_mixture_nesting_depth: 1, + min_experts: Some(1), + allow_duplicate_experts: Some(false), + required_experts: Vec::new(), + forbidden_expert_pairs: Vec::new(), + }, + eval_time_limit_seconds: 1.0, + time_budget_seconds: 2.0, + min_throughput_bytes_per_second: 1.0, + max_memory_bytes: u64::MAX, + output_config_path: output_path.to_string(), + seed: 7, + report_path: Some(report_path.to_string()), + } +} + +#[cfg(feature = "backend-ctw")] +fn action_alphabet(n: usize) -> ActionAlphabet { + ActionAlphabet::try_from_usize(n).expect("test action alphabet must be non-zero") +} + +#[cfg(feature = "backend-ctw")] +fn causal_dataset_value(codec_hash: &str, payload_key: &str, payload: Value) -> Value { + let mut object = serde_json::Map::new(); + object.insert("schema_version".to_string(), serde_json::json!(1)); + object.insert("environment_id".to_string(), serde_json::json!("test-env")); + object.insert( + "environment_config_crc32".to_string(), + serde_json::json!("00000000"), + ); + object.insert("codec_hash".to_string(), serde_json::json!(codec_hash)); + object.insert( + "reset_convention".to_string(), + serde_json::json!("reset-before-episode"), + ); + object.insert( + "action_alphabet".to_string(), + serde_json::json!({"size": 2}), + ); + object.insert( + "percept_schema".to_string(), + serde_json::json!({ + "encoding": "bytes", + "channels": [{"channel": "percept", "domain": "bytes"}], + }), + ); + object.insert( + "reward_encoding".to_string(), + serde_json::json!({ + "encoding": "bytes", + "channel": "reward", + "domain": "binary", + }), + ); + object.insert( + "terminal_encoding".to_string(), + serde_json::json!({ + "encoding": "bytes", + "channel": "terminal", + "domain": "binary", + }), + ); + object.insert("collection_policy".to_string(), serde_json::json!("test")); + object.insert( + "target_domains".to_string(), + serde_json::json!({ + "bytes": {"kind": "byte_alphabet"}, + "binary": {"kind": "enumerated_payloads", "payloads": [[0], [1]]} + }), + ); + object.insert( + "event_grammar".to_string(), + serde_json::json!({ + "context_channels": ["action"], + "observe_target_no_score": [ + {"channel": "percept", "domain": "bytes"}, + {"channel": "percept", "domain": "binary"}, + {"channel": "reward", "domain": "binary"}, + {"channel": "terminal", "domain": "binary"} + ], + "target": [ + {"channel": "percept", "domain": "bytes"}, + {"channel": "percept", "domain": "binary"}, + {"channel": "reward", "domain": "binary"}, + {"channel": "terminal", "domain": "binary"} + ] + }), + ); + object.insert(payload_key.to_string(), payload); + Value::Object(object) +} + +#[cfg(feature = "backend-ctw")] +fn planner_interface_for_baseline(candidate: &CompressionBackend) -> TunePlannerInterfaceSpec { + let json = crate::spec::compression_backend_to_json_value(candidate) + .expect("baseline candidate must serialize"); + let actions = (collect_numeric_leaves(&json).len() * 2).max(1); + TunePlannerInterfaceSpec { + observation_bits: 8, + observation_stream_len: 1, + observation_key_mode: ObservationKeyMode::FullStream, + reward_bits: 16, + agent_actions: action_alphabet(actions), + } +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn synthesized_planner_bridge_inherits_tune_spec_environment() { + let base_dir = temp_path("bridge-base", ""); + std::fs::create_dir_all(&base_dir).expect("create base dir"); + let output_path = base_dir.join("best.json"); + let report_path = base_dir.join("report.json"); + let mut spec = sample_tune_spec( + "relative-dataset.bin", + &output_path.to_string_lossy(), + &report_path.to_string_lossy(), + ); + spec.controller = TuneControllerSpec::AiqiDiscounted(AiqiDiscountedTuneControllerSpec { + interface: TunePlannerInterfaceSpec { + observation_bits: 8, + observation_stream_len: 1, + observation_key_mode: ObservationKeyMode::FullStream, + reward_bits: 8, + agent_actions: action_alphabet(1), + }, + planner_simulations_per_step: 1, + return_horizon: 1, + return_bins: 2, + discount_factor: 0.5, + min_improvement: 0.0, + max_improvement: 1.0, + }); + let env = SpecEnvironment::new(&base_dir); + let compiled = spec.compile_in(&env).expect("compile tune spec"); + assert_eq!(compiled.base_dir(), base_dir.as_path()); + + let dataset = LoadedDataset { + kind: DatasetKind::PassiveBytes, + objective_target: ObjectiveTarget::PassiveAc, + lowering_version: PASSIVE_DATASET_LOWERING_VERSION, + codec_hash: "passive-identity-bytes".to_string(), + event_grammar_hash: "passive-target-only-byte-stream".to_string(), + target_domain_support_hash: crc32_hex(b"passive-byte-alphabet"), + causal_header_profile_hash: crc32_hex(b"passive-none"), + target_size_function: "passive-bytes-len", + canonical_content_hash: crc32_hex(b"dataset"), + lowered_skeleton_hash: crc32_hex(b"passive-bytes-target-only"), + resolved_path: base_dir + .join("relative-dataset.bin") + .to_string_lossy() + .to_string(), + source_size_bytes: 7, + raw_bytes: b"dataset".to_vec(), + events: Vec::new(), + causal_profile: None, + dataset_units: 7.0, + target_events: 1, + }; + let verified = VerifiedTheoremInputs::default(); + let contract = + planner_controller_contract(compiled.controller(), &compiled, &dataset, &verified) + .expect("planner contract"); + let reward_encoder = contract + .reward_encoder( + &dataset, + &CandidateEvalResult { + status: CandidateEvalStatus::Success, + compressed_bytes: 7, + elapsed_seconds: 0.1, + effective_eval_time_limit_seconds: 1.0, + throughput_bytes_per_second: 70.0, + peak_memory_bytes: 0, + target_loss_bits: 7.0, + objective_bits: 7.0, + deployable: true, + }, + &verified, + ) + .expect("reward encoder"); + let planner_run = compile_tuner_planner_run_spec( + compiled.controller(), + &contract, + &reward_encoder, + &compiled, + &env, + ) + .expect("compile bridge"); + let binding = planner_run + .resolved_assets() + .iter() + .find(|binding| binding.id == "dataset") + .expect("dataset asset binding"); + let crate::spec::AssetRef::Filesystem(path) = &binding.asset; + assert_eq!(path, &base_dir.join("relative-dataset.bin")); + + let _ = std::fs::remove_dir_all(base_dir); +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn tuner_warmstart_task_fingerprint_binds_input_dataset_content_not_teacher_bytes() { + let base_dir = temp_path("warmstart-fingerprint", ""); + std::fs::create_dir_all(&base_dir).expect("create base dir"); + let dataset_path = base_dir.join("dataset.bin"); + let teacher_path = base_dir.join("teacher.json"); + let teacher_alt_path = base_dir.join("teacher-alt.json"); + std::fs::write(&dataset_path, b"dataset-v1").expect("write dataset v1"); + std::fs::write(&teacher_path, b"teacher-v1").expect("write teacher v1"); + std::fs::write(&teacher_alt_path, b"teacher-v1-alt-path").expect("write alternate teacher"); + + let output_path = base_dir.join("best.json"); + let report_path = base_dir.join("report.json"); + let mut spec = sample_tune_spec( + "dataset.bin", + &output_path.to_string_lossy(), + &report_path.to_string_lossy(), + ); + spec.assets.push(AssetBinding { + id: "teacher".to_string(), + path: "teacher.json".to_string(), + }); + let interface = TunePlannerInterfaceSpec { + observation_bits: 8, + observation_stream_len: 1, + observation_key_mode: ObservationKeyMode::FullStream, + reward_bits: 8, + agent_actions: action_alphabet(1), + }; + spec.controller = + TuneControllerSpec::AiqiWarmstartExactJh(WarmStartExactJhTuneControllerSpec { + interface: interface.clone(), + planner_simulations_per_step: 1, + return_horizon: 1, + warmstart_teacher_dataset_asset: "teacher".to_string(), + label_phase_period: 1, + }); + + let env = SpecEnvironment::new(&base_dir); + let compiled = spec.compile_in(&env).expect("compile tune spec"); + let contract = PlannerControllerContract { + interface, + planner_simulations_per_step: 1, + return_horizon: Some(1), + label_phase_period: Some(1), + discount_factor: 1.0, + reward_semantics: PlannerRewardSemantics::ExactObjectiveDifference, + clipping_interval: None, + teacher: None, + warmstart_self_improvement: true, + }; + let reward_encoder = TunerRewardEncoder::ExactIntegerObjectiveDifference { + max_reward: 3, + objective_difference_to_symbol: None, + }; + let planner_run = compile_tuner_planner_run_spec( + compiled.controller(), + &contract, + &reward_encoder, + &compiled, + &env, + ) + .expect("compile warmstart bridge"); + let fingerprint_v1 = + crate::aixi::warmstart_contract::warmstart_exact_jh_planner_task_fingerprint(&planner_run) + .expect("fingerprint v1"); + + std::fs::write(&teacher_path, b"teacher-v2").expect("write teacher v2"); + let fingerprint_after_teacher_change = + crate::aixi::warmstart_contract::warmstart_exact_jh_planner_task_fingerprint(&planner_run) + .expect("fingerprint after teacher change"); + assert_eq!( + fingerprint_v1, fingerprint_after_teacher_change, + "teacher dataset bytes are intentionally excluded to avoid a circular task fingerprint" + ); + + let mut moved_teacher_spec = spec.clone(); + if let Some(binding) = moved_teacher_spec + .assets + .iter_mut() + .find(|binding| binding.id == "teacher") + { + binding.path = "teacher-alt.json".to_string(); + } + let moved_teacher_compiled = moved_teacher_spec + .compile_in(&env) + .expect("compile moved-teacher tune spec"); + let moved_teacher_planner_run = compile_tuner_planner_run_spec( + moved_teacher_compiled.controller(), + &contract, + &reward_encoder, + &moved_teacher_compiled, + &env, + ) + .expect("compile moved-teacher warmstart bridge"); + let fingerprint_after_teacher_path_change = + crate::aixi::warmstart_contract::warmstart_exact_jh_planner_task_fingerprint( + &moved_teacher_planner_run, + ) + .expect("fingerprint after teacher path change"); + assert_eq!( + fingerprint_v1, fingerprint_after_teacher_path_change, + "teacher dataset asset path is intentionally excluded from same-task identity" + ); + + std::fs::write(&dataset_path, b"dataset-v2").expect("write dataset v2"); + let fingerprint_v2 = + crate::aixi::warmstart_contract::warmstart_exact_jh_planner_task_fingerprint(&planner_run) + .expect("fingerprint v2"); + assert_ne!( + fingerprint_v1, fingerprint_v2, + "same-path input dataset byte changes must invalidate same-task warm-start teachers" + ); + + let _ = std::fs::remove_dir_all(base_dir); +} + +#[cfg(feature = "backend-ctw")] +fn write_test_exact_reward_certificate( + path: &std::path::Path, + dataset_path: &std::path::Path, + bounds: &TuneBoundsSpec, + controller_kind: &str, +) { + let dataset = load_dataset(dataset_path).expect("load dataset for certificate"); + let execution = TuneExecutionConfig::default(); + let runtime_profile = resolve_evaluator_runtime_profile(&execution, false) + .expect("resolve evaluator runtime profile for certificate"); + let evaluator_profile = EvaluatorProfile { + dataset_kind: dataset.kind, + objective_target: dataset.objective_target, + dataset_lowering_version: dataset.lowering_version, + dataset_codec_hash: dataset.codec_hash.clone(), + event_grammar_hash: dataset.event_grammar_hash.clone(), + target_domain_support_hash: dataset.target_domain_support_hash.clone(), + causal_header_profile_hash: dataset.causal_header_profile_hash.clone(), + target_size_function: dataset.target_size_function, + evaluator_interface_version: TUNER_EVALUATOR_INTERFACE_VERSION, + candidate_canonicalization_version: "bounds-v1".to_string(), + warmup_baseline_runs: 0, + diagnostic_chunk_bytes: None, + eval_time_limit_seconds: 1.0, + evaluator_threads: execution.evaluator_threads(), + worker_isolation_mode: "spawn_exec_worker", + worker_executable_identity: runtime_profile.worker_executable_identity.clone(), + resolved_memory_accounting_kind: runtime_profile.memory_accounting_kind.name(), + resolved_memory_accounting_strict_theorem_facing: runtime_profile + .strict_theorem_memory_certified(), + resolved_evaluator_cgroup_parent: runtime_profile.resolved_cgroup_parent_string(), + backend_report_component_policy: runtime_profile + .memory_accounting_kind + .backend_report_component_policy(), + evaluator_determinism: execution.evaluator_determinism(), + rss_mode: execution.rss_mode, + timing_certification_tier: TimingCertificationTier::BestEffort, + build_profile: option_env!("PROFILE").unwrap_or("unknown"), + feature_set: compiled_feature_set(), + }; + let reward_cert = serde_json::json!({ + "schema_version": 1, + "kind": "exact_reward_encoding", + "dataset_crc32": dataset.canonical_content_hash, + "bounds_crc32": bounds_hash(bounds).expect("bounds hash"), + "evaluator_profile_crc32": evaluator_profile.hash().expect("profile hash"), + "controller_kind": controller_kind, + "action_alphabet_size": 2, + "encoding": "integer_objective_difference", + "scalar_representation": SCALAR_REPRESENTATION_DECLARATION, + "reward_bits": 16, + "max_reward": 65_535u64, + }); + std::fs::write( + path, + serde_json::to_vec(&reward_cert).expect("reward cert json"), + ) + .expect("write reward cert"); +} + +#[test] +fn parse_tune_cli_args_and_theorem_flags() { + let args = vec![ + "infotheory".to_string(), + "tune".to_string(), + "spec.json".to_string(), + "--max-evaluations".to_string(), + "12".to_string(), + "--timing-tier".to_string(), + "real_time".to_string(), + "--claim-exact-finite-mdp".to_string(), + "--evaluator-worker-executable".to_string(), + "/tmp/infotheory-worker".to_string(), + "--evaluator-cgroup-parent".to_string(), + "/sys/fs/cgroup/infotheory-tuner".to_string(), + "--emit-exact-reward-encoding-certificate".to_string(), + "emit-reward-cert.json".to_string(), + ]; + let parsed = parse_tune_command_args(&args).expect("parse tune args"); + assert_eq!(parsed.spec_path, "spec.json"); + assert_eq!(parsed.execution.max_evaluations, Some(12)); + assert_eq!( + parsed.execution.theorem.timing_certification_tier, + TimingCertificationTier::RealTime + ); + assert!(parsed.execution.theorem.claim_exact_finite_mdp); + assert_eq!( + parsed.execution.evaluator_worker_executable.as_deref(), + Some("/tmp/infotheory-worker") + ); + assert_eq!( + parsed.execution.evaluator_cgroup_parent.as_deref(), + Some("/sys/fs/cgroup/infotheory-tuner") + ); + assert_eq!( + parsed.emit_exact_reward_encoding_certificate.as_deref(), + Some("emit-reward-cert.json") + ); +} + +#[test] +fn tune_execution_config_accepts_nested_theorem_json() { + let value = serde_json::json!({ + "warmup_baseline_runs": 2, + "planner_deployable_model": true, + "theorem": { + "claim_exact_observed_markov": true, + "timing_certification_tier": "isolated" + } + }); + let cfg = TuneExecutionConfig::from_json_value(&value).expect("config parse"); + assert_eq!(cfg.warmup_baseline_runs, 2); + assert!(cfg.planner_deployable_model); + assert!(cfg.theorem.claim_exact_observed_markov); + assert_eq!( + cfg.theorem.timing_certification_tier, + TimingCertificationTier::Isolated + ); +} + +#[test] +fn tune_execution_config_reports_executor_profile_semantics() { + let value = serde_json::json!({ + "max_evaluations": 7, + "annealer_kernel_profile": "compiled_uniform_metropolis_hastings", + "cpu_affinity": "0-1", + "threads": 2, + "evaluator_worker_executable": "/tmp/infotheory-worker", + "evaluator_cgroup_parent": "/sys/fs/cgroup/infotheory-tuner", + "warmup_baseline_runs": 1, + "self_improvement_rounds": 3, + "stagnation_reset_evals": 5, + "log_path": "tune.log", + "diagnostic_chunk_bytes": 4096, + "rss_mode": "hybrid_strict_max", + "planner_deployable_model": true, + "warmstart_trace_refresh": true, + "theorem": { + "claim_exact_finite_mdp": true, + "claim_exact_observed_markov": true, + "claim_planner_convergence": true, + "timing_certification_tier": "deterministic_table", + "determinism_deadline_certificate": "cert://deadline", + "observation_adapter_spec_ref": "adapter://single-channel", + "exact_state_encoder_spec_ref": "state://encoder", + "scalar_representation_ref": "scalar://finite-f64", + "finite_planner_state_certificate": "cert://finite-state", + "no_hidden_state_certificate": "cert://no-hidden", + "exact_reward_encoding_certificate": "cert://reward", + "exact_state_observation_certificate": "cert://observation", + "deterministic_evaluator_table": "table://deterministic" + } + }); + let cfg = TuneExecutionConfig::from_json_value(&value).expect("config parse"); + + assert_eq!(cfg.max_evaluations, Some(7)); + assert_eq!( + cfg.annealer_kernel_profile, + AnnealerKernelProfile::CompiledUniformMetropolisHastings + ); + assert_eq!(cfg.evaluator_threads(), 2); + assert_eq!( + cfg.evaluator_determinism(), + "requires_backend_determinism_when_threaded" + ); + assert!(cfg.warmstart_trace_refresh); + assert_eq!( + cfg.theorem.timing_certification_tier, + TimingCertificationTier::DeterministicTable + ); + + let profile = cfg.to_json_value(); + assert_eq!(profile["max_evaluations"], serde_json::json!(7)); + assert_eq!( + profile["annealer_kernel_profile"], + serde_json::json!("compiled_uniform_metropolis_hastings") + ); + assert_eq!( + profile["evaluator_determinism"], + serde_json::json!("requires_backend_determinism_when_threaded") + ); + assert_eq!( + profile["evaluator_worker_executable"], + serde_json::json!("/tmp/infotheory-worker") + ); + assert_eq!( + profile["evaluator_cgroup_parent"], + serde_json::json!("/sys/fs/cgroup/infotheory-tuner") + ); + assert_eq!( + profile["theorem"]["deterministic_evaluator_table"], + serde_json::json!("table://deterministic") + ); + + let controls = executor_controls_report( + &cfg, + &ResolvedEvaluatorRuntimeProfile { + worker_executable: Some(std::path::PathBuf::from("/tmp/infotheory-worker")), + worker_executable_identity: Some("crc32:00000000:bytes:0".to_string()), + resolved_evaluator_cgroup_parent: Some(std::path::PathBuf::from( + "/sys/fs/cgroup/infotheory-tuner", + )), + memory_accounting_kind: strict_mode_test_accounting_kind(), + }, + ); + assert_eq!( + controls["cpu_affinity"]["requested"], + serde_json::json!("0-1") + ); + assert_eq!( + controls["threads"]["worker_isolation_mode"], + serde_json::json!("spawn_exec_worker") + ); + assert_eq!( + controls["rss_mode"]["requested"], + serde_json::json!("hybrid_strict_max") + ); +} + +#[test] +fn tune_execution_config_rejects_empty_certificate_references() { + for field in [ + "determinism_deadline_certificate", + "observation_adapter_spec_ref", + "exact_state_encoder_spec_ref", + "scalar_representation_ref", + "finite_planner_state_certificate", + "no_hidden_state_certificate", + "exact_reward_encoding_certificate", + "exact_state_observation_certificate", + "deterministic_evaluator_table", + ] { + let mut theorem = serde_json::Map::::new(); + theorem.insert(field.to_string(), serde_json::json!(" ")); + let value = serde_json::json!({ + "theorem": theorem + }); + let err = TuneExecutionConfig::from_json_value(&value) + .expect_err("empty theorem reference must be rejected"); + assert!(err.contains("must be a non-empty string"), "{field}: {err}"); + } +} + +#[test] +fn tune_execution_config_rejects_empty_worker_executable() { + let value = serde_json::json!({ + "evaluator_worker_executable": " " + }); + let err = TuneExecutionConfig::from_json_value(&value) + .expect_err("empty evaluator_worker_executable must be rejected"); + assert!(err.contains("evaluator_worker_executable"), "{err}"); +} + +#[test] +fn tune_execution_config_rejects_empty_cgroup_parent() { + let value = serde_json::json!({ + "evaluator_cgroup_parent": " " + }); + let err = TuneExecutionConfig::from_json_value(&value) + .expect_err("empty evaluator_cgroup_parent must be rejected"); + assert!(err.contains("evaluator_cgroup_parent"), "{err}"); +} + +#[test] +fn run_tune_rejects_invalid_execution_config_direct_call() { + let request = TuneCommandRequest { + spec_path: "nonexistent-spec.json".to_string(), + emit_exact_reward_encoding_certificate: None, + execution: TuneExecutionConfig { + threads: Some(0), + ..TuneExecutionConfig::default() + }, + }; + let err = run_tune(&request).expect_err("invalid execution config must fail at run_tune"); + assert!(err.contains("threads must be >= 1 when set"), "{err}"); +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn tune_planner_interface_requires_explicit_observation_key_mode() { + let mut value = SpecDocument::Tune(sample_tune_spec("dataset.bin", "out.json", "report.json")) + .to_canonical_json_value() + .expect("canonical tune json"); + value["controller"] = serde_json::json!({ + "kind": "mc_aixi_fac_ctw", + "interface": { + "observation_bits": 8, + "observation_stream_len": 1, + "reward_bits": 8, + "agent_actions": 1 + }, + "planner_simulations_per_step": 1 + }); + let err = match SpecDocument::parse_json_value(&value, Path::new(".")) { + Ok(_) => panic!("missing tune observation_key_mode must fail"), + Err(err) => err, + }; + assert!( + err.to_string() + .contains("controller.interface.observation_key_mode is required"), + "{err}" + ); +} + +#[test] +fn tune_execution_config_rejects_observation_certified_boolean() { + let value = serde_json::json!({ + "theorem": { + "exact_state_observation_certified": true + } + }); + let err = TuneExecutionConfig::from_json_value(&value) + .expect_err("unchecked observation proof boolean must be rejected"); + assert!(err.contains("unknown execution config field"), "{err}"); +} + +fn passive_loaded_dataset(raw_bytes: Vec) -> LoadedDataset { + LoadedDataset { + kind: DatasetKind::PassiveBytes, + objective_target: ObjectiveTarget::PassiveAc, + lowering_version: PASSIVE_DATASET_LOWERING_VERSION, + codec_hash: "codec".to_string(), + event_grammar_hash: "none".to_string(), + target_domain_support_hash: "none".to_string(), + causal_header_profile_hash: "none".to_string(), + target_size_function: "bytes", + canonical_content_hash: "content".to_string(), + lowered_skeleton_hash: "skeleton".to_string(), + resolved_path: "dataset.bin".to_string(), + source_size_bytes: raw_bytes.len(), + dataset_units: raw_bytes.len() as f64, + raw_bytes, + events: Vec::new(), + causal_profile: None, + target_events: 0, + } +} + +#[test] +fn diagnostic_chunking_report_preserves_executor_only_contract() { + let dataset = passive_loaded_dataset((0_u8..10).collect::>()); + + let disabled = diagnostic_chunking_report(&dataset, None); + assert_eq!(disabled["enabled"], serde_json::json!(false)); + assert_eq!( + disabled["affects_canonical_candidate_identity"], + serde_json::json!(false) + ); + assert_eq!(disabled["chunk_count"], serde_json::json!(0)); + + let enabled = diagnostic_chunking_report(&dataset, Some(4)); + assert_eq!(enabled["enabled"], serde_json::json!(true)); + assert_eq!(enabled["charged_payload_bytes"], serde_json::json!(10)); + assert_eq!(enabled["chunk_count"], serde_json::json!(3)); + assert_eq!(enabled["last_chunk_bytes"], serde_json::json!(2)); + assert_eq!(enabled["affects_objective"], serde_json::json!(false)); + assert_eq!( + enabled["affects_canonical_candidate_identity"], + serde_json::json!(false) + ); +} + +#[test] +fn causal_profile_report_describes_domains_and_event_grammar() { + let percept_channel = CausalChannelDomain { + channel: "obs".to_string(), + domain: "byte".to_string(), + }; + let reward_channel = CausalChannelDomain { + channel: "reward".to_string(), + domain: "reward_symbols".to_string(), + }; + let terminal_channel = CausalChannelDomain { + channel: "terminal".to_string(), + domain: "terminal_symbols".to_string(), + }; + let mut domains = BTreeMap::::new(); + domains.insert("byte".to_string(), CausalTargetDomain::ByteAlphabet); + domains.insert( + "reward_symbols".to_string(), + CausalTargetDomain::EnumeratedPayloads { + payloads: vec![vec![0], vec![1]], + }, + ); + let mut channel_set = BTreeSet::::new(); + channel_set.insert("obs".to_string()); + channel_set.insert("reward".to_string()); + channel_set.insert("terminal".to_string()); + let mut percept_channels = BTreeSet::::new(); + percept_channels.insert(percept_channel.clone()); + let mut context_channels = BTreeSet::::new(); + context_channels.insert("context".to_string()); + let mut observe_target_no_score = BTreeSet::::new(); + observe_target_no_score.insert(percept_channel.clone()); + let mut target = BTreeSet::::new(); + target.insert(reward_channel.clone()); + let profile = CausalEvaluationProfile { + domains, + channel_set, + domain_support_hash: "domain-crc".to_string(), + byte_alphabet_symbol_width: 1, + header_profile_hash: "header-crc".to_string(), + event_grammar: CausalEventGrammar { + context_channels, + observe_target_no_score, + target, + }, + action_alphabet_size: 3, + collection_policy: "test-policy".to_string(), + percept_channels, + reward_channel, + terminal_channel, + }; + let mut dataset = passive_loaded_dataset(Vec::new()); + dataset.kind = DatasetKind::CausalPrefixDataset; + dataset.causal_profile = Some(profile); + + let report = causal_profile_report(&dataset); + assert_eq!( + report["domain_support_crc32"], + serde_json::json!("domain-crc") + ); + assert_eq!( + report["header_profile_crc32"], + serde_json::json!("header-crc") + ); + assert_eq!(report["action_alphabet_size"], serde_json::json!(3)); + assert_eq!( + report["byte_alphabet_expansion_policy"], + serde_json::json!("multi_byte_targets_expand_to_single_byte_events") + ); + assert_eq!( + report["reward_encoding"], + serde_json::json!({"channel": "reward", "domain": "reward_symbols"}) + ); + assert_eq!(report["domains"].as_array().expect("domains").len(), 2); +} + +#[test] +fn theorem_timing_and_evaluator_execution_models_report_verified_basis() { + let deterministic_table = VerifiedDeterministicEvaluatorTable { + base: VerifiedCertificate { + ref_value: "table://deterministic".to_string(), + content_hash: "table-crc".to_string(), + }, + rows: HashMap::new(), + }; + let verified_table = VerifiedTheoremInputs { + deterministic_table: Some(deterministic_table), + ..VerifiedTheoremInputs::default() + }; + let table_theorem = TuneTheoremConfig { + timing_certification_tier: TimingCertificationTier::DeterministicTable, + ..TuneTheoremConfig::default() + }; + assert_eq!( + evaluator_execution_model(verified_table.deterministic_table.as_ref()), + "deterministic_table" + ); + assert_eq!( + theorem_timing_basis(&table_theorem, &verified_table), + "verified_deterministic_evaluator_table" + ); + + let verified_deadline = VerifiedTheoremInputs { + determinism_deadline: Some(VerifiedCertificate { + ref_value: "deadline://cert".to_string(), + content_hash: "deadline-crc".to_string(), + }), + ..VerifiedTheoremInputs::default() + }; + let real_time_theorem = TuneTheoremConfig { + timing_certification_tier: TimingCertificationTier::RealTime, + ..TuneTheoremConfig::default() + }; + assert_eq!( + theorem_timing_basis(&real_time_theorem, &verified_deadline), + "verified_real_time_deadline_certificate" + ); + assert_eq!( + theorem_timing_basis(&real_time_theorem, &VerifiedTheoremInputs::default()), + "operational_only_uncertified" + ); + + let deployability = planner_deployability_report(true, 128, -1.0, false); + assert_eq!( + deployability["update_latency_seconds"], + serde_json::json!(0.0) + ); + assert_eq!( + deployability["deployable_under_executor_limits"], + serde_json::json!(false) + ); +} + +#[test] +fn finite_reward_map_accepts_non_contiguous_injective_symbols() { + let value = serde_json::json!({ + "values": [ + {"objective_difference": 0, "symbol": 0}, + {"objective_difference": 3, "symbol": 7} + ] + }); + let map = parse_finite_reward_map(value.as_object().expect("object"), 4, 15) + .expect("finite reward map"); + assert_eq!(map.objective_difference_to_symbol.get(&0), Some(&0)); + assert_eq!(map.objective_difference_to_symbol.get(&3), Some(&7)); + assert_eq!(map.complete_nonnegative_interval_max, None); +} + +#[test] +fn finite_reward_map_rejects_reachable_rewards_alias() { + let value = serde_json::json!({ + "reachable_rewards": [ + {"objective_difference": 0, "symbol": 0}, + {"objective_difference": 1, "symbol": 1} + ] + }); + let err = parse_finite_reward_map(value.as_object().expect("object"), 4, 15) + .expect_err("reachable_rewards alias must be rejected"); + assert!(err.contains("requires a 'values' array"), "{err}"); +} + +#[test] +fn finite_reward_map_rejects_duplicate_symbols() { + let value = serde_json::json!({ + "values": [ + {"objective_difference": 0, "symbol": 1}, + {"objective_difference": 2, "symbol": 1} + ] + }); + let err = parse_finite_reward_map(value.as_object().expect("object"), 4, 15) + .expect_err("duplicate symbol must fail"); + assert!(err.contains("duplicates reward symbol"), "{err}"); +} + +#[test] +fn finite_reward_map_rejects_incomplete_declared_interval() { + let value = serde_json::json!({ + "complete_nonnegative_interval_max": 3, + "values": [ + {"objective_difference": 0, "symbol": 0}, + {"objective_difference": 1, "symbol": 1}, + {"objective_difference": 3, "symbol": 3} + ] + }); + let err = parse_finite_reward_map(value.as_object().expect("object"), 4, 15) + .expect_err("declared complete interval must contain every difference"); + assert!(err.contains("missing objective_difference 2"), "{err}"); +} + +#[test] +fn exact_finite_reward_map_encodes_objective_difference_not_symbol_arithmetic() { + let encoder = TunerRewardEncoder::ExactIntegerObjectiveDifference { + max_reward: 2, + objective_difference_to_symbol: Some(BTreeMap::from([(0, 0), (1, 2), (2, 1)])), + }; + assert_eq!(encoder.encode(1.0).expect("mapped reward"), 2); + assert_eq!(encoder.encode(2.0).expect("mapped reward"), 1); +} + +#[cfg(all(feature = "backend-ctw", target_os = "linux"))] +#[test] +fn cgroup_peak_reader_parses_fixture_file() { + let path = temp_path("cgroup-memory-peak", ".txt"); + fs::write(&path, b"12345\n").expect("write cgroup fixture"); + assert_eq!(read_u64_from_file(&path).expect("parse cgroup peak"), 12345); + fs::write(&path, b"max\n").expect("write cgroup sentinel fixture"); + let err = read_u64_from_file(&path).expect_err("max sentinel is not a measurement"); + assert!(err.contains("unbounded sentinel"), "{err}"); + let _ = fs::remove_file(path); +} + +#[test] +fn executor_controls_report_reflects_requested_rss_mode() { + let config = TuneExecutionConfig { + rss_mode: PeakMemoryMode::HybridStrictMax, + ..TuneExecutionConfig::default() + }; + let report = executor_controls_report( + &config, + &ResolvedEvaluatorRuntimeProfile { + worker_executable: Some(std::path::PathBuf::from("/tmp/worker")), + worker_executable_identity: Some("crc32:11111111:bytes:1".to_string()), + resolved_evaluator_cgroup_parent: Some(std::path::PathBuf::from("/sys/fs/cgroup/test")), + memory_accounting_kind: strict_mode_test_accounting_kind(), + }, + ); + assert_eq!(report["rss_mode"]["requested"], "hybrid_strict_max"); + let effective = report["rss_mode"]["effective_measurement"] + .as_str() + .expect("effective measurement"); + #[cfg(target_os = "linux")] + assert_eq!(effective, "strict_linux_max_process_rss_cgroup_v2_peak"); + #[cfg(all(unix, not(target_os = "linux")))] + assert_eq!(effective, "unix_process_rss_fallback_explicit"); + #[cfg(not(unix))] + assert_eq!(effective, "deterministic_evaluator_table_row_peak_memory"); +} + +#[test] +fn executor_controls_report_uses_explicit_deterministic_table_provenance() { + let config = TuneExecutionConfig { + rss_mode: PeakMemoryMode::BackendReported, + ..TuneExecutionConfig::default() + }; + let report = executor_controls_report( + &config, + &ResolvedEvaluatorRuntimeProfile { + worker_executable: None, + worker_executable_identity: None, + resolved_evaluator_cgroup_parent: None, + memory_accounting_kind: ResolvedMemoryAccountingKind::DeterministicEvaluatorTable, + }, + ); + assert_eq!(report["rss_mode"]["requested"], "backend_reported"); + assert_eq!( + report["rss_mode"]["effective_measurement"], + "deterministic_evaluator_table_row_peak_memory" + ); +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn evaluator_profile_cache_key_changes_with_execution_profile_only() { + let profile_a = EvaluatorProfile { + dataset_kind: DatasetKind::PassiveBytes, + objective_target: ObjectiveTarget::PassiveAc, + dataset_lowering_version: PASSIVE_DATASET_LOWERING_VERSION, + dataset_codec_hash: "passive-identity-bytes".to_string(), + event_grammar_hash: "passive-target-only-byte-stream".to_string(), + target_domain_support_hash: crc32_hex(b"passive-byte-alphabet"), + causal_header_profile_hash: crc32_hex(b"passive-none"), + target_size_function: "passive-bytes-len", + evaluator_interface_version: TUNER_EVALUATOR_INTERFACE_VERSION, + candidate_canonicalization_version: "bounds-v1".to_string(), + warmup_baseline_runs: 0, + diagnostic_chunk_bytes: None, + eval_time_limit_seconds: 1.0, + evaluator_threads: 1, + worker_isolation_mode: "spawn_exec_worker", + worker_executable_identity: None, + resolved_memory_accounting_kind: "unix_process_rss_fallback_explicit", + resolved_memory_accounting_strict_theorem_facing: false, + resolved_evaluator_cgroup_parent: None, + backend_report_component_policy: "none", + evaluator_determinism: "deterministic_under_h", + rss_mode: PeakMemoryMode::ProcessRssPeak, + timing_certification_tier: TimingCertificationTier::BestEffort, + build_profile: "test", + feature_set: vec!["test"], + }; + let mut profile_b = profile_a.clone(); + profile_b.warmup_baseline_runs = 3; + let mut profile_c = profile_a.clone(); + profile_c.eval_time_limit_seconds = 0.5; + let mut profile_d = profile_a.clone(); + profile_d.diagnostic_chunk_bytes = Some(4096); + + let candidate = CompressionBackend::Rate { + rate_backend: RateBackend::Ctw { depth: 8 }, + coder: crate::coders::CoderType::AC, + framing: FramingMode::Framed, + }; + let candidate_bytes = candidate + .compile() + .expect("compile candidate") + .canonical_bytes() + .as_slice() + .to_vec(); + let dataset_hash = crc32_hex(b"same-dataset"); + let key_a = + cache_key_for_candidate(&candidate_bytes, &profile_a, &dataset_hash).expect("cache key a"); + let key_b = + cache_key_for_candidate(&candidate_bytes, &profile_b, &dataset_hash).expect("cache key b"); + let key_c = + cache_key_for_candidate(&candidate_bytes, &profile_c, &dataset_hash).expect("cache key c"); + let key_d = + cache_key_for_candidate(&candidate_bytes, &profile_d, &dataset_hash).expect("cache key d"); + assert_ne!(key_a, key_b); + assert_ne!(key_a, key_c); + assert_ne!(key_a, key_d); + assert_eq!(key_a.candidate_canonical_bytes, candidate_bytes); + assert_eq!( + key_b.candidate_canonical_bytes, + key_a.candidate_canonical_bytes + ); + assert_eq!( + key_c.candidate_canonical_bytes, + key_a.candidate_canonical_bytes + ); + assert_eq!(key_b.dataset_identity, key_a.dataset_identity); + assert_ne!(key_b.evaluator_profile_bytes, key_a.evaluator_profile_bytes); + assert_ne!(key_c.evaluator_profile_bytes, key_a.evaluator_profile_bytes); +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn deterministic_table_evaluation_enforces_exact_objective_formula() { + let candidate = CompressionBackend::Rate { + rate_backend: RateBackend::Ctw { depth: 4 }, + coder: crate::coders::CoderType::AC, + framing: FramingMode::Framed, + } + .compile() + .expect("compile candidate"); + let candidate_crc32 = crc32_hex(candidate.canonical_bytes().as_slice()); + let model_bytes: usize = 17; + let target_loss_bits = 23.5; + let table = VerifiedDeterministicEvaluatorTable { + base: VerifiedCertificate { + ref_value: "test://deterministic-table".to_string(), + content_hash: "00000000".to_string(), + }, + rows: HashMap::from([( + candidate_crc32, + DeterministicEvaluatorRow { + status: CandidateEvalStatus::Success, + compressed_bytes: 3, + target_loss_bits, + elapsed_seconds: 0.25, + peak_memory_bytes: 16, + }, + )]), + }; + let dataset = LoadedDataset { + kind: DatasetKind::PassiveBytes, + objective_target: ObjectiveTarget::PassiveAc, + lowering_version: PASSIVE_DATASET_LOWERING_VERSION, + codec_hash: "passive-identity-bytes".to_string(), + event_grammar_hash: "passive-target-only-byte-stream".to_string(), + target_domain_support_hash: crc32_hex(b"passive-byte-alphabet"), + causal_header_profile_hash: crc32_hex(b"passive-none"), + target_size_function: "passive-bytes-len", + canonical_content_hash: crc32_hex(b"dataset"), + lowered_skeleton_hash: crc32_hex(b"passive-bytes-target-only"), + resolved_path: "test://dataset".to_string(), + source_size_bytes: 11, + raw_bytes: b"hello world".to_vec(), + events: Vec::new(), + causal_profile: None, + dataset_units: 11.0, + target_events: 1, + }; + + let result = table + .evaluate(&candidate, &dataset, model_bytes, 1.0, 1024, 1.0) + .expect("deterministic table evaluation"); + assert_eq!(result.status, CandidateEvalStatus::Success); + assert_eq!(result.target_loss_bits, target_loss_bits); + assert_eq!( + result.objective_bits, + (model_bytes as f64 * 8.0) + target_loss_bits + ); + assert_eq!(result.throughput_bytes_per_second, 44.0); + assert!(result.deployable); +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn deterministic_table_success_row_exceeding_effective_limit_is_timeout() { + let candidate = CompressionBackend::Rate { + rate_backend: RateBackend::Ctw { depth: 4 }, + coder: crate::coders::CoderType::AC, + framing: FramingMode::Framed, + } + .compile() + .expect("compile candidate"); + let candidate_crc32 = crc32_hex(candidate.canonical_bytes().as_slice()); + let table = VerifiedDeterministicEvaluatorTable { + base: VerifiedCertificate { + ref_value: "test://deterministic-table".to_string(), + content_hash: "00000000".to_string(), + }, + rows: HashMap::from([( + candidate_crc32, + DeterministicEvaluatorRow { + status: CandidateEvalStatus::Success, + compressed_bytes: 3, + target_loss_bits: 23.5, + elapsed_seconds: 2.0, + peak_memory_bytes: 16, + }, + )]), + }; + let dataset = LoadedDataset { + kind: DatasetKind::PassiveBytes, + objective_target: ObjectiveTarget::PassiveAc, + lowering_version: PASSIVE_DATASET_LOWERING_VERSION, + codec_hash: "passive-identity-bytes".to_string(), + event_grammar_hash: "passive-target-only-byte-stream".to_string(), + target_domain_support_hash: crc32_hex(b"passive-byte-alphabet"), + causal_header_profile_hash: crc32_hex(b"passive-none"), + target_size_function: "passive-bytes-len", + canonical_content_hash: crc32_hex(b"dataset"), + lowered_skeleton_hash: crc32_hex(b"passive-bytes-target-only"), + resolved_path: "test://dataset".to_string(), + source_size_bytes: 11, + raw_bytes: b"hello world".to_vec(), + events: Vec::new(), + causal_profile: None, + dataset_units: 11.0, + target_events: 1, + }; + + let effective_limit_seconds = 1.0; + let result = table + .evaluate(&candidate, &dataset, 17, 1.0, 1024, effective_limit_seconds) + .expect("deterministic table evaluation"); + + assert_eq!(result.status, CandidateEvalStatus::Timeout); + assert_eq!(result.elapsed_seconds, 2.0); + assert_eq!( + result.effective_eval_time_limit_seconds, + effective_limit_seconds + ); + assert_eq!(result.peak_memory_bytes, 16); + assert_eq!(result.target_loss_bits, f64::INFINITY); + assert_eq!(result.objective_bits, f64::INFINITY); + assert_eq!(result.throughput_bytes_per_second, 0.0); + assert!(!result.deployable); +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn candidate_bounds_validation_enforces_parameter_ranges() { + let candidate = CompressionBackend::Rate { + rate_backend: RateBackend::Ctw { depth: 8 }, + coder: crate::coders::CoderType::AC, + framing: FramingMode::Framed, + }; + let bounds = TuneBoundsSpec { + allowed_backends: vec!["ctw".to_string()], + forbidden_backends: Vec::new(), + parameter_ranges: vec![crate::spec::TuneParameterRangeSpec { + parameter: "rate_backend.depth".to_string(), + min: 4.0, + max: 6.0, + }], + max_experts: 2, + max_mixture_nesting_depth: 1, + min_experts: Some(1), + allow_duplicate_experts: Some(false), + required_experts: Vec::new(), + forbidden_expert_pairs: Vec::new(), + }; + let err = validate_candidate_against_tune_bounds(&candidate, &bounds) + .expect_err("depth out of range must fail"); + assert!(err.contains("rate_backend.depth")); +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn canonical_proposal_kernel_accounts_exact_integer_masses() { + let candidate = CompressionBackend::Rate { + rate_backend: RateBackend::Ctw { depth: 2 }, + coder: crate::coders::CoderType::AC, + framing: FramingMode::Framed, + }; + let bounds = TuneBoundsSpec { + allowed_backends: vec!["ctw".to_string()], + forbidden_backends: Vec::new(), + parameter_ranges: vec![crate::spec::TuneParameterRangeSpec { + parameter: "rate_backend.depth".to_string(), + min: 1.0, + max: 3.0, + }], + max_experts: 2, + max_mixture_nesting_depth: 1, + min_experts: Some(1), + allow_duplicate_experts: Some(false), + required_experts: Vec::new(), + forbidden_expert_pairs: Vec::new(), + }; + let env = SpecEnvironment::new("."); + let current = candidate.compile_in(&env).expect("compile current"); + let current_bytes = current.canonical_bytes().as_slice().to_vec(); + let kernel = compile_canonical_proposal_kernel(&candidate, &bounds, 1, 1, &env, ¤t_bytes) + .expect("compile proposal kernel"); + assert_eq!(kernel.total_raw_actions, 2); + assert_eq!(kernel.transitions.len(), 2); + assert!( + kernel + .transitions + .iter() + .all(|proposal| proposal.raw_action_count == 1) + ); + + let lower = CompressionBackend::Rate { + rate_backend: RateBackend::Ctw { depth: 1 }, + coder: crate::coders::CoderType::AC, + framing: FramingMode::Framed, + }; + let lower_bytes = lower + .compile_in(&env) + .expect("compile lower") + .canonical_bytes() + .as_slice() + .to_vec(); + assert_eq!(kernel.proposal_mass_to_canonical_bytes(&lower_bytes), 1); + + let reverse = compile_canonical_proposal_kernel(&lower, &bounds, 1, 1, &env, &lower_bytes) + .expect("compile reverse kernel"); + assert_eq!(reverse.total_raw_actions, 2); + assert_eq!(reverse.proposal_mass_to_canonical_bytes(¤t_bytes), 1); + assert_eq!(reverse.transitions.len(), 1); +} + +#[cfg(feature = "backend-match")] +#[test] +fn canonical_proposal_kernel_explores_bounded_float_only_search_space() { + use crate::api::RateBackend; + use crate::compression::FramingMode; + use crate::spec::{TuneBoundsSpec, TuneParameterRangeSpec}; + + let candidate = CompressionBackend::Rate { + rate_backend: RateBackend::Match { + hash_bits: 18, + min_len: 4, + max_len: 64, + base_mix: 0.02, + confidence_scale: 1.0, + }, + coder: crate::coders::CoderType::AC, + framing: FramingMode::Framed, + }; + let bounds = TuneBoundsSpec { + allowed_backends: vec!["match".to_string()], + forbidden_backends: Vec::new(), + parameter_ranges: vec![TuneParameterRangeSpec { + parameter: "rate_backend.base_mix".to_string(), + min: 0.01, + max: 0.04, + }], + max_experts: 2, + max_mixture_nesting_depth: 1, + min_experts: Some(1), + allow_duplicate_experts: Some(false), + required_experts: Vec::new(), + forbidden_expert_pairs: Vec::new(), + }; + let env = SpecEnvironment::new("."); + let current = candidate.compile_in(&env).expect("compile current"); + let current_bytes = current.canonical_bytes().as_slice().to_vec(); + let kernel = compile_canonical_proposal_kernel(&candidate, &bounds, 2, 2, &env, ¤t_bytes) + .expect("compile float proposal kernel"); + + assert_eq!(kernel.total_raw_actions, 4); + assert!( + !kernel.transitions.is_empty(), + "bounded float-only search spaces must produce non-self proposals" + ); + + for proposal in &kernel.transitions { + let CompressionBackend::Rate { + rate_backend: RateBackend::Match { base_mix, .. }, + .. + } = &proposal.candidate + else { + panic!("expected match proposal"); + }; + assert!((0.01..=0.04).contains(base_mix)); + + let reverse = compile_canonical_proposal_kernel( + &proposal.candidate, + &bounds, + 2, + 2, + &env, + &proposal.candidate_canonical_bytes, + ) + .expect("compile reverse float proposal kernel"); + assert_eq!(reverse.total_raw_actions, 4); + assert_eq!( + reverse.proposal_mass_to_canonical_bytes(¤t_bytes), + proposal.raw_action_count + ); + } +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn reversible_metropolis_acceptance_uses_objective_bits_temperature() { + let proposal = AnnealedProposal { + candidate: CompressionBackend::Rate { + rate_backend: RateBackend::Ctw { depth: 1 }, + coder: crate::coders::CoderType::AC, + framing: FramingMode::Framed, + }, + forward_raw_action_count: 1, + forward_total_raw_actions: 2, + reverse_raw_action_count: 1, + reverse_total_raw_actions: 2, + }; + let uphill = annealer_acceptance_probability( + AnnealerKernelProfile::ReversibleElementaryMetropolis, + 3.0, + 2.0, + &proposal, + ) + .expect("reversible metropolis probability"); + assert!((uphill - (-1.5f64).exp()).abs() <= f64::EPSILON); + let downhill = annealer_acceptance_probability( + AnnealerKernelProfile::ReversibleElementaryMetropolis, + -2.0, + 3.0, + &proposal, + ) + .expect("reversible metropolis probability"); + assert!((downhill - 1.0).abs() <= f64::EPSILON); +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn metropolis_acceptance_envelope_sweep() { + use crate::api::{CompressionBackend, RateBackend}; + use crate::compression::FramingMode; + use crate::tuner::AnnealedProposal; + use crate::tuner::annealer::annealer_acceptance_probability; + use crate::tuner::config::AnnealerKernelProfile; + + let proposal = AnnealedProposal { + candidate: CompressionBackend::Rate { + rate_backend: RateBackend::Ctw { depth: 1 }, + coder: crate::coders::CoderType::AC, + framing: FramingMode::Framed, + }, + forward_raw_action_count: 1, + forward_total_raw_actions: 2, + reverse_raw_action_count: 1, + reverse_total_raw_actions: 2, + }; + + let mut previous_probability: Option = None; + for t in 1..=100 { + let temp = t as f64; + let delta = 2.0; + let prob = annealer_acceptance_probability( + AnnealerKernelProfile::ReversibleElementaryMetropolis, + delta, + temp, + &proposal, + ) + .unwrap(); + let expected = (-delta / temp).exp().clamp(0.0, 1.0); + + assert!( + (prob - expected).abs() <= f64::EPSILON, + "uphill Metropolis probability must equal exp(-delta / temperature): prob={prob}, expected={expected}, temperature={temp}" + ); + if let Some(previous) = previous_probability { + assert!( + previous < prob, + "uphill Metropolis probability must strictly increase with temperature: previous={previous}, current={prob}, temperature={temp}" + ); + } + previous_probability = Some(prob); + + let delta_neg = -2.0; + let prob_neg = annealer_acceptance_probability( + AnnealerKernelProfile::ReversibleElementaryMetropolis, + delta_neg, + temp, + &proposal, + ) + .unwrap(); + assert_eq!(prob_neg, 1.0, "Negative delta must always be accepted"); + } +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn compiled_uniform_mh_uses_hastings_ratio_for_asymmetric_boundary_mass() { + let proposal = AnnealedProposal { + candidate: CompressionBackend::Rate { + rate_backend: RateBackend::Ctw { depth: 1 }, + coder: crate::coders::CoderType::AC, + framing: FramingMode::Framed, + }, + forward_raw_action_count: 1, + forward_total_raw_actions: 2, + reverse_raw_action_count: 1, + reverse_total_raw_actions: 4, + }; + let probability = annealer_acceptance_probability( + AnnealerKernelProfile::CompiledUniformMetropolisHastings, + 1.0, + 1.0, + &proposal, + ) + .expect("mh probability"); + let expected = (-1.0f64).exp() * 0.5; + assert!((probability - expected).abs() <= f64::EPSILON); + let err = annealer_acceptance_probability( + AnnealerKernelProfile::ReversibleElementaryMetropolis, + 1.0, + 1.0, + &proposal, + ) + .expect_err("default profile must reject asymmetric masses"); + assert!(err.contains("reversibility check")); +} + +#[test] +fn key_less_uses_canonical_bytes_on_objective_ties() { + let eval = CandidateEvalResult { + status: CandidateEvalStatus::Success, + compressed_bytes: 0, + elapsed_seconds: 1.0, + effective_eval_time_limit_seconds: 1.0, + throughput_bytes_per_second: 1.0, + peak_memory_bytes: 1, + target_loss_bits: 1.0, + objective_bits: 42.0, + deployable: true, + }; + let smaller = vec![0x01_u8, 0x02_u8]; + let larger = vec![0x01_u8, 0x03_u8]; + assert!(key_less(&eval, &smaller, &eval, &larger)); + assert!(!key_less(&eval, &larger, &eval, &smaller)); +} + +#[test] +fn key_less_requires_exact_objective_tie_before_byte_tiebreak() { + let incumbent = CandidateEvalResult { + status: CandidateEvalStatus::Success, + compressed_bytes: 0, + elapsed_seconds: 1.0, + effective_eval_time_limit_seconds: 1.0, + throughput_bytes_per_second: 1.0, + peak_memory_bytes: 1, + target_loss_bits: 1.0, + objective_bits: 42.0, + deployable: true, + }; + let candidate = CandidateEvalResult { + objective_bits: f64::from_bits(incumbent.objective_bits.to_bits() + 1), + ..incumbent.clone() + }; + let candidate_bytes = vec![0x01_u8, 0x00_u8]; + let incumbent_bytes = vec![0x01_u8, 0x01_u8]; + assert!( + !key_less(&candidate, &candidate_bytes, &incumbent, &incumbent_bytes), + "byte-order tiebreak must not apply unless objective bits are exactly equal" + ); +} + +fn decode_observation_optional_f64(bytes: &[u8], field_index: usize) -> Option { + let mut offset = 1usize; + for index in 0..5 { + let present = bytes[offset]; + offset += 1; + if present == 1 { + let mut raw = [0_u8; 8]; + raw.copy_from_slice(&bytes[offset..offset + 8]); + let value = f64::from_bits(u64::from_le_bytes(raw)); + offset += 8; + if index == field_index { + return Some(value); + } + } else if index == field_index { + return None; + } + } + None +} + +#[test] +fn raw_observation_timeout_sets_tau_one_and_invalid_uses_sentinel() { + let incumbent = CandidateEvalResult { + status: CandidateEvalStatus::Success, + compressed_bytes: 10, + elapsed_seconds: 0.5, + effective_eval_time_limit_seconds: 1.0, + throughput_bytes_per_second: 2.0, + peak_memory_bytes: 1, + target_loss_bits: 80.0, + objective_bits: 100.0, + deployable: true, + }; + let timeout = timeout_eval_result(0.25, 1, 0.25); + let timeout_observation = TunerRawObservation::from_runtime_step( + Some(&incumbent), + 10.0, + Some(&timeout), + Some(b"candidate-timeout"), + Some(0.25), + "evaluator_timeout", + false, + ); + assert_eq!( + decode_observation_optional_f64(timeout_observation.encoded_bytes(), 2), + Some(1.0) + ); + let invalid = CandidateEvalResult { + status: CandidateEvalStatus::Invalid, + compressed_bytes: 0, + elapsed_seconds: 0.0, + effective_eval_time_limit_seconds: 1.0, + throughput_bytes_per_second: 0.0, + peak_memory_bytes: 0, + target_loss_bits: f64::INFINITY, + objective_bits: f64::INFINITY, + deployable: false, + }; + let invalid_observation = TunerRawObservation::from_runtime_step( + Some(&incumbent), + 10.0, + Some(&invalid), + Some(b"candidate-invalid"), + Some(1.0), + "evaluator_invalid", + false, + ); + assert_eq!( + decode_observation_optional_f64(invalid_observation.encoded_bytes(), 2), + None + ); +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn planner_percept_encoding_distinguishes_diagnostic_tokens() { + let interface = TunePlannerInterfaceSpec { + observation_bits: 16, + observation_stream_len: 2, + observation_key_mode: ObservationKeyMode::FullStream, + reward_bits: 8, + agent_actions: action_alphabet(2), + }; + let current = vec![1_u8, 2, 3]; + let incumbent_eval = CandidateEvalResult { + status: CandidateEvalStatus::Success, + compressed_bytes: 12, + elapsed_seconds: 0.25, + effective_eval_time_limit_seconds: 1.0, + throughput_bytes_per_second: 4.0, + peak_memory_bytes: 1, + target_loss_bits: 96.0, + objective_bits: 128.0, + deployable: true, + }; + let inapplicable = encode_tuner_planner_percept( + &interface, + Some(&incumbent_eval), + 16.0, + 0, + "inapplicable_action", + None, + None, + None, + false, + ) + .expect("inapplicable percept"); + let invalid = encode_tuner_planner_percept( + &interface, + Some(&incumbent_eval), + 16.0, + 0, + "invalid_action_index", + None, + None, + None, + false, + ) + .expect("invalid percept"); + let nondeployable = encode_tuner_planner_percept( + &interface, + Some(&incumbent_eval), + 16.0, + 0, + "nondeployable_candidate", + Some(&CandidateEvalResult { + status: CandidateEvalStatus::Invalid, + compressed_bytes: 0, + elapsed_seconds: 0.0, + effective_eval_time_limit_seconds: 1.0, + throughput_bytes_per_second: 0.0, + peak_memory_bytes: 0, + target_loss_bits: f64::INFINITY, + objective_bits: f64::INFINITY, + deployable: false, + }), + Some(¤t), + Some(1.0), + false, + ) + .expect("nondeployable percept"); + assert_ne!(inapplicable.observations, invalid.observations); + assert_ne!(inapplicable.observations, nondeployable.observations); + assert_ne!(invalid.observations, nondeployable.observations); +} + +#[test] +fn theorem_claims_reject_float_planner_mutation_domains() { + let actions = vec![ + PlannerMutationAction::NumericStep { + path: "rate_backend.temperature".to_string(), + pointer: "/rate_backend/temperature".to_string(), + kind: NumericKind::Float, + delta: 0.05, + range: None, + }, + PlannerMutationAction::Noop, + ]; + let mut theorem = TuneTheoremConfig::default(); + validate_theorem_planner_mutation_domain(&actions, &theorem) + .expect("operational run may use float mutation leaves"); + theorem.claim_exact_finite_mdp = true; + let err = validate_theorem_planner_mutation_domain(&actions, &theorem) + .expect_err("exact theorem claim must reject float mutation leaves"); + assert!(err.contains("theorem_finite_state_unsafe"), "{err}"); +} + +#[cfg(all(feature = "backend-ctw", feature = "backend-mixture"))] +#[test] +fn planner_float_mutation_uses_bounds_scale_for_small_positive_alpha() { + use std::sync::Arc; + + let candidate = CompressionBackend::Rate { + rate_backend: RateBackend::Mixture { + spec: Arc::new( + crate::api::MixtureSpec::new( + crate::api::MixtureKind::Neural, + vec![ + crate::api::MixtureExpertSpec::new(RateBackend::Ctw { depth: 4 }) + .with_name("ctw"), + ], + ) + .with_alpha(0.03), + ), + }, + coder: crate::coders::CoderType::AC, + framing: FramingMode::Framed, + }; + let action = PlannerMutationAction::NumericStep { + path: "rate_backend.spec.alpha".to_string(), + pointer: "/rate_backend/spec/alpha".to_string(), + kind: NumericKind::Float, + delta: -0.05, + range: Some((0.005, 0.2)), + }; + + let mutated = apply_planner_mutation_action(&candidate, &action) + .expect("planner mutation should decode") + .expect("bounded float action should remain applicable"); + let json = crate::spec::compression_backend_to_json_value(&mutated) + .expect("mutated candidate should serialize"); + let alpha = json + .pointer("/rate_backend/spec/alpha") + .and_then(Value::as_f64) + .expect("mutated mixture alpha"); + + assert!(alpha > 0.005, "alpha should remain within declared bounds"); + assert!(alpha < 0.03, "negative planner step should decrease alpha"); +} + +#[cfg(all(feature = "backend-ctw", feature = "backend-mixture"))] +#[test] +fn planner_unparsable_float_mutation_is_inapplicable_not_fatal() { + use std::sync::Arc; + + let candidate = CompressionBackend::Rate { + rate_backend: RateBackend::Mixture { + spec: Arc::new( + crate::api::MixtureSpec::new( + crate::api::MixtureKind::Neural, + vec![ + crate::api::MixtureExpertSpec::new(RateBackend::Ctw { depth: 4 }) + .with_name("ctw"), + ], + ) + .with_alpha(0.03), + ), + }, + coder: crate::coders::CoderType::AC, + framing: FramingMode::Framed, + }; + let action = PlannerMutationAction::NumericStep { + path: "rate_backend.spec.alpha".to_string(), + pointer: "/rate_backend/spec/alpha".to_string(), + kind: NumericKind::Float, + delta: -0.05, + range: None, + }; + + let mutated = apply_planner_mutation_action(&candidate, &action) + .expect("unparsable planner edit should not abort the run"); + assert!( + mutated.is_none(), + "invalid float planner edit should be treated as inapplicable" + ); +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn theorem_claims_continue_as_uncertified_when_requested_prereqs_are_missing() { + let dataset_path = temp_path("dataset-theorem-policy", ".bin"); + let output_path = temp_path("output-theorem-policy", ".json"); + let report_path = temp_path("report-theorem-policy", ".json"); + std::fs::write(&dataset_path, b"theorem policy dataset").expect("write dataset"); + let mut spec = sample_tune_spec( + dataset_path.to_str().expect("dataset path"), + output_path.to_str().expect("output path"), + report_path.to_str().expect("report path"), + ); + let interface = planner_interface_for_baseline(&spec.baseline_candidate); + spec.controller = TuneControllerSpec::McAixiFacCtw(McAixiFacCtwTuneControllerSpec { + interface, + planner_simulations_per_step: 2, + }); + let best_candidate = spec.baseline_candidate.clone(); + let compiled = spec.compile().expect("compile tune spec"); + let dataset = load_dataset(&dataset_path).expect("load dataset"); + let search = SearchSummary { + status: "completed_mc_aixi_fac_ctw", + warning: None, + fatal_evaluator_failure: None, + fatal_evaluator_failures: 0, + best_candidate, + best_candidate_crc32: "00000000".to_string(), + best_eval: CandidateEvalResult { + status: CandidateEvalStatus::Success, + compressed_bytes: 8, + elapsed_seconds: 0.1, + effective_eval_time_limit_seconds: 1.0, + throughput_bytes_per_second: 10.0, + peak_memory_bytes: 1, + target_loss_bits: 64.0, + objective_bits: 128.0, + deployable: true, + }, + cache_key_digest: "00000000".to_string(), + cache_hits: 0, + cache_misses: 1, + candidate_evaluations_executed: 1, + non_warmup_candidate_results_seen: 1, + post_baseline_candidate_results_seen: 0, + proposals_attempted: 0, + proposals_invalid: 0, + self_loop_proposals: 0, + invalid_reason_counts: InvalidReasonCounts::default(), + successful_non_deployable: 0, + candidate_result_counts: CandidateResultCounts { + success_deployable: 1, + success_non_deployable: 0, + timeout: 0, + invalid: 0, + error_recoverable: 0, + }, + final_best_move_reward: 0.0, + realized_trace_counts_by_round: None, + trace_refresh_merges_by_round: None, + controller_report: Value::Null, + }; + let theorem = TuneTheoremConfig { + claim_exact_finite_mdp: true, + claim_exact_observed_markov: true, + claim_planner_convergence: true, + ..TuneTheoremConfig::default() + }; + + let report = theorem_claims_report( + &theorem, + &VerifiedTheoremInputs::default(), + compiled.controller(), + &dataset, + &search, + false, + ); + for pointer in [ + "/exact_finite_mdp/status", + "/exact_observed_markov/status", + "/planner_convergence/status", + ] { + assert_eq!( + report.pointer(pointer).and_then(Value::as_str), + Some("uncertified") + ); + } + assert!( + report["exact_observed_markov"]["missing_prerequisites"] + .as_array() + .expect("missing prerequisites") + .iter() + .any(|item| item.as_str() == Some("verified_exact_state_observation_certificate")) + ); + + let _ = std::fs::remove_file(dataset_path); + let _ = std::fs::remove_file(output_path); + let _ = std::fs::remove_file(report_path); +} + +#[test] +fn candidate_external_asset_references_are_rejected() { + let candidate = CompressionBackend::zpaq("file:./candidate-model.zpaq"); + let err = reject_candidate_local_external_artifacts(&candidate) + .expect_err("external file reference must fail"); + assert_eq!( + err.reason, + TuneInvalidReason::CandidateExternalAssetForbidden + ); + assert!( + err.diagnostic + .contains(TuneInvalidReason::CandidateExternalAssetForbidden.as_str()) + ); + assert!( + err.diagnostic + .contains("candidate-local external filesystem/model path") + ); +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn run_tune_writes_output_and_report_for_baseline_pass() { + let dataset_path = temp_path("dataset", ".bin"); + let spec_path = temp_path("spec", ".json"); + let output_path = temp_path("output", ".json"); + let report_path = temp_path("report", ".json"); + std::fs::write(&dataset_path, b"hello baseline").expect("write dataset"); + + let spec = sample_tune_spec( + dataset_path.to_str().expect("dataset path"), + output_path.to_str().expect("output path"), + report_path.to_str().expect("report path"), + ); + let spec_json = SpecDocument::Tune(spec) + .to_canonical_json() + .expect("spec json"); + std::fs::write(&spec_path, spec_json).expect("write spec"); + + let request = TuneCommandRequest { + spec_path: spec_path.to_string_lossy().to_string(), + emit_exact_reward_encoding_certificate: None, + execution: TuneExecutionConfig::default(), + }; + run_tune(&request).expect("run tune"); + + let output = std::fs::read_to_string(&output_path).expect("output exists"); + assert!(output.contains("\"kind\": \"rate-ac\"")); + let report = std::fs::read_to_string(&report_path).expect("report exists"); + assert!(report.contains("\"kind\": \"tune_report\"")); + + let _ = std::fs::remove_file(dataset_path); + let _ = std::fs::remove_file(spec_path); + let _ = std::fs::remove_file(output_path); + let _ = std::fs::remove_file(report_path); +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn run_tune_fails_when_baseline_not_deployable() { + let dataset_path = temp_path("dataset", ".bin"); + let spec_path = temp_path("spec", ".json"); + let output_path = temp_path("output", ".json"); + let report_path = temp_path("report", ".json"); + std::fs::write(&dataset_path, vec![0u8; 4096]).expect("write dataset"); + + let mut spec = sample_tune_spec( + dataset_path.to_str().expect("dataset path"), + output_path.to_str().expect("output path"), + report_path.to_str().expect("report path"), + ); + spec.min_throughput_bytes_per_second = f64::MAX; + let spec_json = SpecDocument::Tune(spec) + .to_canonical_json() + .expect("spec json"); + std::fs::write(&spec_path, spec_json).expect("write spec"); + + let request = TuneCommandRequest { + spec_path: spec_path.to_string_lossy().to_string(), + emit_exact_reward_encoding_certificate: None, + execution: TuneExecutionConfig::default(), + }; + let err = run_tune(&request).expect_err("non-deployable baseline must fail"); + assert!(err.contains("not deployable")); + + let report = std::fs::read_to_string(&report_path).expect("report exists"); + assert!(report.contains("\"status\": \"baseline_not_deployable\"")); + assert!(!output_path.exists()); + + let _ = std::fs::remove_file(dataset_path); + let _ = std::fs::remove_file(spec_path); + let _ = std::fs::remove_file(report_path); +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn run_tune_can_emit_exact_reward_certificate_and_exit() { + let dataset_path = temp_path("dataset-emit-reward", ".bin"); + let spec_path = temp_path("spec-emit-reward", ".json"); + let output_path = temp_path("output-emit-reward", ".json"); + let report_path = temp_path("report-emit-reward", ".json"); + let emitted_cert_path = temp_path("exact-reward-emitted", ".json"); + std::fs::write(&dataset_path, b"emit-reward-dataset").expect("write dataset"); + + let mut spec = sample_tune_spec( + dataset_path.to_str().expect("dataset path"), + output_path.to_str().expect("output path"), + report_path.to_str().expect("report path"), + ); + spec.controller = TuneControllerSpec::McAixiFacCtw(McAixiFacCtwTuneControllerSpec { + interface: planner_interface_for_baseline(&spec.baseline_candidate), + planner_simulations_per_step: 4, + }); + let spec_json = SpecDocument::Tune(spec) + .to_canonical_json() + .expect("spec json"); + std::fs::write(&spec_path, spec_json).expect("write spec"); + + let request = TuneCommandRequest { + spec_path: spec_path.to_string_lossy().to_string(), + emit_exact_reward_encoding_certificate: Some( + emitted_cert_path.to_string_lossy().to_string(), + ), + execution: TuneExecutionConfig::default(), + }; + run_tune(&request).expect("emit exact reward certificate"); + + let cert_bytes = std::fs::read(&emitted_cert_path).expect("read emitted certificate"); + let cert: Value = serde_json::from_slice(&cert_bytes).expect("parse emitted certificate"); + assert_eq!(cert["kind"], serde_json::json!("exact_reward_encoding")); + assert_eq!( + cert["controller_kind"], + serde_json::json!("mc_aixi_fac_ctw") + ); + assert_eq!( + cert["encoding"], + serde_json::json!("integer_objective_difference") + ); + assert_eq!( + cert["scalar_representation"], + serde_json::json!(SCALAR_REPRESENTATION_DECLARATION) + ); + assert!( + cert["dataset_crc32"].as_str().is_some(), + "dataset_crc32 must be emitted" + ); + assert!( + cert["bounds_crc32"].as_str().is_some(), + "bounds_crc32 must be emitted" + ); + assert!( + cert["evaluator_profile_crc32"].as_str().is_some(), + "evaluator_profile_crc32 must be emitted" + ); + assert!( + !output_path.exists(), + "emit mode should not run candidate evaluation or write output config" + ); + assert!( + !report_path.exists(), + "emit mode should exit before tune report generation" + ); + + let _ = std::fs::remove_file(dataset_path); + let _ = std::fs::remove_file(spec_path); + let _ = std::fs::remove_file(emitted_cert_path); +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn run_tune_emit_exact_reward_certificate_rejects_non_exact_controller_family() { + let dataset_path = temp_path("dataset-emit-reward-nonexact", ".bin"); + let spec_path = temp_path("spec-emit-reward-nonexact", ".json"); + let output_path = temp_path("output-emit-reward-nonexact", ".json"); + let report_path = temp_path("report-emit-reward-nonexact", ".json"); + let emitted_cert_path = temp_path("exact-reward-emitted-nonexact", ".json"); + std::fs::write(&dataset_path, b"emit-reward-dataset-nonexact").expect("write dataset"); + let spec = sample_tune_spec( + dataset_path.to_str().expect("dataset path"), + output_path.to_str().expect("output path"), + report_path.to_str().expect("report path"), + ); + let spec_json = SpecDocument::Tune(spec) + .to_canonical_json() + .expect("spec json"); + std::fs::write(&spec_path, spec_json).expect("write spec"); + + let request = TuneCommandRequest { + spec_path: spec_path.to_string_lossy().to_string(), + emit_exact_reward_encoding_certificate: Some( + emitted_cert_path.to_string_lossy().to_string(), + ), + execution: TuneExecutionConfig::default(), + }; + let err = run_tune(&request).expect_err("non-exact family must be rejected"); + assert!( + err.contains("exact reward-encoding certificate emission is only supported"), + "{err}" + ); + + assert!( + !emitted_cert_path.exists(), + "rejected emitter path must not write a certificate" + ); + + let _ = std::fs::remove_file(dataset_path); + let _ = std::fs::remove_file(spec_path); +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn structured_causal_dataset_objects_lower_into_charged_targets() { + let dataset_path = temp_path("dataset-causal", ".json"); + std::fs::write( + &dataset_path, + causal_dataset_value( + "test-codec", + "events", + serde_json::json!([ + {"kind": "context", "channel": "action", "bytes": [1]}, + {"kind": "observe_target_no_score", "channel": "percept", "domain": "bytes", "bytes": [2]}, + {"kind": "target", "channel": "percept", "domain": "bytes", "bytes": [3, 4]} + ]), + ) + .to_string(), + ) + .expect("write dataset"); + + let dataset = load_dataset(&dataset_path).expect("causal dataset lowers"); + assert_eq!(dataset.kind, DatasetKind::InteractiveTrace); + assert_eq!( + dataset.objective_target, + ObjectiveTarget::InteractiveCausalAc + ); + assert_eq!(dataset.lowering_version, INTERACTIVE_TRACE_LOWERING_VERSION); + assert_eq!(dataset.codec_hash, "test-codec"); + assert_eq!(dataset.raw_bytes, vec![3, 4]); + assert_eq!(dataset.target_events, 2); + assert_eq!(dataset.dataset_units, 2.0); + assert!(matches!( + &dataset.events[0], + LoweredCausalEvent::Context { channel, bytes } + if channel == "action" && bytes == &[1] + )); + assert!(matches!( + &dataset.events[1], + LoweredCausalEvent::ObserveTargetNoScore { + channel, + domain, + bytes, + } if channel == "percept" && domain == "bytes" && bytes == &[2] + )); + assert!(matches!( + &dataset.events[2], + LoweredCausalEvent::Target { + channel, + domain, + bytes, + weight, + } if channel == "percept" && domain == "bytes" && bytes == &[3] && *weight == 1.0 + )); + assert!(matches!( + &dataset.events[3], + LoweredCausalEvent::Target { + channel, + domain, + bytes, + weight, + } if channel == "percept" && domain == "bytes" && bytes == &[4] && *weight == 1.0 + )); + + let _ = std::fs::remove_file(dataset_path); +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn causal_prefix_lowering_resets_examples_and_replays_targets_without_score() { + let dataset_path = temp_path("dataset-prefix-semantics", ".json"); + std::fs::write( + &dataset_path, + causal_dataset_value( + "test-prefix-codec", + "examples", + serde_json::json!([ + { + "history": [ + {"kind": "observe_target_no_score", "channel": "percept", "domain": "bytes", "bytes": [7]} + ], + "action": [1], + "channel": "percept", + "domain": "bytes", + "target": [8], + "weight": 2.0 + }, + { + "action": [0], + "channel": "percept", + "domain": "bytes", + "target": [9] + } + ]), + ) + .to_string(), + ) + .expect("write causal-prefix dataset"); + + let dataset = load_dataset(&dataset_path).expect("causal-prefix dataset lowers"); + assert_eq!(dataset.kind, DatasetKind::CausalPrefixDataset); + assert_eq!(dataset.raw_bytes, vec![8, 9]); + assert_eq!(dataset.target_events, 2); + assert_eq!(dataset.dataset_units, 3.0); + assert_eq!( + dataset + .events + .iter() + .filter(|event| matches!(event, LoweredCausalEvent::Reset)) + .count(), + 2 + ); + assert!(matches!(&dataset.events[0], LoweredCausalEvent::Reset)); + assert!(matches!( + &dataset.events[1], + LoweredCausalEvent::ObserveTargetNoScore { bytes, .. } if bytes == &[7] + )); + assert!(matches!( + &dataset.events[2], + LoweredCausalEvent::Context { channel, bytes } + if channel == "action" && bytes == &[1] + )); + assert!(matches!( + &dataset.events[3], + LoweredCausalEvent::Target { bytes, weight, .. } + if bytes == &[8] && *weight == 2.0 + )); + assert!(matches!(&dataset.events[4], LoweredCausalEvent::Reset)); + assert!(matches!( + &dataset.events[5], + LoweredCausalEvent::Context { channel, bytes } + if channel == "action" && bytes == &[0] + )); + assert!(matches!( + &dataset.events[6], + LoweredCausalEvent::Target { bytes, weight, .. } + if bytes == &[9] && *weight == 1.0 + )); + + let _ = std::fs::remove_file(dataset_path); +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn structured_causal_dataset_rejects_missing_header_and_charged_history() { + let missing_header_path = temp_path("dataset-missing-causal-header", ".json"); + std::fs::write( + &missing_header_path, + serde_json::json!({ + "schema_version": 1, + "events": [{"kind": "target", "bytes": [1]}] + }) + .to_string(), + ) + .expect("write missing-header dataset"); + let err = load_dataset(&missing_header_path).expect_err("header must be required"); + assert!(err.contains("environment_id is required"), "{err}"); + + let malformed_structured_path = temp_path("dataset-malformed-structured", ".json"); + std::fs::write( + &malformed_structured_path, + serde_json::json!({ + "schema_version": 1, + "codec_hash": "looks-structured" + }) + .to_string(), + ) + .expect("write malformed structured dataset"); + let err = load_dataset(&malformed_structured_path) + .expect_err("structured object must not be passive"); + assert!( + err.contains("must match a canonical tuner causal dataset kind"), + "{err}" + ); + + let charged_history_path = temp_path("dataset-charged-history", ".json"); + std::fs::write( + &charged_history_path, + causal_dataset_value( + "charged-history", + "examples", + serde_json::json!([{ + "history": [{"kind": "target", "channel": "percept", "domain": "bytes", "bytes": [7]}], + "action": [1], + "channel": "percept", + "domain": "bytes", + "target": [8] + }]), + ) + .to_string(), + ) + .expect("write charged-history dataset"); + let err = load_dataset(&charged_history_path).expect_err("charged history must fail"); + assert!(err.contains("observe_target_no_score"), "{err}"); + + let _ = std::fs::remove_file(missing_header_path); + let _ = std::fs::remove_file(malformed_structured_path); + let _ = std::fs::remove_file(charged_history_path); +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn causal_dataset_header_and_event_grammar_are_strict() { + let invalid_header_path = temp_path("dataset-invalid-header-types", ".json"); + std::fs::write( + &invalid_header_path, + serde_json::json!({ + "schema_version": 1, + "environment_id": 7, + "environment_config_crc32": "00000000", + "codec_hash": "codec", + "reset_convention": "reset-before-episode", + "action_alphabet": {"size": 2}, + "percept_schema": {"encoding": "bytes"}, + "reward_encoding": {"encoding": "bytes"}, + "terminal_encoding": {"encoding": "bytes"}, + "collection_policy": "test", + "events": [{"kind": "target", "channel": "percept", "domain": "bytes", "bytes": [1]}] + }) + .to_string(), + ) + .expect("write invalid-header dataset"); + let err = load_dataset(&invalid_header_path).expect_err("invalid header type must fail"); + assert!(err.contains("environment_id is required"), "{err}"); + + let alias_event_path = temp_path("dataset-alias-event-kind", ".json"); + std::fs::write( + &alias_event_path, + causal_dataset_value( + "test-codec", + "events", + serde_json::json!([ + {"kind": "context", "channel": "action", "bytes": [1]}, + {"kind": "observe", "channel": "percept", "domain": "bytes", "bytes": [2]}, + ]), + ) + .to_string(), + ) + .expect("write alias-event dataset"); + let err = load_dataset(&alias_event_path).expect_err("alias event kind must fail"); + assert!(err.contains("unknown causal event kind"), "{err}"); + + let missing_event_grammar_path = temp_path("dataset-missing-event-grammar", ".json"); + std::fs::write( + &missing_event_grammar_path, + serde_json::json!({ + "schema_version": 1, + "environment_id": "env", + "environment_config_crc32": "00000000", + "codec_hash": "codec", + "reset_convention": "reset-before-episode", + "action_alphabet": {"size": 2}, + "percept_schema": {"encoding": "bytes", "channels": [{"channel": "percept", "domain": "bytes"}]}, + "reward_encoding": {"encoding": "bytes", "channel": "reward", "domain": "binary"}, + "terminal_encoding": {"encoding": "bytes", "channel": "terminal", "domain": "binary"}, + "collection_policy": "test", + "target_domains": { + "bytes": {"kind": "byte_alphabet"}, + "binary": {"kind": "enumerated_payloads", "payloads": [[0], [1]]} + }, + "events": [{"kind": "target", "channel": "percept", "domain": "bytes", "bytes": [1]}] + }) + .to_string(), + ) + .expect("write missing-event-grammar dataset"); + let err = + load_dataset(&missing_event_grammar_path).expect_err("missing event_grammar must fail"); + assert!(err.contains("requires event_grammar"), "{err}"); + + let missing_domain_path = temp_path("dataset-missing-domain", ".json"); + std::fs::write( + &missing_domain_path, + causal_dataset_value( + "test-codec", + "events", + serde_json::json!([ + {"kind": "context", "channel": "action", "bytes": [1]}, + {"kind": "target", "channel": "percept", "bytes": [2]}, + ]), + ) + .to_string(), + ) + .expect("write missing-domain dataset"); + let err = load_dataset(&missing_domain_path).expect_err("missing domain must fail"); + assert!(err.contains(".domain is required"), "{err}"); + + let _ = std::fs::remove_file(invalid_header_path); + let _ = std::fs::remove_file(alias_event_path); + let _ = std::fs::remove_file(missing_event_grammar_path); + let _ = std::fs::remove_file(missing_domain_path); +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn byte_alphabet_payloads_expand_to_single_byte_events() { + let dataset_path = temp_path("dataset-byte-alphabet-expand", ".json"); + std::fs::write( + &dataset_path, + causal_dataset_value( + "byte-expand-codec", + "events", + serde_json::json!([ + {"kind": "observe_target_no_score", "channel": "percept", "domain": "bytes", "bytes": [3, 4]}, + {"kind": "target", "channel": "percept", "domain": "bytes", "bytes": [7, 8], "weight": 2.0} + ]), + ) + .to_string(), + ) + .expect("write byte-alphabet expansion dataset"); + let dataset = load_dataset(&dataset_path).expect("dataset lowers"); + assert!(matches!( + &dataset.events[0], + LoweredCausalEvent::ObserveTargetNoScore { bytes, .. } if bytes == &[3] + )); + assert!(matches!( + &dataset.events[1], + LoweredCausalEvent::ObserveTargetNoScore { bytes, .. } if bytes == &[4] + )); + assert!(matches!( + &dataset.events[2], + LoweredCausalEvent::Target { bytes, weight, .. } if bytes == &[7] && *weight == 2.0 + )); + assert!(matches!( + &dataset.events[3], + LoweredCausalEvent::Target { bytes, weight, .. } if bytes == &[8] && *weight == 2.0 + )); + let _ = std::fs::remove_file(dataset_path); +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn byte_alphabet_empty_payload_is_rejected() { + let dataset_path = temp_path("dataset-byte-alphabet-empty", ".json"); + std::fs::write( + &dataset_path, + causal_dataset_value( + "byte-empty-codec", + "events", + serde_json::json!([ + {"kind": "target", "channel": "percept", "domain": "bytes", "bytes": []} + ]), + ) + .to_string(), + ) + .expect("write byte-alphabet empty payload dataset"); + let err = load_dataset(&dataset_path).expect_err("empty byte-alphabet payload must fail"); + assert!(err.contains("must contain at least one byte"), "{err}"); + let _ = std::fs::remove_file(dataset_path); +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn causal_header_cross_checks_enforce_grammar_and_action_contracts() { + let undeclared_descriptor_path = temp_path("dataset-undeclared-event-descriptor", ".json"); + std::fs::write( + &undeclared_descriptor_path, + causal_dataset_value( + "descriptor-codec", + "events", + serde_json::json!([ + {"kind": "target", "channel": "other", "domain": "bytes", "bytes": [1]} + ]), + ) + .to_string(), + ) + .expect("write undeclared descriptor dataset"); + let err = + load_dataset(&undeclared_descriptor_path).expect_err("undeclared descriptor must fail"); + assert!( + err.contains("not declared in event_grammar.target"), + "{err}" + ); + + let invalid_action_path = temp_path("dataset-invalid-action-context", ".json"); + std::fs::write( + &invalid_action_path, + causal_dataset_value( + "invalid-action-codec", + "events", + serde_json::json!([ + {"kind": "context", "channel": "action", "bytes": [2]}, + {"kind": "target", "channel": "percept", "domain": "bytes", "bytes": [1]} + ]), + ) + .to_string(), + ) + .expect("write invalid action dataset"); + let err = load_dataset(&invalid_action_path).expect_err("invalid action context must fail"); + assert!(err.contains("outside action_alphabet.size"), "{err}"); + + let mut invalid_grammar = causal_dataset_value( + "invalid-grammar-codec", + "events", + serde_json::json!([ + {"kind": "target", "channel": "percept", "domain": "bytes", "bytes": [1]} + ]), + ); + let grammar = invalid_grammar + .get_mut("event_grammar") + .and_then(Value::as_object_mut) + .expect("event_grammar object"); + let targets = grammar + .get_mut("target") + .and_then(Value::as_array_mut) + .expect("target grammar array"); + targets.push(serde_json::json!({"channel": "ghost", "domain": "ghost"})); + let invalid_grammar_path = temp_path("dataset-invalid-grammar-domain", ".json"); + std::fs::write( + &invalid_grammar_path, + serde_json::to_string(&invalid_grammar).expect("invalid grammar json"), + ) + .expect("write invalid grammar dataset"); + let err = + load_dataset(&invalid_grammar_path).expect_err("grammar with undeclared domain must fail"); + assert!( + err.contains("event_grammar references undeclared target domain"), + "{err}" + ); + + let _ = std::fs::remove_file(undeclared_descriptor_path); + let _ = std::fs::remove_file(invalid_action_path); + let _ = std::fs::remove_file(invalid_grammar_path); +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn byte_alphabet_expansion_matches_chain_rule_loss() { + let dataset_expanded_from_multibyte = temp_path("dataset-byte-chain-multibyte", ".json"); + std::fs::write( + &dataset_expanded_from_multibyte, + causal_dataset_value( + "chain-rule-codec", + "events", + serde_json::json!([ + {"kind": "target", "channel": "percept", "domain": "bytes", "bytes": [65, 66]} + ]), + ) + .to_string(), + ) + .expect("write multi-byte dataset"); + let dataset_explicit_singletons = temp_path("dataset-byte-chain-singletons", ".json"); + std::fs::write( + &dataset_explicit_singletons, + causal_dataset_value( + "chain-rule-codec", + "events", + serde_json::json!([ + {"kind": "target", "channel": "percept", "domain": "bytes", "bytes": [65]}, + {"kind": "target", "channel": "percept", "domain": "bytes", "bytes": [66]} + ]), + ) + .to_string(), + ) + .expect("write singleton dataset"); + + let dataset_a = load_dataset(&dataset_expanded_from_multibyte).expect("load dataset a"); + let dataset_b = load_dataset(&dataset_explicit_singletons).expect("load dataset b"); + let candidate = CompressionBackend::Rate { + rate_backend: RateBackend::Ctw { depth: 8 }, + coder: crate::coders::CoderType::AC, + framing: FramingMode::Framed, + } + .compile() + .expect("compile ctw candidate"); + let deadline = Instant::now() + Duration::from_secs(2); + let (compressed_a, loss_a) = + evaluate_candidate_causal_loss(&candidate, &dataset_a, deadline).expect("eval a"); + let (compressed_b, loss_b) = + evaluate_candidate_causal_loss(&candidate, &dataset_b, deadline).expect("eval b"); + assert_eq!(compressed_a, compressed_b); + assert!( + (loss_a - loss_b).abs() < 1.0e-10, + "loss_a={loss_a}, loss_b={loss_b}" + ); + + let _ = std::fs::remove_file(dataset_expanded_from_multibyte); + let _ = std::fs::remove_file(dataset_explicit_singletons); +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn causal_dataset_domains_are_profile_fixed_and_support_checked() { + let dataset_path = temp_path("dataset-enumerated-domain", ".json"); + std::fs::write( + &dataset_path, + causal_dataset_value( + "test-enumerated-codec", + "events", + serde_json::json!([ + {"kind": "context", "channel": "action", "bytes": [1]}, + {"kind": "target", "channel": "percept", "domain": "binary", "bytes": [1]} + ]), + ) + .to_string(), + ) + .expect("write enumerated-domain dataset"); + let dataset = load_dataset(&dataset_path).expect("enumerated domain dataset lowers"); + let causal_profile = dataset.causal_profile.as_ref().expect("causal profile"); + assert!(causal_profile.domains.contains_key("binary")); + assert_eq!( + dataset.target_domain_support_hash, + causal_profile.domain_support_hash + ); + assert_ne!( + dataset.target_domain_support_hash, + crc32_hex(b"passive-byte-alphabet") + ); + + let out_of_support_path = temp_path("dataset-enumerated-domain-out", ".json"); + std::fs::write( + &out_of_support_path, + causal_dataset_value( + "test-enumerated-codec", + "events", + serde_json::json!([ + {"kind": "target", "channel": "percept", "domain": "binary", "bytes": [2]} + ]), + ) + .to_string(), + ) + .expect("write out-of-support dataset"); + let err = load_dataset(&out_of_support_path).expect_err("out-of-support target fails"); + assert!(err.contains("outside target-domain support"), "{err}"); + + let _ = std::fs::remove_file(dataset_path); + let _ = std::fs::remove_file(out_of_support_path); +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn causal_event_channel_and_domain_affect_skeleton_identity() { + let path_a = temp_path("dataset-channel-a", ".json"); + let path_b = temp_path("dataset-channel-b", ".json"); + std::fs::write( + &path_a, + causal_dataset_value( + "same-codec", + "events", + serde_json::json!([ + {"kind": "target", "channel": "percept", "domain": "bytes", "bytes": [7]} + ]), + ) + .to_string(), + ) + .expect("write channel a"); + std::fs::write( + &path_b, + causal_dataset_value( + "same-codec", + "events", + serde_json::json!([ + {"kind": "target", "channel": "reward", "domain": "binary", "bytes": [1]} + ]), + ) + .to_string(), + ) + .expect("write channel b"); + + let dataset_a = load_dataset(&path_a).expect("load channel a"); + let dataset_b = load_dataset(&path_b).expect("load channel b"); + assert_ne!(dataset_a.event_grammar_hash, dataset_b.event_grammar_hash); + assert_eq!(dataset_a.codec_hash, dataset_b.codec_hash); + + let _ = std::fs::remove_file(path_a); + let _ = std::fs::remove_file(path_b); +} + +#[test] +fn annealer_schedule_matches_normative_log_linear_law() { + let mid = annealer_temperature(0.5); + let expected_mid = ANNEALER_T_MIN_BITS * (ANNEALER_T0_BITS / ANNEALER_T_MIN_BITS).powf(0.5); + assert_eq!(annealer_progress_from_elapsed(0.0, 10.0), 0.0); + assert_eq!(annealer_progress_from_elapsed(5.0, 10.0), 0.5); + assert_eq!(annealer_progress_from_elapsed(20.0, 10.0), 1.0); + assert!((annealer_temperature(0.0) - ANNEALER_T0_BITS).abs() < 1.0e-12); + assert!((annealer_temperature(1.0) - ANNEALER_T_MIN_BITS).abs() < 1.0e-12); + assert!((mid - expected_mid).abs() < 1.0e-12); +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn exact_state_observation_projection_supports_stream_hash() { + let interface = PlannerInterfaceSpec { + observation_bits: 3, + observation_stream_len: 2, + observation_key_mode: ObservationKeyMode::StreamHash, + reward_bits: 16, + agent_actions: action_alphabet(2), + }; + let projected = project_observation_output("stream_hash", &[9, 2], interface.observation_bits) + .expect("stream_hash projection"); + assert_eq!(projected, vec![130]); +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn discounted_aiqi_exact_theorem_claims_remain_uncertified_by_family() { + let controller = + crate::spec::CompiledTuneController::AiqiDiscounted(AiqiDiscountedTuneControllerSpec { + interface: TunePlannerInterfaceSpec { + observation_bits: 8, + observation_stream_len: 1, + observation_key_mode: ObservationKeyMode::FullStream, + reward_bits: 16, + agent_actions: action_alphabet(2), + }, + planner_simulations_per_step: 1, + return_horizon: 1, + return_bins: 2, + discount_factor: 0.0, + min_improvement: 0.0, + max_improvement: 1.0, + }); + let candidate = CompressionBackend::Rate { + rate_backend: RateBackend::Ctw { depth: 8 }, + coder: crate::coders::CoderType::AC, + framing: FramingMode::Framed, + }; + let search = SearchSummary { + status: "test", + warning: None, + fatal_evaluator_failure: None, + fatal_evaluator_failures: 0, + best_candidate: candidate, + best_candidate_crc32: "00000000".to_string(), + best_eval: CandidateEvalResult { + status: CandidateEvalStatus::Success, + compressed_bytes: 1, + elapsed_seconds: 0.1, + effective_eval_time_limit_seconds: 1.0, + throughput_bytes_per_second: 10.0, + peak_memory_bytes: 1, + target_loss_bits: 8.0, + objective_bits: 16.0, + deployable: true, + }, + cache_key_digest: "00000000".to_string(), + cache_hits: 0, + cache_misses: 0, + candidate_evaluations_executed: 1, + non_warmup_candidate_results_seen: 1, + post_baseline_candidate_results_seen: 0, + proposals_attempted: 0, + proposals_invalid: 0, + self_loop_proposals: 0, + invalid_reason_counts: InvalidReasonCounts::default(), + successful_non_deployable: 0, + candidate_result_counts: CandidateResultCounts { + success_deployable: 1, + success_non_deployable: 0, + timeout: 0, + invalid: 0, + error_recoverable: 0, + }, + final_best_move_reward: 0.0, + realized_trace_counts_by_round: None, + trace_refresh_merges_by_round: None, + controller_report: Value::Null, + }; + let theorem = TuneTheoremConfig { + claim_exact_finite_mdp: true, + scalar_representation_ref: Some(SCALAR_REPRESENTATION_DECLARATION.to_string()), + ..TuneTheoremConfig::default() + }; + let verified = VerifiedTheoremInputs { + finite_planner_state: Some(VerifiedCertificate { + ref_value: "finite.json".to_string(), + content_hash: "00000000".to_string(), + }), + no_hidden_state: Some(VerifiedCertificate { + ref_value: "hidden.json".to_string(), + content_hash: "00000000".to_string(), + }), + exact_reward_encoding: Some(VerifiedExactRewardEncodingCertificate { + base: VerifiedCertificate { + ref_value: "reward.json".to_string(), + content_hash: "00000000".to_string(), + }, + max_reward: 65_535, + reward_bits: 16, + scalar_representation: SCALAR_REPRESENTATION_DECLARATION.to_string(), + mode: VerifiedRewardEncodingMode::IntegerObjectiveDifferenceInterval, + }), + exact_state_observation: None, + determinism_deadline: None, + deterministic_table: None, + }; + let missing = + exact_finite_mdp_missing_prereqs(&theorem, &verified, &controller, &search, false); + assert!(missing.contains(&"exact_objective_difference_controller")); +} + +#[test] +fn warmstart_trace_merge_is_content_deduplicated_and_structurally_ordered() { + let mut teacher = WarmStartExactJhTeacherDataset::new( + WarmStartExactJhTeacherContract { + schema_version: 1, + task_fingerprint: TaskFingerprint::parse_hex( + "0102030401020304010203040102030401020304010203040102030401020304", + ) + .expect("test fingerprint"), + action_alphabet_size: 2, + observation_bits: 1, + observation_stream_len: 1, + observation_key_mode: "first".to_string(), + observation_adapter_spec_ref: String::new(), + observation_adapter_content_crc32: String::new(), + reward_bits: 1, + return_horizon: 1, + label_phase_period: 1, + scalar_representation: String::new(), + exact_reward_encoding_certificate: String::new(), + }, + Vec::new(), + ); + let high_structural_trace = WarmStartExactJhTeacherTrace { + transitions: vec![WarmStartExactJhTransition { + action: 1_u64, + observations: vec![2], + reward: 3, + }], + }; + let low_structural_trace = WarmStartExactJhTeacherTrace { + transitions: vec![WarmStartExactJhTransition { + action: 0_u64, + observations: vec![1], + reward: 1, + }], + }; + teacher.traces.push(high_structural_trace.clone()); + + assert!( + merge_warmstart_trace_deterministic(&mut teacher, low_structural_trace.clone()) + .expect("merge distinct trace") + ); + assert_eq!( + teacher.traces, + vec![low_structural_trace.clone(), high_structural_trace] + ); + assert_eq!(teacher.traces.len(), 2); + + assert!( + !merge_warmstart_trace_deterministic(&mut teacher, low_structural_trace) + .expect("duplicate merge remains idempotent") + ); + assert_eq!(teacher.traces.len(), 2); +} + +#[test] +fn warmstart_trace_refresh_merge_counter_counts_structural_inserts_only() { + let mut teacher = WarmStartExactJhTeacherDataset::new( + WarmStartExactJhTeacherContract { + schema_version: 1, + task_fingerprint: TaskFingerprint::parse_hex( + "0102030401020304010203040102030401020304010203040102030401020304", + ) + .expect("test fingerprint"), + action_alphabet_size: 2, + observation_bits: 1, + observation_stream_len: 1, + observation_key_mode: "first".to_string(), + observation_adapter_spec_ref: String::new(), + observation_adapter_content_crc32: String::new(), + reward_bits: 1, + return_horizon: 1, + label_phase_period: 1, + scalar_representation: String::new(), + exact_reward_encoding_certificate: String::new(), + }, + Vec::new(), + ); + let live_trace = WarmStartExactJhTeacherTrace { + transitions: vec![WarmStartExactJhTransition { + action: 0_u64, + observations: vec![1], + reward: 0, + }], + }; + let mut warmstart_trace_refresh_merges: usize = 0; + for _ in 0..2 { + if merge_warmstart_trace_deterministic(&mut teacher, live_trace.clone()) + .expect("merge live trace") + { + warmstart_trace_refresh_merges = warmstart_trace_refresh_merges.saturating_add(1); + } + } + assert_eq!(warmstart_trace_refresh_merges, 1); + assert_eq!(teacher.traces.len(), 1); +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn run_tune_annealed_reports_search_activity() { + let dataset_path = temp_path("dataset-annealed", ".bin"); + let spec_path = temp_path("spec-annealed", ".json"); + let output_path = temp_path("output-annealed", ".json"); + let report_path = temp_path("report-annealed", ".json"); + std::fs::write(&dataset_path, b"annealed-search-dataset").expect("write dataset"); + + let mut spec = sample_tune_spec( + dataset_path.to_str().expect("dataset path"), + output_path.to_str().expect("output path"), + report_path.to_str().expect("report path"), + ); + spec.bounds.parameter_ranges = vec![crate::spec::TuneParameterRangeSpec { + parameter: "rate_backend.depth".to_string(), + min: 1.0, + max: 16.0, + }]; + let spec_json = SpecDocument::Tune(spec) + .to_canonical_json() + .expect("spec json"); + std::fs::write(&spec_path, spec_json).expect("write spec"); + + let request = TuneCommandRequest { + spec_path: spec_path.to_string_lossy().to_string(), + emit_exact_reward_encoding_certificate: None, + execution: TuneExecutionConfig { + max_evaluations: Some(3), + ..TuneExecutionConfig::default() + }, + }; + run_tune(&request).expect("run tune"); + + let report = std::fs::read_to_string(&report_path).expect("report exists"); + assert!(report.contains("\"status\": \"completed_annealed\"")); + assert!(report.contains("\"proposals_attempted\":")); + let report_json: Value = serde_json::from_str(&report).expect("report json"); + assert_eq!( + report_json + .pointer("/search/baseline_counts_toward_max_evaluations") + .and_then(Value::as_bool), + Some(true) + ); + let non_warmup_results = report_json + .pointer("/search/non_warmup_candidate_results_seen") + .and_then(Value::as_u64) + .expect("non_warmup_candidate_results_seen"); + let post_baseline_results = report_json + .pointer("/search/post_baseline_candidate_results_seen") + .and_then(Value::as_u64) + .expect("post_baseline_candidate_results_seen"); + assert!((1..=3).contains(&non_warmup_results)); + assert_eq!(post_baseline_results + 1, non_warmup_results); + let cache_calls = report_json + .pointer("/cache/actual_evaluator_calls_excluding_warmups") + .and_then(Value::as_u64) + .expect("actual_evaluator_calls_excluding_warmups"); + assert_eq!( + report_json + .pointer("/cache/candidate_evaluations_executed") + .and_then(Value::as_u64), + Some(cache_calls) + ); + + let _ = std::fs::remove_file(dataset_path); + let _ = std::fs::remove_file(spec_path); + let _ = std::fs::remove_file(output_path); + let _ = std::fs::remove_file(report_path); +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn planner_family_controller_executes_runtime_path() { + let dataset_path = temp_path("dataset-planner", ".bin"); + let spec_path = temp_path("spec-planner", ".json"); + let output_path = temp_path("output-planner", ".json"); + let report_path = temp_path("report-planner", ".json"); + std::fs::write(&dataset_path, b"planner-controller-dataset").expect("write dataset"); + + let mut spec = sample_tune_spec( + dataset_path.to_str().expect("dataset path"), + output_path.to_str().expect("output path"), + report_path.to_str().expect("report path"), + ); + spec.controller = TuneControllerSpec::McAixiFacCtw(McAixiFacCtwTuneControllerSpec { + interface: planner_interface_for_baseline(&spec.baseline_candidate), + planner_simulations_per_step: 8, + }); + let bounds = spec.bounds.clone(); + let spec_json = SpecDocument::Tune(spec) + .to_canonical_json() + .expect("spec json"); + std::fs::write(&spec_path, spec_json).expect("write spec"); + let reward_cert_path = temp_path("reward-cert-planner", ".json"); + write_test_exact_reward_certificate( + &reward_cert_path, + &dataset_path, + &bounds, + "mc_aixi_fac_ctw", + ); + + let request = TuneCommandRequest { + spec_path: spec_path.to_string_lossy().to_string(), + emit_exact_reward_encoding_certificate: None, + execution: TuneExecutionConfig { + theorem: TuneTheoremConfig { + exact_reward_encoding_certificate: Some( + reward_cert_path.to_string_lossy().to_string(), + ), + ..TuneTheoremConfig::default() + }, + ..TuneExecutionConfig::default() + }, + }; + run_tune(&request).expect("run tune"); + + let report = std::fs::read_to_string(&report_path).expect("report exists"); + assert!(report.contains("\"status\": \"completed_mc_aixi_fac_ctw\"")); + assert!(report.contains("\"runtime_path\": \"finite_mutation_agent_bridge_mcaixi_fac_ctw\"")); + + let _ = std::fs::remove_file(dataset_path); + let _ = std::fs::remove_file(spec_path); + let _ = std::fs::remove_file(output_path); + let _ = std::fs::remove_file(report_path); + let _ = std::fs::remove_file(reward_cert_path); +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn executor_controls_are_excluded_from_canonical_tune_but_included_in_evaluator_profile() { + let passive_dataset_path = temp_path("dataset-passive", ".bin"); + let trace_dataset_path = temp_path("dataset-trace", ".json"); + let prefix_dataset_path = temp_path("dataset-prefix", ".json"); + let output_path = temp_path("output-identity", ".json"); + let report_path = temp_path("report-identity", ".json"); + std::fs::write(&passive_dataset_path, b"identity-passive").expect("write passive"); + std::fs::write( + &trace_dataset_path, + causal_dataset_value( + "identity-trace-codec", + "events", + serde_json::json!([ + {"kind": "context", "channel": "action", "bytes": [1]}, + {"kind": "target", "channel": "percept", "domain": "bytes", "bytes": [2, 3]} + ]), + ) + .to_string(), + ) + .expect("write trace"); + std::fs::write( + &prefix_dataset_path, + causal_dataset_value( + "identity-prefix-codec", + "examples", + serde_json::json!([ + { + "history": [{"kind": "observe_target_no_score", "channel": "percept", "domain": "bytes", "bytes": [7]}], + "action": [1], + "channel": "percept", + "domain": "bytes", + "target": [8], + "weight": 2.0 + } + ]), + ) + .to_string(), + ) + .expect("write prefix"); + + let dataset_paths = [ + passive_dataset_path.as_path(), + trace_dataset_path.as_path(), + prefix_dataset_path.as_path(), + ]; + for dataset_path in dataset_paths { + let mut base = sample_tune_spec( + dataset_path.to_str().expect("dataset path"), + output_path.to_str().expect("output path"), + report_path.to_str().expect("report path"), + ); + let interface = planner_interface_for_baseline(&base.baseline_candidate); + let controllers = vec![ + TuneControllerSpec::AnnealedHillClimbing(AnnealedHillClimbingTuneControllerSpec { + max_mutation_radius: 1, + }), + TuneControllerSpec::McAixiFacCtw(McAixiFacCtwTuneControllerSpec { + interface: interface.clone(), + planner_simulations_per_step: 2, + }), + TuneControllerSpec::AiqiDiscounted(AiqiDiscountedTuneControllerSpec { + interface: interface.clone(), + planner_simulations_per_step: 2, + return_horizon: 1, + return_bins: 2, + discount_factor: 0.5, + min_improvement: 0.0, + max_improvement: 1.0, + }), + TuneControllerSpec::AiqiWarmstartExactJh(WarmStartExactJhTuneControllerSpec { + interface: interface.clone(), + planner_simulations_per_step: 1, + return_horizon: 1, + warmstart_teacher_dataset_asset: "teacher".to_string(), + label_phase_period: 1, + }), + ]; + for controller in controllers { + base.controller = controller; + base.assets.retain(|asset| asset.id == "dataset"); + if matches!(base.controller, TuneControllerSpec::AiqiWarmstartExactJh(_)) { + base.assets.push(AssetBinding { + id: "teacher".to_string(), + path: passive_dataset_path.to_string_lossy().to_string(), + }); + } + let mut request_a = TuneCommandRequest { + spec_path: "spec-a.json".to_string(), + emit_exact_reward_encoding_certificate: None, + execution: TuneExecutionConfig::default(), + }; + let mut request_b = request_a.clone(); + request_b.spec_path = "spec-b.json".to_string(); + request_b.execution.max_evaluations = Some(1); + request_b.execution.annealer_kernel_profile = + AnnealerKernelProfile::CompiledUniformMetropolisHastings; + request_b.execution.warmup_baseline_runs = 3; + request_b.execution.diagnostic_chunk_bytes = Some(4096); + request_b.execution.evaluator_cgroup_parent = + Some("/sys/fs/cgroup/infotheory-tuner".to_string()); + request_b.execution.theorem.timing_certification_tier = + TimingCertificationTier::RealTime; + request_b.execution.theorem.determinism_deadline_certificate = + Some("cert://deadline".to_string()); + + let canonical_value_a = SpecDocument::Tune(base.clone()) + .to_canonical_json_value() + .expect("canonical tune a"); + for field in [ + "max_evaluations", + "annealer_kernel_profile", + "cpu_affinity", + "threads", + "evaluator_worker_executable", + "evaluator_cgroup_parent", + "warmup_baseline_runs", + "self_improvement_rounds", + "stagnation_reset_evals", + "log_path", + "diagnostic_chunk_bytes", + "rss_mode", + "planner_deployable_model", + "warmstart_trace_refresh", + "theorem", + ] { + assert!( + canonical_value_a.get(field).is_none(), + "canonical tune document must not contain executor field '{field}'" + ); + } + let spec_path_a = temp_path("identity-spec-a", ".json"); + let spec_path_b = temp_path("identity-spec-b", ".json"); + let canonical_text = serde_json::to_vec(&canonical_value_a).expect("canonical json"); + std::fs::write(&spec_path_a, &canonical_text).expect("write spec a"); + std::fs::write(&spec_path_b, &canonical_text).expect("write spec b"); + request_a.spec_path = spec_path_a.to_string_lossy().to_string(); + request_b.spec_path = spec_path_b.to_string_lossy().to_string(); + let canonical_a = crate::spec::load_spec_document(&request_a.spec_path) + .expect("load spec a") + .validate() + .expect("validate spec a") + .canonical_bytes() + .as_slice() + .to_vec(); + let canonical_b = crate::spec::load_spec_document(&request_b.spec_path) + .expect("load spec b") + .validate() + .expect("validate spec b") + .canonical_bytes() + .as_slice() + .to_vec(); + assert_ne!(request_a.spec_path, request_b.spec_path); + assert_ne!(request_a.execution, request_b.execution); + assert_eq!(canonical_a, canonical_b); + let _ = std::fs::remove_file(spec_path_a); + let _ = std::fs::remove_file(spec_path_b); + let loaded = load_dataset(dataset_path).expect("dataset mode loads"); + let profile_a = EvaluatorProfile { + dataset_kind: loaded.kind, + objective_target: loaded.objective_target, + dataset_lowering_version: loaded.lowering_version, + dataset_codec_hash: loaded.codec_hash.clone(), + event_grammar_hash: loaded.event_grammar_hash.clone(), + target_domain_support_hash: loaded.target_domain_support_hash.clone(), + causal_header_profile_hash: loaded.causal_header_profile_hash.clone(), + target_size_function: loaded.target_size_function, + evaluator_interface_version: TUNER_EVALUATOR_INTERFACE_VERSION, + candidate_canonicalization_version: "bounds-v1".to_string(), + warmup_baseline_runs: 0, + diagnostic_chunk_bytes: None, + eval_time_limit_seconds: base.eval_time_limit_seconds, + evaluator_threads: 1, + worker_isolation_mode: "spawn_exec_worker", + worker_executable_identity: None, + resolved_memory_accounting_kind: "unix_process_rss_fallback_explicit", + resolved_memory_accounting_strict_theorem_facing: false, + resolved_evaluator_cgroup_parent: None, + backend_report_component_policy: "none", + evaluator_determinism: "deterministic_under_h", + rss_mode: PeakMemoryMode::ProcessRssPeak, + timing_certification_tier: TimingCertificationTier::BestEffort, + build_profile: "test", + feature_set: vec!["test"], + }; + let mut profile_b = profile_a.clone(); + profile_b.warmup_baseline_runs = request_b.execution.warmup_baseline_runs; + profile_b.diagnostic_chunk_bytes = request_b.execution.diagnostic_chunk_bytes; + profile_b.timing_certification_tier = + request_b.execution.theorem.timing_certification_tier; + assert_ne!( + profile_a.hash().expect("profile a"), + profile_b.hash().expect("profile b") + ); + } + } + + let _ = std::fs::remove_file(passive_dataset_path); + let _ = std::fs::remove_file(trace_dataset_path); + let _ = std::fs::remove_file(prefix_dataset_path); +} +// --- Group 1: Canonicalization and model-code properties --- + +#[cfg(feature = "backend-ctw")] +fn sample_enabled_leaf_rate_backend_for_canonical_tests() -> Option { + crate::runtime::RATE_BACKEND_REGISTRY + .iter() + .filter(|descriptor| descriptor.enabled) + .find_map(|descriptor| crate::runtime::default_rate_backend_spec(descriptor.kind)) +} + +#[cfg(feature = "backend-ctw")] +fn sample_roundtrip_compression_backends_for_canonical_tests() -> Vec { + let mut out = Vec::::new(); + let leaf = sample_enabled_leaf_rate_backend_for_canonical_tests(); + for descriptor in crate::runtime::COMPRESSION_BACKEND_REGISTRY { + if !descriptor.enabled { + continue; + } + match descriptor.kind { + crate::runtime::CompressionBackendKind::Zpaq => out.push(CompressionBackend::zpaq("5")), + crate::runtime::CompressionBackendKind::RateAc => { + if let Some(rate_backend) = leaf.clone() { + out.push(CompressionBackend::Rate { + rate_backend, + coder: crate::coders::CoderType::AC, + framing: FramingMode::Framed, + }); + } + } + crate::runtime::CompressionBackendKind::RateRans => { + if let Some(rate_backend) = leaf.clone() { + out.push(CompressionBackend::Rate { + rate_backend, + coder: crate::coders::CoderType::RANS, + framing: FramingMode::Raw, + }); + } + } + #[cfg(feature = "backend-rwkv")] + crate::runtime::CompressionBackendKind::Rwkv7 => { + let opts = crate::spec::CompressionBackendShorthandOptions { + default_framing: crate::compression::FramingMode::Raw, + ..Default::default() + }; + let rwkv = crate::spec::parse_compression_backend_name_method( + "rwkv7", + Some( + "cfg:hidden=64,intermediate=64,layers=1,train=sgd,lr=0.01;policy:schedule=0..100:infer", + ), + None, + &opts, + ) + .expect("rwkv shorthand should parse"); + out.push(rwkv); + } + #[cfg(not(feature = "backend-rwkv"))] + crate::runtime::CompressionBackendKind::Rwkv7 => {} + } + } + out +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn canonical_code_roundtrip_and_idempotence_hold_on_enabled_corpus() { + use std::path::Path; + + let corpus = sample_roundtrip_compression_backends_for_canonical_tests(); + assert!( + !corpus.is_empty(), + "at least one enabled compression backend must be available for canonicalization tests" + ); + + for candidate in corpus { + let compiled = candidate.compile().expect("compile candidate"); + let canonical_bytes = compiled.canonical_bytes().as_slice().to_vec(); + let canonical_doc = SpecDocument::CompressionBackend(compiled.canonical_spec().clone()); + let canonical_doc_bytes = canonical_doc.to_binary(); + + let reparsed = SpecDocument::from_binary(&canonical_doc_bytes, Path::new(".")) + .expect("parse canonical bytes"); + let recompiled = reparsed.compile().expect("compile reparsed document"); + let crate::spec::CompiledSpecDocument::CompressionBackend(recompiled_candidate) = + recompiled + else { + panic!("expected compression backend document"); + }; + + assert_eq!( + canonical_bytes, + recompiled_candidate.canonical_bytes().as_slice() + ); + assert_eq!( + canonical_doc.to_canonical_json().expect("canonical json"), + SpecDocument::CompressionBackend(recompiled_candidate.canonical_spec().clone()) + .to_canonical_json() + .expect("reparsed canonical json") + ); + } +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn canonical_code_image_is_injective_and_prefix_free_on_enabled_corpus() { + let corpus = sample_roundtrip_compression_backends_for_canonical_tests(); + let mut image = std::collections::BTreeMap::, String>::new(); + + for candidate in corpus { + let compiled = candidate.compile().expect("compile candidate"); + let canonical_doc = SpecDocument::CompressionBackend(compiled.canonical_spec().clone()); + let bytes = compiled.canonical_bytes().as_slice().to_vec(); + let json = canonical_doc.to_canonical_json().expect("canonical json"); + + if let Some(existing) = image.insert(bytes.clone(), json.clone()) { + assert_eq!( + existing, json, + "equal canonical bytes must denote identical canonical candidate JSON" + ); + } + } + + let keys = image.keys().cloned().collect::>>(); + for i in 0..keys.len() { + for j in 0..keys.len() { + if i == j { + continue; + } + assert!( + !keys[i].starts_with(&keys[j]), + "canonical code image must be prefix-free" + ); + } + } +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn syntactic_aliases_canonicalize_to_same_model_code_length() { + use crate::compression::FramingMode; + let canonical_syntax = serde_json::json!({ + "kind": "rate-ac", + "rate_backend": {"kind": "ctw", "depth": 16}, + "framing": "framed" + }); + let alias_syntax = serde_json::json!({ + "kind": "rate-ac", + "backend_spec": {"kind": "ctw"} + }); + assert_ne!(canonical_syntax, alias_syntax); + + let z1_ast = crate::spec::parse_compression_backend_json( + &canonical_syntax, + std::path::Path::new("."), + None, + FramingMode::Framed, + ) + .unwrap(); + let z2_ast = crate::spec::parse_compression_backend_json( + &alias_syntax, + std::path::Path::new("."), + None, + FramingMode::Framed, + ) + .unwrap(); + let z1 = z1_ast.compile().unwrap(); + let z2 = z2_ast.compile().unwrap(); + assert_eq!( + z1.canonical_bytes().as_slice(), + z2.canonical_bytes().as_slice() + ); + assert_eq!(z1.canonical_bytes().len(), z2.canonical_bytes().len()); + assert_eq!( + 8_usize * z1.canonical_bytes().len(), + 8_usize * z2.canonical_bytes().len() + ); +} + +#[cfg(all(feature = "backend-ctw", feature = "backend-mixture"))] +#[test] +fn canonicalization_preserves_order_sensitivity_for_mixture_experts() { + use std::sync::Arc; + + let left = CompressionBackend::Rate { + rate_backend: RateBackend::Mixture { + spec: Arc::new(crate::api::MixtureSpec::new( + crate::api::MixtureKind::Bayes, + vec![ + crate::api::MixtureExpertSpec::new(RateBackend::Ctw { depth: 4 }) + .with_name("left"), + crate::api::MixtureExpertSpec::new(RateBackend::Ctw { depth: 6 }) + .with_name("right"), + ], + )), + }, + coder: crate::coders::CoderType::AC, + framing: FramingMode::Framed, + }; + let right = CompressionBackend::Rate { + rate_backend: RateBackend::Mixture { + spec: Arc::new(crate::api::MixtureSpec::new( + crate::api::MixtureKind::Bayes, + vec![ + crate::api::MixtureExpertSpec::new(RateBackend::Ctw { depth: 6 }) + .with_name("right"), + crate::api::MixtureExpertSpec::new(RateBackend::Ctw { depth: 4 }) + .with_name("left"), + ], + )), + }, + coder: crate::coders::CoderType::AC, + framing: FramingMode::Framed, + }; + + let left_bytes = left + .compile() + .expect("compile left mixture") + .canonical_bytes() + .as_slice() + .to_vec(); + let right_bytes = right + .compile() + .expect("compile right mixture") + .canonical_bytes() + .as_slice() + .to_vec(); + assert_ne!( + left_bytes, right_bytes, + "expert order is semantic in order-sensitive mixture canonicalization paths" + ); +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn canonicalization_is_insensitive_to_json_object_key_order() { + use std::path::Path; + + let mut top_a = serde_json::Map::new(); + top_a.insert("kind".to_string(), serde_json::json!("rate-ac")); + let mut rate_a = serde_json::Map::new(); + rate_a.insert("kind".to_string(), serde_json::json!("ctw")); + rate_a.insert("depth".to_string(), serde_json::json!(8)); + top_a.insert("rate_backend".to_string(), Value::Object(rate_a)); + top_a.insert("framing".to_string(), serde_json::json!("framed")); + + let mut top_b = serde_json::Map::new(); + top_b.insert("framing".to_string(), serde_json::json!("framed")); + let mut rate_b = serde_json::Map::new(); + rate_b.insert("depth".to_string(), serde_json::json!(8)); + rate_b.insert("kind".to_string(), serde_json::json!("ctw")); + top_b.insert("rate_backend".to_string(), Value::Object(rate_b)); + top_b.insert("kind".to_string(), serde_json::json!("rate-ac")); + + let parsed_a = crate::spec::parse_compression_backend_json( + &Value::Object(top_a), + Path::new("."), + None, + FramingMode::Framed, + ) + .expect("parse A"); + let parsed_b = crate::spec::parse_compression_backend_json( + &Value::Object(top_b), + Path::new("."), + None, + FramingMode::Framed, + ) + .expect("parse B"); + + let bytes_a = parsed_a + .compile() + .expect("compile A") + .canonical_bytes() + .as_slice() + .to_vec(); + let bytes_b = parsed_b + .compile() + .expect("compile B") + .canonical_bytes() + .as_slice() + .to_vec(); + assert_eq!(bytes_a, bytes_b); +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn binary_canonical_deserializer_rejects_trailing_bytes() { + use std::path::Path; + + let candidate = sample_roundtrip_compression_backends_for_canonical_tests() + .into_iter() + .next() + .expect("candidate corpus must be non-empty"); + let compiled = candidate.compile().expect("compile candidate"); + let mut payload = + SpecDocument::CompressionBackend(compiled.canonical_spec().clone()).to_binary(); + payload.extend_from_slice(&[0x00_u8, 0x01_u8]); + + let err = match SpecDocument::from_binary(&payload, Path::new(".")) { + Ok(_) => panic!("trailing bytes must fail"), + Err(err) => err, + }; + assert!( + err.to_string().contains("unexpected trailing bytes"), + "{err}" + ); +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn cache_key_includes_effective_timeout() { + use crate::api::{CompressionBackend, RateBackend}; + use crate::compression::FramingMode; + use crate::tuner::tests::temp_path; + use crate::tuner::{EvaluatorProfile, cache_key_for_candidate}; + use std::time::Instant; + let z = CompressionBackend::Rate { + rate_backend: RateBackend::Ctw { depth: 4 }, + coder: crate::coders::CoderType::AC, + framing: FramingMode::Framed, + } + .compile() + .unwrap(); + + let dataset_path = temp_path("dataset-cache-key", ".bin"); + std::fs::write(&dataset_path, b"test-data").unwrap(); + let dataset = crate::tuner::tests::load_dataset(&dataset_path).unwrap(); + let output_path = temp_path("cache-key-output", ".json"); + let report_path = temp_path("cache-key-report", ".json"); + let mut spec = sample_tune_spec( + dataset_path.to_str().unwrap(), + output_path.to_str().unwrap(), + report_path.to_str().unwrap(), + ); + spec.eval_time_limit_seconds = 10.0; + spec.time_budget_seconds = 60.0; + let full_budget_compiled = spec.compile().unwrap(); + spec.time_budget_seconds = 9.999; + let truncated_budget_compiled = spec.compile().unwrap(); + let tune_started = Instant::now(); + let full_effective_limit = + effective_eval_limit_seconds(&full_budget_compiled, tune_started, Some(10.0), None); + let truncated_effective_limit = + effective_eval_limit_seconds(&truncated_budget_compiled, tune_started, Some(10.0), None); + assert_eq!(full_effective_limit, 10.0); + assert!(truncated_effective_limit > 0.0); + assert!(truncated_effective_limit < full_effective_limit); + + let profile1 = EvaluatorProfile { + dataset_kind: dataset.kind, + objective_target: dataset.objective_target, + dataset_lowering_version: dataset.lowering_version, + dataset_codec_hash: dataset.codec_hash.clone(), + event_grammar_hash: dataset.event_grammar_hash.clone(), + target_domain_support_hash: dataset.target_domain_support_hash.clone(), + causal_header_profile_hash: dataset.causal_header_profile_hash.clone(), + target_size_function: dataset.target_size_function, + evaluator_interface_version: crate::tuner::TUNER_EVALUATOR_INTERFACE_VERSION, + candidate_canonicalization_version: "bounds-v1".to_string(), + warmup_baseline_runs: 0, + diagnostic_chunk_bytes: None, + eval_time_limit_seconds: full_effective_limit, + evaluator_threads: 1, + worker_isolation_mode: "spawn_exec_worker", + worker_executable_identity: None, + resolved_memory_accounting_kind: "unix_process_rss_fallback_explicit", + resolved_memory_accounting_strict_theorem_facing: false, + resolved_evaluator_cgroup_parent: None, + backend_report_component_policy: "none", + evaluator_determinism: "deterministic_under_h", + rss_mode: crate::tuner::PeakMemoryMode::ProcessRssPeak, + timing_certification_tier: crate::tuner::TimingCertificationTier::BestEffort, + build_profile: "unknown", + feature_set: crate::tuner::compiled_feature_set(), + }; + + let mut profile2 = profile1.clone(); + profile2.eval_time_limit_seconds = truncated_effective_limit; + + let bytes = z.canonical_bytes().as_slice(); + let key1 = cache_key_for_candidate(bytes, &profile1, &dataset.canonical_content_hash).unwrap(); + let key2 = cache_key_for_candidate(bytes, &profile2, &dataset.canonical_content_hash).unwrap(); + assert_ne!(key1, key2); + assert_eq!( + key1.candidate_canonical_bytes, + key2.candidate_canonical_bytes + ); + assert_eq!(key1.dataset_identity, key2.dataset_identity); + assert_ne!(key1.evaluator_profile_bytes, key2.evaluator_profile_bytes); + assert_ne!(profile1.hash().unwrap(), profile2.hash().unwrap()); + let _ = std::fs::remove_file(dataset_path); +} + +// --- Group 3: Deployability and objective-totalization properties --- + +#[cfg(feature = "backend-ctw")] +#[test] +fn objective_totalizes_nondeployable_to_infinity_over_randomized_states() { + use crate::api::{CompressionBackend, RateBackend}; + use crate::compression::FramingMode; + use crate::tuner::eval::evaluate_candidate; + use crate::tuner::tests::temp_path; + use crate::tuner::{ + CandidateEvalStatus, DeterministicEvaluatorRow, VerifiedDeterministicEvaluatorTable, + }; + use std::collections::HashMap; + + let z1 = CompressionBackend::Rate { + rate_backend: RateBackend::Ctw { depth: 4 }, + coder: crate::coders::CoderType::AC, + framing: FramingMode::Framed, + } + .compile() + .unwrap(); + let dataset_path = temp_path("dataset-dep", ".bin"); + std::fs::write(&dataset_path, b"1234567890123456").unwrap(); // 16 bytes + let dataset = crate::tuner::tests::load_dataset(&dataset_path).unwrap(); + let crc = crate::tuner::crc32_hex(z1.canonical_bytes().as_slice()); + + let mut rng = crate::tuner::RandomGenerator::new(); + let runtime_profile = crate::tuner::eval::ResolvedEvaluatorRuntimeProfile { + worker_executable: None, + worker_executable_identity: None, + resolved_evaluator_cgroup_parent: None, + memory_accounting_kind: + crate::tuner::eval::ResolvedMemoryAccountingKind::DeterministicEvaluatorTable, + }; + + // Generative test over 100 random states + for _ in 0..100 { + let status = match rng.next_u64() % 4 { + 0 => CandidateEvalStatus::Success, + 1 => CandidateEvalStatus::Timeout, + 2 => CandidateEvalStatus::Invalid, + _ => CandidateEvalStatus::Error, + }; + // Generate random elapsed seconds between 0.001 and 10.0 + let elapsed = 0.001 + (rng.next_u64() as f64 / u64::MAX as f64) * 9.999; + // Generate random peak memory up to 10MB + let peak_mem = rng.next_u64() % 10_000_000; + // fixed target_loss_bits for simplicity, it's valid + let target_loss = 80.0; + + let mut rows = HashMap::new(); + rows.insert( + crc.clone(), + DeterministicEvaluatorRow { + status: status.clone(), + compressed_bytes: 10, + elapsed_seconds: elapsed, + peak_memory_bytes: peak_mem, + target_loss_bits: target_loss, + }, + ); + let table = VerifiedDeterministicEvaluatorTable { + base: crate::tuner::VerifiedCertificate { + ref_value: "ref".to_string(), + content_hash: "hash".to_string(), + }, + rows, + }; + + // Strict thresholds + let min_tp = 50.0; + let max_mem = 2000; + + let res = evaluate_candidate( + &z1, + &dataset, + 1, + min_tp, + max_mem, + 10.0, + 1, + &runtime_profile, + Some(&table), + ) + .unwrap(); + + let tp = 16.0 / elapsed; + let is_deployable = + matches!(status, CandidateEvalStatus::Success) && tp >= min_tp && peak_mem <= max_mem; + + assert_eq!( + res.deployable, is_deployable, + "deployable flag mismatch for status={:?}, tp={}, mem={}", + status, tp, peak_mem + ); + if is_deployable { + assert!( + res.objective_bits.is_finite(), + "deployable candidate must have finite objective" + ); + } else { + assert_eq!( + res.objective_bits, + f64::INFINITY, + "non-deployable candidate must totalize to infinity" + ); + } + } + + let _ = std::fs::remove_file(dataset_path); +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn cache_key_is_exact_tuple_of_candidate_profile_and_dataset_identity() { + use crate::api::{CompressionBackend, RateBackend}; + use crate::compression::FramingMode; + let z1 = CompressionBackend::Rate { + rate_backend: RateBackend::Ctw { depth: 4 }, + coder: crate::coders::CoderType::AC, + framing: FramingMode::Framed, + } + .compile() + .unwrap(); + let z2 = CompressionBackend::Rate { + rate_backend: RateBackend::Ctw { depth: 5 }, + coder: crate::coders::CoderType::AC, + framing: FramingMode::Framed, + } + .compile() + .unwrap(); + let profile1 = EvaluatorProfile { + dataset_kind: DatasetKind::PassiveBytes, + objective_target: ObjectiveTarget::PassiveAc, + dataset_lowering_version: PASSIVE_DATASET_LOWERING_VERSION, + dataset_codec_hash: "passive-identity-bytes".to_string(), + event_grammar_hash: "passive-target-only-byte-stream".to_string(), + target_domain_support_hash: crc32_hex(b"passive-byte-alphabet"), + causal_header_profile_hash: crc32_hex(b"passive-none"), + target_size_function: "passive-bytes-len", + evaluator_interface_version: TUNER_EVALUATOR_INTERFACE_VERSION, + candidate_canonicalization_version: "bounds-v1".to_string(), + warmup_baseline_runs: 0, + diagnostic_chunk_bytes: None, + eval_time_limit_seconds: 1.0, + evaluator_threads: 1, + worker_isolation_mode: "spawn_exec_worker", + worker_executable_identity: None, + resolved_memory_accounting_kind: "unix_process_rss_fallback_explicit", + resolved_memory_accounting_strict_theorem_facing: false, + resolved_evaluator_cgroup_parent: None, + backend_report_component_policy: "none", + evaluator_determinism: "deterministic_under_h", + rss_mode: PeakMemoryMode::ProcessRssPeak, + timing_certification_tier: TimingCertificationTier::BestEffort, + build_profile: "test", + feature_set: vec!["test"], + }; + let mut profile2 = profile1.clone(); + profile2.eval_time_limit_seconds = 2.0; + let dataset1 = crc32_hex(b"dataset-one"); + let dataset2 = crc32_hex(b"dataset-two"); + let z1_bytes = z1.canonical_bytes().as_slice(); + let z2_bytes = z2.canonical_bytes().as_slice(); + + let key = cache_key_for_candidate(z1_bytes, &profile1, &dataset1).unwrap(); + let same = cache_key_for_candidate(z1_bytes, &profile1, &dataset1).unwrap(); + let changed_candidate = cache_key_for_candidate(z2_bytes, &profile1, &dataset1).unwrap(); + let changed_profile = cache_key_for_candidate(z1_bytes, &profile2, &dataset1).unwrap(); + let changed_dataset = cache_key_for_candidate(z1_bytes, &profile1, &dataset2).unwrap(); + + assert_eq!(key, same); + assert_ne!(key, changed_candidate); + assert_ne!(key, changed_profile); + assert_ne!(key, changed_dataset); + assert_eq!(key.candidate_canonical_bytes, z1_bytes); + assert_eq!(key.dataset_identity, dataset1); + assert_ne!( + key.evaluator_profile_bytes, + changed_profile.evaluator_profile_bytes + ); +} + +// --- Group 5: Causal evaluator semantic properties --- + +#[cfg(feature = "backend-ctw")] +#[test] +fn observe_target_no_score_contributes_zero_bits() { + // Property: the action context field (ObserveTargetNoScore) conditions the + // predictor state but must itself contribute exactly zero bits to target_loss_bits. + // + // Pure semantic proof without heuristic numeric bounds: + // Evaluate Dataset A: [Observe(0)], Dataset B: [Target(0)], + // and Dataset C: [Observe(0), Target(0)]. + // + // We assert: + // - loss(A) == 0.0 (an ObserveTargetNoScore event literally costs 0.0 bits) + // - loss(C) != loss(B), proving the replay event changed future predictor state. + use crate::api::{CompressionBackend, RateBackend}; + use crate::compression::FramingMode; + use crate::tuner::eval::evaluate_candidate_causal_loss; + use crate::tuner::tests::{causal_dataset_value, temp_path}; + + let z = CompressionBackend::Rate { + rate_backend: RateBackend::Ctw { depth: 8 }, + coder: crate::coders::CoderType::AC, + framing: FramingMode::Framed, + } + .compile() + .unwrap(); + + let path_a = temp_path("causal-obs-a", ".json"); + std::fs::write( + &path_a, + causal_dataset_value( + "json-causal-byte-events-v1", + "events", + serde_json::json!([ + { "kind": "observe_target_no_score", "channel": "percept", "domain": "binary", "bytes": [0] } + ]), + ) + .to_string(), + ) + .unwrap(); + let path_b = temp_path("causal-target-b", ".json"); + std::fs::write( + &path_b, + causal_dataset_value( + "json-causal-byte-events-v1", + "events", + serde_json::json!([ + { "kind": "target", "channel": "percept", "domain": "binary", "bytes": [0] } + ]), + ) + .to_string(), + ) + .unwrap(); + let path_c = temp_path("causal-obs-c", ".json"); + std::fs::write( + &path_c, + causal_dataset_value( + "json-causal-byte-events-v1", + "events", + serde_json::json!([ + { "kind": "observe_target_no_score", "channel": "percept", "domain": "binary", "bytes": [0] }, + { "kind": "target", "channel": "percept", "domain": "binary", "bytes": [0] } + ]), + ) + .to_string(), + ) + .unwrap(); + + let ds_a = crate::tuner::tests::load_dataset(&path_a).unwrap(); + let ds_b = crate::tuner::tests::load_dataset(&path_b).unwrap(); + let ds_c = crate::tuner::tests::load_dataset(&path_c).unwrap(); + + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + let res_a = evaluate_candidate_causal_loss(&z, &ds_a, deadline).unwrap(); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + let res_b = evaluate_candidate_causal_loss(&z, &ds_b, deadline).unwrap(); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + let res_c = evaluate_candidate_causal_loss(&z, &ds_c, deadline).unwrap(); + + let loss_a: f64 = res_a.1; + let loss_b: f64 = res_b.1; + let loss_c: f64 = res_c.1; + + assert_eq!( + loss_a, 0.0, + "An isolated ObserveTargetNoScore event must contribute exactly 0.0 bits" + ); + assert!( + loss_b.is_finite() && loss_c.is_finite(), + "charged target losses must be finite for the binary target-domain test" + ); + assert_ne!( + loss_b.to_bits(), + loss_c.to_bits(), + "ObserveTargetNoScore must update future predictor state; otherwise replay+target would equal target-only" + ); + + let _ = std::fs::remove_file(path_a); + let _ = std::fs::remove_file(path_b); + let _ = std::fs::remove_file(path_c); +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn exact_mdl_map_equivalence_over_finite_semantic_class() { + let candidates = [ + (16.0_f64, 2.0_f64.powi(-20)), + (24.0_f64, 2.0_f64.powi(-6)), + (32.0_f64, 2.0_f64.powi(-1)), + (8.0_f64, 0.0_f64), + ]; + let objectives = candidates + .iter() + .map(|(model_bits, likelihood)| { + if *likelihood == 0.0 { + f64::INFINITY + } else { + *model_bits - likelihood.log2() + } + }) + .collect::>(); + let posterior_scores = candidates + .iter() + .map(|(model_bits, likelihood)| 2.0_f64.powf(-*model_bits) * *likelihood) + .collect::>(); + let prior_mass = candidates + .iter() + .map(|(model_bits, _)| 2.0_f64.powf(-*model_bits)) + .sum::(); + let normalized_scores = candidates + .iter() + .map(|(model_bits, likelihood)| (2.0_f64.powf(-*model_bits) / prior_mass) * *likelihood) + .collect::>(); + let argmin_objective = objectives + .iter() + .enumerate() + .min_by(|(_, a), (_, b)| a.total_cmp(b)) + .map(|(index, _)| index) + .unwrap(); + let argmax_posterior = posterior_scores + .iter() + .enumerate() + .max_by(|(_, a), (_, b)| a.total_cmp(b)) + .map(|(index, _)| index) + .unwrap(); + let argmax_normalized = normalized_scores + .iter() + .enumerate() + .max_by(|(_, a), (_, b)| a.total_cmp(b)) + .map(|(index, _)| index) + .unwrap(); + let mixture_codelength = -posterior_scores.iter().sum::().log2(); + let best_objective = objectives[argmin_objective]; + + assert_eq!(argmin_objective, argmax_posterior); + assert_eq!(argmin_objective, argmax_normalized); + assert!(mixture_codelength <= best_objective); +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn finite_incumbent_oracle_enforces_monotone_key_and_objective() { + fn eval(objective_bits: f64, deployable: bool) -> CandidateEvalResult { + CandidateEvalResult { + status: if deployable { + CandidateEvalStatus::Success + } else { + CandidateEvalStatus::Timeout + }, + compressed_bytes: 0, + elapsed_seconds: 1.0, + effective_eval_time_limit_seconds: 1.0, + throughput_bytes_per_second: if deployable { 1.0 } else { 0.0 }, + peak_memory_bytes: 0, + target_loss_bits: objective_bits, + objective_bits, + deployable, + } + } + + let mut best_eval = eval(10.0, true); + let mut best_bytes = vec![0x20]; + let mut updates = vec![(best_eval.clone(), best_bytes.clone())]; + let candidates = vec![ + (eval(f64::INFINITY, false), vec![0x00]), + (eval(12.0, true), vec![0x00]), + (eval(8.0, true), vec![0xff]), + (eval(8.0, true), vec![0x01]), + (eval(9.0, true), vec![0x00]), + (eval(5.0, true), vec![0x80]), + ]; + + for (candidate_eval, candidate_bytes) in candidates { + let old_best_eval = best_eval.clone(); + let old_best_bytes = best_bytes.clone(); + let should_update = candidate_eval.deployable + && key_less(&candidate_eval, &candidate_bytes, &best_eval, &best_bytes); + + if should_update { + assert!(candidate_eval.objective_bits <= old_best_eval.objective_bits); + if candidate_eval.objective_bits == old_best_eval.objective_bits { + assert!(candidate_bytes < old_best_bytes); + } + best_eval = candidate_eval; + best_bytes = candidate_bytes; + updates.push((best_eval.clone(), best_bytes.clone())); + } else { + assert_eq!( + best_eval.objective_bits.to_bits(), + old_best_eval.objective_bits.to_bits() + ); + assert_eq!(best_bytes, old_best_bytes); + } + } + + for pair in updates.windows(2) { + let (previous_eval, previous_bytes) = &pair[0]; + let (next_eval, next_bytes) = &pair[1]; + assert!(key_less( + next_eval, + next_bytes, + previous_eval, + previous_bytes + )); + assert!(next_eval.objective_bits <= previous_eval.objective_bits); + } + + let accepted_current_eval = eval(7.0, true); + let accepted_current_bytes = vec![0x00]; + assert!(!key_less( + &accepted_current_eval, + &accepted_current_bytes, + &best_eval, + &best_bytes + )); + assert!(accepted_current_eval.objective_bits > best_eval.objective_bits); + assert_eq!(best_eval.objective_bits, 5.0); + assert_eq!(best_bytes, vec![0x80]); +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn exact_reward_encoder_telescopes_incumbent_objective_decreases() { + fn eval(objective_bits: f64, deployable: bool) -> CandidateEvalResult { + CandidateEvalResult { + status: if deployable { + CandidateEvalStatus::Success + } else { + CandidateEvalStatus::Timeout + }, + compressed_bytes: 0, + elapsed_seconds: 1.0, + effective_eval_time_limit_seconds: 1.0, + throughput_bytes_per_second: if deployable { 1.0 } else { 0.0 }, + peak_memory_bytes: 0, + target_loss_bits: objective_bits, + objective_bits, + deployable, + } + } + + let encoder = TunerRewardEncoder::ExactIntegerObjectiveDifference { + max_reward: 20, + objective_difference_to_symbol: None, + }; + for reward in 0..=20 { + assert_eq!(encoder.encode(reward as f64).unwrap(), reward); + } + + let baseline_eval = eval(10.0, true); + let baseline_bytes = vec![0x20]; + let mut best_eval = baseline_eval.clone(); + let mut best_bytes = baseline_bytes; + let mut decoded_reward_sum = 0_i64; + let candidates = vec![ + (eval(12.0, true), vec![0x00]), + (eval(7.0, true), vec![0xff]), + (eval(7.0, true), vec![0x01]), + (eval(f64::INFINITY, false), vec![0x00]), + (eval(4.0, true), vec![0x80]), + ]; + + for (candidate_eval, candidate_bytes) in candidates { + let improves_best = candidate_eval.deployable + && key_less(&candidate_eval, &candidate_bytes, &best_eval, &best_bytes); + let raw_improvement = if improves_best { + (best_eval.objective_bits - candidate_eval.objective_bits).max(0.0) + } else { + 0.0 + }; + let reward = encoder.encode(raw_improvement).unwrap(); + decoded_reward_sum = decoded_reward_sum.saturating_add(reward); + if improves_best { + best_eval = candidate_eval; + best_bytes = candidate_bytes; + } + } + + assert_eq!( + decoded_reward_sum as f64, + baseline_eval.objective_bits - best_eval.objective_bits + ); + assert_eq!(decoded_reward_sum, 6); + assert_eq!(best_eval.objective_bits, 4.0); +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn normalized_clipped_improvement_stays_in_unit_interval_and_rejects_degenerate_bounds() { + assert_eq!(normalized_clipped_improvement(-1.0, 0.0, 2.0).unwrap(), 0.0); + assert_eq!(normalized_clipped_improvement(1.0, 0.0, 2.0).unwrap(), 0.5); + assert_eq!(normalized_clipped_improvement(3.0, 0.0, 2.0).unwrap(), 1.0); + assert!(normalized_clipped_improvement(1.0, 2.0, 2.0).is_err()); + + let encoder = TunerRewardEncoder::NormalizedClipped { + min_improvement: 0.0, + max_improvement: 2.0, + max_reward: 10, + }; + assert_eq!(encoder.encode(-1.0).unwrap(), 0); + assert_eq!(encoder.encode(1.0).unwrap(), 5); + assert_eq!(encoder.encode(3.0).unwrap(), 10); +} + +// --- Group 6: Reversible elementary kernel properties --- + +#[cfg(feature = "backend-ctw")] +#[test] +fn inactive_radius_moves_become_self_loops() { + // Property: out-of-radius and boundary-crossing moves must not appear in the + // transition kernel — the move is silently absent (a self-loop in MH terms), + // never clipped to the nearest valid value. + // + // We test two distinct cases: + // + // Case 1 — active_radius=0: no move with any magnitude is within radius, so + // the entire transition map must be empty and sampling yields Exhausted. + // + // Case 2 — boundary self-loop: candidate is at the lower bound (depth=1), so + // the downward delta=-1 move (depth=0) violates the [1,16] bounds and must + // be ABSENT from transitions. The upward delta=+1 move (depth=2) is valid + // and must be PRESENT. This proves boundary moves self-loop rather than clip. + use crate::api::{CompressionBackend, RateBackend}; + use crate::compression::FramingMode; + use crate::spec::TuneParameterRangeSpec; + use crate::tuner::annealer::{ + apply_integer_descriptor, collect_numeric_leaves, compile_canonical_proposal_kernel, + integer_leaf_bounds, + }; + use crate::tuner::tests::sample_tune_spec; + + let mut bounds = sample_tune_spec("", "", "").bounds; + bounds.parameter_ranges = vec![TuneParameterRangeSpec { + parameter: "rate_backend.depth".to_string(), + min: 1.0, + max: 16.0, + }]; + let env = crate::spec::SpecEnvironment::new(std::path::Path::new(".")); + + // --- Case 1: active_radius = 0 yields an empty kernel (Exhausted). --- + let candidate_mid = CompressionBackend::Rate { + rate_backend: RateBackend::Ctw { depth: 4 }, + coder: crate::coders::CoderType::AC, + framing: FramingMode::Framed, + }; + let compiled_mid = candidate_mid.compile().unwrap(); + let bytes_mid = compiled_mid.canonical_bytes().as_slice(); + let kernel_zero_radius = + compile_canonical_proposal_kernel(&candidate_mid, &bounds, 3, 0, &env, bytes_mid).unwrap(); + assert!( + kernel_zero_radius.transitions.is_empty(), + "active_radius=0 must produce an empty transition map" + ); + let mut rng = crate::tuner::RandomGenerator::new(); + let draw = crate::tuner::annealer::sample_annealed_proposal( + &candidate_mid, + &bounds, + 3, + 0, + &env, + &mut rng, + ) + .unwrap(); + assert!( + matches!(draw, crate::tuner::AnnealedProposalDraw::Exhausted), + "sampling from empty kernel must yield Exhausted" + ); + + // --- Case 2: boundary self-loop — candidate at lower bound (depth=1). --- + // With active_radius=1 and max_mutation_radius=3, only magnitude-1 moves are + // within radius. depth=1 is the lower bound, so delta=-1 → depth=0 is out of + // [1,16] and must be absent. delta=+1 → depth=2 is valid and must be present. + let candidate_lb = CompressionBackend::Rate { + rate_backend: RateBackend::Ctw { depth: 1 }, // at lower bound + coder: crate::coders::CoderType::AC, + framing: FramingMode::Framed, + }; + let compiled_lb = candidate_lb.compile().unwrap(); + let bytes_lb = compiled_lb.canonical_bytes().as_slice().to_vec(); + let kernel_lb = + compile_canonical_proposal_kernel(&candidate_lb, &bounds, 3, 1, &env, &bytes_lb).unwrap(); + assert_eq!( + kernel_lb.total_raw_actions, 6, + "kernel raw action space must retain inactive and boundary self-loop descriptors" + ); + assert_eq!( + kernel_lb.transitions.len(), + 1, + "at the lower bound with active radius 1, only the valid upward move may be emitted" + ); + assert_eq!( + kernel_lb.proposal_mass_to_canonical_bytes(&bytes_lb), + 0, + "self-loops must not be emitted as explicit current-candidate transitions" + ); + + assert!( + kernel_lb + .transitions + .iter() + .all(|t| t.candidate_canonical_bytes != bytes_lb), + "boundary and inactive self-loops must remain implicit, not emitted as clipped current transitions" + ); + + // The upward move (depth=2) MUST appear in transitions. + let depth2_candidate = CompressionBackend::Rate { + rate_backend: RateBackend::Ctw { depth: 2 }, + coder: crate::coders::CoderType::AC, + framing: FramingMode::Framed, + }; + let depth2_bytes = depth2_candidate + .compile() + .unwrap() + .canonical_bytes() + .as_slice() + .to_vec(); + let has_depth2 = kernel_lb + .transitions + .iter() + .any(|t| t.candidate_canonical_bytes == depth2_bytes); + assert!( + has_depth2, + "depth=2 (valid +1 from lower bound) must be present in transition kernel" + ); + + // Direct application: apply_integer_descriptor must return false for the + // boundary-crossing delta, confirming no implicit clipping occurs. + let json_lb = crate::spec::compression_backend_to_json_value(&candidate_lb).unwrap(); + let leaves = collect_numeric_leaves(&json_lb); + let depth_leaf = leaves + .iter() + .find(|l| l.path.contains("depth")) + .expect("depth leaf"); + let (min_b, max_b) = integer_leaf_bounds(depth_leaf.kind, Some((1.0, 16.0))).unwrap(); + let mut json_mut = json_lb.clone(); + let applied_down = apply_integer_descriptor( + &mut json_mut, + depth_leaf, + depth_leaf.kind, + min_b, + max_b, + 1, + -1, + ); + assert!( + !applied_down, + "apply_integer_descriptor must return false for depth 1 + delta -1 (out of bounds)" + ); + assert_eq!( + json_mut, json_lb, + "failed boundary move must leave the candidate JSON unchanged rather than clipping" + ); + let mut json_mut2 = json_lb.clone(); + let applied_up = apply_integer_descriptor( + &mut json_mut2, + depth_leaf, + depth_leaf.kind, + min_b, + max_b, + 1, + 1, + ); + assert!( + applied_up, + "apply_integer_descriptor must return true for depth 1 + delta +1 (valid move)" + ); +} + +#[cfg(feature = "backend-rosa")] +#[test] +fn signed_range_keeps_integer_kind_stable_across_zero_for_reversibility() { + use crate::api::{CompressionBackend, RateBackend}; + use crate::compression::FramingMode; + use crate::spec::{TuneBoundsSpec, TuneParameterRangeSpec}; + use crate::tuner::annealer::{compile_canonical_proposal_kernel, sample_annealed_proposal}; + + let candidate = CompressionBackend::Rate { + rate_backend: RateBackend::RosaPlus { max_order: -1 }, + coder: crate::coders::CoderType::AC, + framing: FramingMode::Framed, + }; + let bounds = TuneBoundsSpec { + allowed_backends: vec!["rosaplus".to_string()], + forbidden_backends: Vec::new(), + parameter_ranges: vec![TuneParameterRangeSpec { + parameter: "rate_backend.max_order".to_string(), + min: -1.0, + max: 8.0, + }], + max_experts: 2, + max_mixture_nesting_depth: 1, + min_experts: Some(1), + allow_duplicate_experts: Some(false), + required_experts: Vec::new(), + forbidden_expert_pairs: Vec::new(), + }; + let env = crate::spec::SpecEnvironment::new("."); + + let current = candidate.compile_in(&env).expect("compile current"); + let current_bytes = current.canonical_bytes().as_slice().to_vec(); + let kernel = compile_canonical_proposal_kernel(&candidate, &bounds, 1, 1, &env, ¤t_bytes) + .expect("compile proposal kernel"); + assert!( + !kernel.transitions.is_empty(), + "kernel must include at least one transition from max_order=-1" + ); + + let target_zero = CompressionBackend::Rate { + rate_backend: RateBackend::RosaPlus { max_order: 0 }, + coder: crate::coders::CoderType::AC, + framing: FramingMode::Framed, + }; + let target_zero_bytes = target_zero + .compile_in(&env) + .expect("compile max_order=0") + .canonical_bytes() + .as_slice() + .to_vec(); + let zero_proposal = kernel + .transitions + .iter() + .find(|proposal| proposal.candidate_canonical_bytes == target_zero_bytes) + .expect("expected transition from max_order=-1 to max_order=0"); + + let reverse = + compile_canonical_proposal_kernel(&target_zero, &bounds, 1, 1, &env, &target_zero_bytes) + .expect("compile reverse kernel"); + let reverse_mass = reverse.proposal_mass_to_canonical_bytes(¤t_bytes); + assert_eq!( + reverse_mass, zero_proposal.raw_action_count, + "reverse proposal mass must match forward raw action count across -1 <-> 0 boundary" + ); + + let mut rng = crate::tuner::RandomGenerator::new(); + let _ = sample_annealed_proposal(&candidate, &bounds, 1, 1, &env, &mut rng) + .expect("proposal sampling should not fail on signed boundary transition"); +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn finite_evaluator_table_oracle_selects_minimum_deployable_jh() { + use crate::api::{CompressionBackend, RateBackend}; + use crate::compression::FramingMode; + use crate::tuner::tests::temp_path; + use crate::tuner::{ + CandidateEvalResult, CandidateEvalStatus, DeterministicEvaluatorRow, + VerifiedDeterministicEvaluatorTable, + }; + use std::collections::HashMap; + + let ds_path = temp_path("oracle", ".bin"); + std::fs::write(&ds_path, b"test").unwrap(); + let loaded = crate::tuner::tests::load_dataset(&ds_path).unwrap(); + + let candidates = (1_usize..=8) + .map(|depth| { + ( + depth, + CompressionBackend::Rate { + rate_backend: RateBackend::Ctw { depth }, + coder: crate::coders::CoderType::AC, + framing: FramingMode::Framed, + } + .compile() + .unwrap(), + ) + }) + .collect::>(); + let mut rows = HashMap::::new(); + for (depth, candidate) in &candidates { + let target_loss_bits = match depth { + 1 => 130.0, + 2 => 120.0, + 3 => 90.0, + 4 => 80.0, + 5 => 70.0, + 6 => 20.0, + 7 => 1.0, + 8 => 50.0, + _ => unreachable!(), + }; + let peak_memory_bytes = if *depth == 7 { 10_000 } else { 100 }; + rows.insert( + crate::tuner::crc32_hex(candidate.canonical_bytes().as_slice()), + DeterministicEvaluatorRow { + status: CandidateEvalStatus::Success, + compressed_bytes: 10, + target_loss_bits, + elapsed_seconds: 0.01, + peak_memory_bytes, + }, + ); + } + let table = VerifiedDeterministicEvaluatorTable { + base: crate::tuner::VerifiedCertificate { + ref_value: "test://deterministic-table".to_string(), + content_hash: "00000000".to_string(), + }, + rows, + }; + + let max_memory_bytes: u64 = 1_000; + let mut production_best: Option<(usize, CandidateEvalResult, Vec)> = None; + let mut reference_best: Option<(usize, f64, Vec)> = None; + for (depth, candidate) in &candidates { + let model_bytes = candidate.canonical_bytes().len(); + let eval = table + .evaluate(candidate, &loaded, model_bytes, 1.0, max_memory_bytes, 1.0) + .unwrap(); + let candidate_bytes = candidate.canonical_bytes().as_slice().to_vec(); + let reference_objective = if eval.status == CandidateEvalStatus::Success + && eval.throughput_bytes_per_second >= 1.0 + && eval.peak_memory_bytes <= max_memory_bytes + { + (model_bytes as f64 * 8.0) + eval.target_loss_bits + } else { + f64::INFINITY + }; + + if production_best + .as_ref() + .map(|(_, best_eval, best_bytes)| { + key_less(&eval, &candidate_bytes, best_eval, best_bytes) + }) + .unwrap_or(true) + { + production_best = Some((*depth, eval.clone(), candidate_bytes.clone())); + } + if reference_best + .as_ref() + .map(|(_, best_objective, best_bytes)| { + reference_objective < *best_objective + || (reference_objective == *best_objective && candidate_bytes < *best_bytes) + }) + .unwrap_or(true) + { + reference_best = Some((*depth, reference_objective, candidate_bytes)); + } + } + + let production_best = production_best.unwrap(); + let reference_best = reference_best.unwrap(); + assert_eq!(production_best.0, reference_best.0); + assert_eq!(production_best.0, 6); + assert!(production_best.1.deployable); + + let _ = std::fs::remove_file(ds_path); +} diff --git a/tests/ac_log_loss_cli.rs b/crates/infotheory/tests/ac_log_loss_cli.rs similarity index 99% rename from tests/ac_log_loss_cli.rs rename to crates/infotheory/tests/ac_log_loss_cli.rs index 13ee5a1d..11327930 100644 --- a/tests/ac_log_loss_cli.rs +++ b/crates/infotheory/tests/ac_log_loss_cli.rs @@ -1,4 +1,4 @@ -#![cfg(feature = "cli")] +#![cfg(all(feature = "cli", feature = "all-backends"))] use std::fs; use std::path::{Path, PathBuf}; diff --git a/crates/infotheory/tests/aiqi_validation.rs b/crates/infotheory/tests/aiqi_validation.rs new file mode 100644 index 00000000..756d0730 --- /dev/null +++ b/crates/infotheory/tests/aiqi_validation.rs @@ -0,0 +1,624 @@ +#![cfg(all(feature = "aixi", feature = "all-backends"))] + +//! AIQI validation tests. + +use infotheory::aixi::aiqi::{AiqiAgent, AiqiConfig, AiqiError}; +use infotheory::aixi::common::{ActionAlphabet, DEFAULT_RANDOM_SEED}; +use infotheory::aixi::environment::Environment; +mod support; +use infotheory::aixi::model::{ + RateBackendBitPredictor, RateBackendBitPredictorConfig, RateBackendBitPredictorError, +}; +use infotheory::api::{BitOrder, BitStreamSemantics, MixtureKind, MixtureSpec, RateBackend}; +use std::sync::Arc; +use support::aixi_envs::{DeterministicBinaryEnv, SeededCoinFlipEnv}; + +fn base_config() -> AiqiConfig { + let mut cfg = AiqiConfig::default(); + cfg.rate_backend = RateBackend::Ctw { depth: 8 }; + cfg.bit_stream_semantics = infotheory::api::BitStreamSemantics::BinaryTokens; + cfg.observation_bits = 1; + cfg.observation_stream_len = 1; + cfg.reward_bits = 1; + cfg.agent_actions = + ActionAlphabet::try_from_usize(2).expect("test fixture action alphabet must be valid"); + cfg.min_reward = 0; + cfg.max_reward = 1; + cfg.reward_offset = 0; + cfg.discount_gamma = 0.99; + cfg.return_horizon = 2; + cfg.return_bins = 8; + cfg.augmentation_period = 2; + cfg.history_prune_keep_steps = None; + cfg.baseline_exploration = 0.01; + cfg.random_seed = Some(11); + cfg +} + +#[test] +fn default_aiqi_config_validates() { + AiqiConfig::default() + .validate() + .expect("default AIQI config should satisfy its own contract"); +} + +fn aiqi_mixture_backend(kind: MixtureKind) -> RateBackend { + let experts = vec![ + { + let mut expert = infotheory::api::MixtureExpertSpec::new(RateBackend::Ctw { depth: 8 }); + expert.name = Some("ctw".to_string()); + expert.log_prior = 0.0; + expert + }, + { + let mut expert = infotheory::api::MixtureExpertSpec::new(RateBackend::FacCtw { + base_depth: 8, + num_percept_bits: 8, + encoding_bits: 1, + msb_first: None, + }); + expert.name = Some("fac-ctw".to_string()); + expert.log_prior = 0.0; + expert + }, + ]; + let alpha = match kind { + MixtureKind::Switching => 0.05, + MixtureKind::Convex => 1.25, + _ => 0.03, + }; + RateBackend::Mixture { + spec: Arc::new(MixtureSpec::new(kind, experts).with_alpha(alpha)), + } +} + +fn run_aiqi_env(agent: &mut AiqiAgent, mut env: T, cycles: usize) -> i64 { + let mut total_reward = 0i64; + for _ in 0..cycles { + let action = agent.get_planned_action(); + env.perform_action(action); + let obs_stream = env.drain_observations(); + let rew = env.get_reward(); + agent + .observe_transition(action, &obs_stream, rew) + .expect("transition must be accepted"); + total_reward += rew; + } + total_reward +} + +#[test] +fn aiqi_config_rejects_period_shorter_than_horizon() { + let mut cfg = base_config(); + cfg.return_horizon = 3; + cfg.augmentation_period = 2; + let err = cfg.validate().expect_err("N < H must be rejected"); + assert!(matches!( + err, + AiqiError::AugmentationPeriodTooShort { + augmentation_period: 2, + return_horizon: 3 + } + )); +} + +#[test] +fn aiqi_config_accepts_non_power_of_two_return_bins() { + let mut cfg = base_config(); + cfg.return_bins = 3; + cfg.validate() + .expect("non-power-of-two return_bins are valid AIQI discretization levels"); +} + +#[test] +fn aiqi_config_rejects_zpaq_rate_backend_in_strict_mode() { + let mut cfg = base_config(); + cfg.rate_backend = RateBackend::Zpaq { + method: infotheory::api::ZpaqMethodSpec::literal("1"), + }; + let err = cfg + .validate() + .expect_err("strict AIQI should reject zpaq rate backend"); + assert!(matches!(err, AiqiError::UnsupportedRateBackend { .. })); +} + +#[test] +fn aiqi_config_rejects_invalid_programmatic_mixture_rate_backend() { + let mut cfg = base_config(); + cfg.rate_backend = RateBackend::Mixture { + spec: Arc::new(MixtureSpec::new(MixtureKind::Bayes, vec![])), + }; + let err = cfg + .validate() + .expect_err("empty mixture backend should be rejected"); + assert!(matches!(err, AiqiError::InvalidRateBackend(_))); +} + +#[test] +fn aiqi_coinflip_smoke_runs() { + let mut agent = AiqiAgent::new(base_config()).expect("valid AIQI config"); + let mut env = SeededCoinFlipEnv::new(0.8); + + let mut total_reward = 0i64; + for _ in 0..64 { + let action = agent.get_planned_action(); + env.perform_action(action); + let obs_stream = env.drain_observations(); + let rew = env.get_reward(); + agent + .observe_transition(action, &obs_stream, rew) + .expect("transition must be accepted"); + total_reward += rew; + } + + assert!(total_reward >= 0); +} + +#[test] +fn aiqi_learns_ctw_test_pattern() { + let mut cfg = base_config(); + cfg.discount_gamma = 0.7; + cfg.return_horizon = 4; + cfg.augmentation_period = 4; + cfg.rate_backend = RateBackend::Ctw { depth: 10 }; + cfg.baseline_exploration = 1e-6; + + let mut agent = AiqiAgent::new(cfg).expect("valid AIQI config"); + let mut env = DeterministicBinaryEnv::new(); + + let mut total_reward = 0i64; + for _ in 0..120 { + let action = agent.get_planned_action(); + env.perform_action(action); + let obs_stream = env.drain_observations(); + let rew = env.get_reward(); + agent + .observe_transition(action, &obs_stream, rew) + .expect("transition must be accepted"); + total_reward += rew; + } + + assert!( + total_reward > 50, + "AIQI failed to learn DeterministicBinaryEnv pattern; total_reward={total_reward}" + ); +} + +#[test] +fn aiqi_with_generic_rate_backend_smoke_runs() { + let mut cfg = base_config(); + cfg.rate_backend = RateBackend::Match { + hash_bits: 16, + min_len: 2, + max_len: 16, + base_mix: 0.05, + confidence_scale: 1.0, + }; + + let mut agent = AiqiAgent::new(cfg).expect("valid AIQI config"); + let mut env = SeededCoinFlipEnv::new(0.7); + + for _ in 0..24 { + let action = agent.get_planned_action(); + env.perform_action(action); + let obs_stream = env.drain_observations(); + let rew = env.get_reward(); + agent + .observe_transition(action, &obs_stream, rew) + .expect("transition must be accepted"); + } + + assert!(agent.steps_observed() >= 24); +} + +#[test] +fn aiqi_with_rosa_generic_planner_smoke_runs() { + let mut cfg = base_config(); + cfg.rate_backend = RateBackend::RosaPlus { max_order: 20 }; + + let mut agent = AiqiAgent::new(cfg).expect("valid AIQI config"); + let mut env = SeededCoinFlipEnv::new(0.7); + + for _ in 0..24 { + let action = agent.get_planned_action(); + env.perform_action(action); + let obs_stream = env.drain_observations(); + let rew = env.get_reward(); + agent + .observe_transition(action, &obs_stream, rew) + .expect("transition must be accepted"); + } + + assert!(agent.steps_observed() >= 24); +} + +#[test] +fn aiqi_bytepacked_ctw_planner_handles_shared_percept_byte() { + let mut cfg = base_config(); + cfg.bit_stream_semantics = BitStreamSemantics::BytePacked { + order: BitOrder::MsbFirst, + }; + cfg.rate_backend = RateBackend::Ctw { depth: 8 }; + cfg.agent_actions = + ActionAlphabet::try_from_usize(129).expect("129 actions require one byte of action bits"); + cfg.observation_bits = 3; + cfg.observation_stream_len = 1; + cfg.reward_bits = 5; + cfg.min_reward = 0; + cfg.max_reward = 31; + cfg.reward_offset = 0; + cfg.return_bins = 256; + cfg.return_horizon = 2; + cfg.augmentation_period = 2; + cfg.baseline_exploration = 1e-12; + cfg.random_seed = Some(0x0A10_1B17); + + let mut left = AiqiAgent::new(cfg.clone()).expect("valid byte-packed AIQI CTW config"); + let mut right = AiqiAgent::new(cfg).expect("valid replay byte-packed AIQI CTW config"); + + let mut planned_actions = Vec::new(); + for step in 0..8usize { + let planned_left = left.get_planned_action(); + let planned_right = right.get_planned_action(); + assert_eq!( + planned_left, planned_right, + "byte-packed AIQI planning must be deterministic at step {step}" + ); + planned_actions.push(planned_left); + + let action = (planned_left ^ ((step as u64).wrapping_mul(37))) % 129; + let observation = [((0b101usize ^ (step * 3) ^ action as usize) & 0b111) as u64]; + let reward = ((0b10001usize ^ (step * 5) ^ action as usize) & 0b1_1111) as i64; + + left.observe_transition(action, &observation, reward) + .expect("left byte-packed transition should be accepted"); + right + .observe_transition(action, &observation, reward) + .expect("right byte-packed transition should be accepted"); + } + + assert_eq!(left.steps_observed(), 8); + assert_eq!(right.steps_observed(), 8); + assert!( + planned_actions.iter().all(|&action| action < 129), + "all byte-packed AIQI actions must stay inside the configured alphabet: {planned_actions:?}" + ); +} + +/// Exercises `BitStreamSemantics::BytePacked` combined with `RateBackend::FacCtw` +/// inside an AIQI planner loop — covering the FacCtw + BytePacked planner path +/// that is distinct from both the native-CTW BytePacked path and the +/// BinaryTokens FacCtw path already exercised elsewhere. +/// +/// Two independently-constructed agents with identical configuration and seed +/// must produce identical action sequences across multiple plan/observe cycles, +/// and every planned action must lie within the configured alphabet. +#[test] +fn aiqi_bytepacked_fac_ctw_planner_handles_shared_percept_byte() { + let mut cfg = base_config(); + cfg.bit_stream_semantics = BitStreamSemantics::BytePacked { + order: BitOrder::MsbFirst, + }; + cfg.rate_backend = RateBackend::FacCtw { + base_depth: 8, + // 3-bit observation + 5-bit reward share one percept byte. + num_percept_bits: 8, + encoding_bits: 1, + msb_first: Some(true), + }; + cfg.agent_actions = + ActionAlphabet::try_from_usize(129).expect("129 actions require one byte of action bits"); + cfg.observation_bits = 3; + cfg.observation_stream_len = 1; + cfg.reward_bits = 5; + cfg.min_reward = 0; + cfg.max_reward = 31; + cfg.reward_offset = 0; + cfg.return_bins = 256; + cfg.return_horizon = 2; + cfg.augmentation_period = 2; + cfg.baseline_exploration = 1e-12; + cfg.random_seed = Some(0xA17F_AC17); + + let mut left = AiqiAgent::new(cfg.clone()).expect("valid byte-packed AIQI FAC-CTW config"); + let mut right = AiqiAgent::new(cfg).expect("valid replay byte-packed AIQI FAC-CTW config"); + + let mut planned_actions = Vec::new(); + for step in 0..8usize { + let planned_left = left.get_planned_action(); + let planned_right = right.get_planned_action(); + assert_eq!( + planned_left, planned_right, + "byte-packed AIQI FAC-CTW planning must be deterministic at step {step}" + ); + planned_actions.push(planned_left); + + let action = (planned_left ^ ((step as u64).wrapping_mul(19))) % 129; + let observation = [((0b110usize ^ (step * 7) ^ action as usize) & 0b111) as u64]; + let reward = ((0b01101usize ^ (step * 9) ^ action as usize) & 0b1_1111) as i64; + + left.observe_transition(action, &observation, reward) + .expect("left byte-packed FAC-CTW transition should be accepted"); + right + .observe_transition(action, &observation, reward) + .expect("right byte-packed FAC-CTW transition should be accepted"); + } + + assert_eq!(left.steps_observed(), 8); + assert_eq!(right.steps_observed(), 8); + assert!( + planned_actions.iter().all(|&action| action < 129), + "all byte-packed AIQI FAC-CTW actions must stay inside configured alphabet: {planned_actions:?}" + ); +} + +/// `BitStreamSemantics::BinaryTokens` with 8-bit MSB FacCtw in a full AIQI planner loop. +/// Native path is [`FacCtwPredictor`] (per `percept_bits` lanes), not byte-prefix MSB hooks. +#[test] +fn aiqi_binarytokens_fac_ctw_native_planner_integration() { + let compiled = RateBackend::FacCtw { + base_depth: 8, + num_percept_bits: 8, + encoding_bits: 8, + msb_first: Some(true), + } + .compile() + .expect("compile fac-ctw backend"); + let caps = compiled.capabilities(); + assert!(caps.supports_native_bit_prediction); + assert!(caps.supports_reversible_bit_updates); + + let mut cfg = base_config(); + cfg.bit_stream_semantics = BitStreamSemantics::BinaryTokens; + cfg.rate_backend = RateBackend::FacCtw { + base_depth: 8, + num_percept_bits: 8, + encoding_bits: 8, + msb_first: Some(true), + }; + cfg.agent_actions = + ActionAlphabet::try_from_usize(129).expect("129 actions require one byte of action bits"); + cfg.observation_bits = 3; + cfg.observation_stream_len = 1; + cfg.reward_bits = 5; + cfg.min_reward = 0; + cfg.max_reward = 31; + cfg.reward_offset = 0; + cfg.return_bins = 256; + cfg.return_horizon = 2; + cfg.augmentation_period = 2; + cfg.baseline_exploration = 1e-12; + cfg.random_seed = Some(0xB17A_1701); + + let mut left = AiqiAgent::new(cfg.clone()).expect("valid BinaryTokens AIQI FAC-CTW config"); + let mut right = AiqiAgent::new(cfg).expect("valid replay BinaryTokens AIQI FAC-CTW config"); + + let mut planned_actions = Vec::new(); + for step in 0..8usize { + let planned_left = left.get_planned_action(); + let planned_right = right.get_planned_action(); + assert_eq!( + planned_left, planned_right, + "BinaryTokens AIQI FAC-CTW planning must be deterministic at step {step}" + ); + planned_actions.push(planned_left); + + let action = (planned_left ^ ((step as u64).wrapping_mul(23))) % 129; + let observation = [((0b101usize ^ (step * 3) ^ action as usize) & 0b111) as u64]; + let reward = ((0b10001usize ^ (step * 5) ^ action as usize) & 0b1_1111) as i64; + + left.observe_transition(action, &observation, reward) + .expect("left BinaryTokens FAC-CTW transition should be accepted"); + right + .observe_transition(action, &observation, reward) + .expect("right BinaryTokens FAC-CTW transition should be accepted"); + } + + assert_eq!(left.steps_observed(), 8); + assert_eq!(right.steps_observed(), 8); + assert!( + planned_actions.iter().all(|&action| action < 129), + "all BinaryTokens AIQI FAC-CTW actions must stay inside configured alphabet: {planned_actions:?}" + ); +} + +#[test] +fn aiqi_optional_history_pruning_smoke_runs() { + let mut cfg = base_config(); + cfg.return_horizon = 3; + cfg.augmentation_period = 4; + cfg.history_prune_keep_steps = Some(16); + + let mut agent = AiqiAgent::new(cfg).expect("valid AIQI config"); + let mut env = SeededCoinFlipEnv::new(0.7); + + for _ in 0..128 { + let action = agent.get_planned_action(); + env.perform_action(action); + let obs_stream = env.drain_observations(); + let rew = env.get_reward(); + agent + .observe_transition(action, &obs_stream, rew) + .expect("transition must be accepted"); + } + + assert_eq!(agent.steps_observed(), 128); +} + +#[test] +fn aiqi_seeded_policy_is_reproducible() { + let mut cfg = base_config(); + cfg.baseline_exploration = 0.35; + cfg.random_seed = Some(987654321); + + let mut a = AiqiAgent::new(cfg.clone()).expect("valid AIQI config"); + let mut b = AiqiAgent::new(cfg).expect("valid AIQI config"); + + for step in 0..128usize { + let act_a = a.get_planned_action(); + let act_b = b.get_planned_action(); + assert_eq!(act_a, act_b, "action mismatch at step {step}"); + + let obs = [(step % 2) as u64]; + let rew = (step % 2) as i64; + a.observe_transition(act_a, &obs, rew) + .expect("transition should be accepted"); + b.observe_transition(act_b, &obs, rew) + .expect("transition should be accepted"); + } +} + +#[test] +fn aiqi_omitted_seed_matches_explicit_default_seed() { + let mut cfg_omitted = base_config(); + cfg_omitted.random_seed = None; + cfg_omitted.baseline_exploration = 0.2; + let mut cfg_explicit = cfg_omitted.clone(); + cfg_explicit.random_seed = Some(DEFAULT_RANDOM_SEED); + + let mut a = AiqiAgent::new(cfg_omitted).expect("agent with omitted seed"); + let mut b = AiqiAgent::new(cfg_explicit).expect("agent with explicit default seed"); + + assert_eq!(a.resolved_random_seed(), DEFAULT_RANDOM_SEED); + assert_eq!(b.resolved_random_seed(), DEFAULT_RANDOM_SEED); + + for step in 0..96usize { + let act_a = a.get_planned_action(); + let act_b = b.get_planned_action(); + assert_eq!(act_a, act_b, "action mismatch at step {step}"); + + let obs = [((step + 1) % 2) as u64]; + let rew = (step % 2) as i64; + a.observe_transition(act_a, &obs, rew) + .expect("transition should be accepted"); + b.observe_transition(act_b, &obs, rew) + .expect("transition should be accepted"); + } +} + +#[test] +fn aiqi_different_seeds_can_change_exploration_trace() { + let mut cfg_a = base_config(); + cfg_a.baseline_exploration = 0.45; + cfg_a.random_seed = Some(11); + let mut cfg_b = cfg_a.clone(); + cfg_b.random_seed = Some(12); + + let mut a = AiqiAgent::new(cfg_a).expect("agent A"); + let mut b = AiqiAgent::new(cfg_b).expect("agent B"); + + let mut diverged = false; + for step in 0..128usize { + let act_a = a.get_planned_action(); + let act_b = b.get_planned_action(); + if act_a != act_b { + diverged = true; + break; + } + let obs = [(step % 2) as u64]; + let rew = ((step + 1) % 2) as i64; + a.observe_transition(act_a, &obs, rew) + .expect("transition should be accepted"); + b.observe_transition(act_b, &obs, rew) + .expect("transition should be accepted"); + } + + assert!( + diverged, + "different random_seed values should be able to produce different exploratory traces" + ); +} + +#[test] +fn rate_backend_bit_predictor_rejects_zpaq_backend() { + let config = RateBackendBitPredictorConfig::compile( + RateBackend::Zpaq { + method: infotheory::api::ZpaqMethodSpec::literal("1"), + }, + 1e-12, + ) + .expect("zpaq compiles before bit-predictor capability check"); + let err = match RateBackendBitPredictor::new(config) { + Ok(_) => panic!("zpaq must be rejected in RateBackendBitPredictor"), + Err(err) => err, + }; + assert!(matches!(err, RateBackendBitPredictorError::UnsupportedZpaq)); +} + +#[test] +fn aiqi_learns_ctw_pattern_with_fac_ctw_world_model() { + let mut cfg = base_config(); + cfg.discount_gamma = 0.7; + cfg.return_horizon = 4; + cfg.augmentation_period = 4; + cfg.baseline_exploration = 1e-6; + cfg.rate_backend = RateBackend::FacCtw { + base_depth: 10, + num_percept_bits: 8, + encoding_bits: 1, + msb_first: None, + }; + + let mut agent = AiqiAgent::new(cfg).expect("valid AIQI FAC-CTW config"); + let total_reward = run_aiqi_env(&mut agent, DeterministicBinaryEnv::new(), 120); + assert!( + total_reward > 50, + "AIQI FAC-CTW world model failed to learn deterministic pattern; total_reward={total_reward}" + ); +} + +#[test] +fn aiqi_mixture_world_models_learn_deterministic_pattern() { + for (kind, label) in [ + (MixtureKind::Bayes, "bayes"), + (MixtureKind::Switching, "switching"), + (MixtureKind::Convex, "convex"), + ] { + let mut cfg = base_config(); + cfg.discount_gamma = 0.8; + cfg.return_horizon = 4; + cfg.augmentation_period = 4; + cfg.baseline_exploration = 0.01; + cfg.rate_backend = aiqi_mixture_backend(kind); + + let mut agent = AiqiAgent::new(cfg).expect("valid AIQI mixture config"); + let total_reward = run_aiqi_env(&mut agent, DeterministicBinaryEnv::new(), 96); + assert!( + total_reward > 35, + "{label} AIQI mixture world model reward too low on deterministic pattern: {total_reward}" + ); + } +} + +#[test] +fn aiqi_mixture_world_models_are_seed_deterministic() { + for (kind, label) in [ + (MixtureKind::Bayes, "bayes"), + (MixtureKind::Switching, "switching"), + (MixtureKind::Convex, "convex"), + ] { + let mut cfg = base_config(); + cfg.rate_backend = aiqi_mixture_backend(kind); + cfg.baseline_exploration = 0.3; + cfg.random_seed = Some(20260429); + + let mut a = AiqiAgent::new(cfg.clone()).expect("mixture agent A"); + let mut b = AiqiAgent::new(cfg).expect("mixture agent B"); + + for step in 0..96usize { + let act_a = a.get_planned_action(); + let act_b = b.get_planned_action(); + assert_eq!( + act_a, act_b, + "{label} action mismatch at step {step} under equal seed/history" + ); + + let obs = [((step + 1) % 2) as u64]; + let rew = (step % 2) as i64; + a.observe_transition(act_a, &obs, rew) + .expect("transition should be accepted"); + b.observe_transition(act_b, &obs, rew) + .expect("transition should be accepted"); + } + } +} diff --git a/crates/infotheory/tests/aixi_discounting.rs b/crates/infotheory/tests/aixi_discounting.rs new file mode 100644 index 00000000..7a77e109 --- /dev/null +++ b/crates/infotheory/tests/aixi_discounting.rs @@ -0,0 +1,148 @@ +#![cfg(feature = "aixi")] + +use infotheory::aixi::common::{Action, ActionAlphabet, ObservationKeyMode, Reward}; +use infotheory::aixi::mcts::AgentSimulator; + +struct NormRewardHarness { + discount_gamma: f64, + horizon: usize, + min_reward: Reward, + max_reward: Reward, +} + +fn approx_eq(a: f64, b: f64, eps: f64) { + assert!( + (a - b).abs() <= eps, + "expected {a} ≈ {b} (|diff|={})", + (a - b).abs() + ); +} + +impl AgentSimulator for NormRewardHarness { + fn get_num_actions(&self) -> ActionAlphabet { + ActionAlphabet::try_from_usize(2).expect("test fixture action alphabet must be valid") + } + + fn get_num_observation_bits(&self) -> usize { + 1 + } + + fn observation_key_mode(&self) -> ObservationKeyMode { + ObservationKeyMode::FullStream + } + + fn get_num_reward_bits(&self) -> usize { + 8 + } + + fn horizon(&self) -> usize { + self.horizon + } + + fn max_reward(&self) -> Reward { + self.max_reward + } + + fn min_reward(&self) -> Reward { + self.min_reward + } + + fn reward_offset(&self) -> i64 { + (-self.min_reward).max(0) + } + + fn discount_gamma(&self) -> f64 { + self.discount_gamma + } + + fn model_update_action(&mut self, _action: Action) {} + + fn gen_percept_and_update(&mut self, _bits: usize) -> u64 { + 0 + } + + fn model_revert(&mut self, _steps: usize) {} + + fn gen_range(&mut self, _end: usize) -> usize { + 0 + } + + fn gen_f64(&mut self) -> f64 { + 0.0 + } + + fn boxed_clone_with_seed(&self, _seed: u64) -> Box { + Box::new(Self { + discount_gamma: self.discount_gamma, + horizon: self.horizon, + min_reward: self.min_reward, + max_reward: self.max_reward, + }) + } +} + +fn mk_agent( + discount_gamma: f64, + horizon: usize, + min_reward: i64, + max_reward: i64, +) -> NormRewardHarness { + NormRewardHarness { + discount_gamma, + horizon, + min_reward, + max_reward, + } +} + +#[test] +fn norm_reward_undiscounted_hits_endpoints() { + let horizon = 5; + let min = -2; + let max = 6; + let agent = mk_agent(1.0, horizon, min, max); + + let sum = horizon as f64; + let min_cum = (min as f64) * sum; + let max_cum = (max as f64) * sum; + + let z0 = agent.norm_reward(min_cum); + let z1 = agent.norm_reward(max_cum); + + approx_eq(z0, 0.0, 1e-12); + approx_eq(z1, 1.0, 1e-12); +} + +#[test] +fn norm_reward_discounted_hits_endpoints() { + let horizon = 10; + let min = -1; + let max = 3; + let gamma = 0.7; + let agent = mk_agent(gamma, horizon, min, max); + + let sum = (1.0 - gamma.powi(horizon as i32)) / (1.0 - gamma); + let min_cum = (min as f64) * sum; + let max_cum = (max as f64) * sum; + + let z0 = agent.norm_reward(min_cum); + let z1 = agent.norm_reward(max_cum); + + approx_eq(z0, 0.0, 1e-10); + approx_eq(z1, 1.0, 1e-10); +} + +#[test] +fn norm_reward_midpoint_is_half() { + let horizon = 7; + let min = -4; + let max = 4; + let gamma = 0.5; + let agent = mk_agent(gamma, horizon, min, max); + + let sum = (1.0 - gamma.powi(horizon as i32)) / (1.0 - gamma); + let mid_cum = ((min + max) as f64 / 2.0) * sum; + + let z = agent.norm_reward(mid_cum); + approx_eq(z, 0.5, 1e-10); +} diff --git a/crates/infotheory/tests/aixi_validation.rs b/crates/infotheory/tests/aixi_validation.rs new file mode 100644 index 00000000..61cb32cc --- /dev/null +++ b/crates/infotheory/tests/aixi_validation.rs @@ -0,0 +1,1057 @@ +#![cfg(all(feature = "aixi", feature = "all-backends"))] + +//! AIXI Module Validation Tests +//! +//! Tests for predictors, environments, and agents. + +use infotheory::aixi::agent::{Agent, AgentConfig, AgentError}; +use infotheory::aixi::common::{Action, ActionAlphabet, DEFAULT_RANDOM_SEED, ObservationKeyMode}; +use infotheory::aixi::environment::Environment; +mod support; +use infotheory::aixi::model::{ + CtwPredictor, Predictor, RateBackendBitPredictor, RateBackendBitPredictorConfig, RosaPredictor, +}; +use infotheory::api::{ + BitOrder, BitStreamSemantics, MAX_MIXTURE_NESTING, MixtureExpertSpec, MixtureKind, MixtureSpec, + RateBackend, +}; +use std::sync::Arc; +use support::aixi_envs::{DeterministicBinaryEnv, SeededCoinFlipEnv}; + +// ============================================================================ +// Predictor Consistency Tests +// ============================================================================ + +fn test_predictor_sum_to_one(mut predictor: Box, name: &str) { + // Feed some history + for &sym in &[true, false, true, true, false] { + predictor.update(sym); + } + + let p_true = predictor.predict_prob(true); + let p_false = predictor.predict_prob(false); + + // Check they sum to 1.0 (binary predictor) + let sum = p_true + p_false; + println!("{name}: P(1)={p_true:.6}, P(0)={p_false:.6}, Sum={sum:.6}"); + assert!( + (sum - 1.0).abs() < 1e-6, + "{name}: Probabilities must sum to 1.0, got {p_true} + {p_false} = {sum}" + ); + + // Check range + assert!( + (0.0..=1.0).contains(&p_true), + "{name}: Prob out of range: {p_true}" + ); +} + +#[test] +fn ctw_probabilities_valid() { + test_predictor_sum_to_one(Box::new(CtwPredictor::new(8)), "CTW"); +} + +#[test] +fn rosa_probabilities_valid() { + test_predictor_sum_to_one(Box::new(RosaPredictor::new(8)), "ROSA"); +} + +fn test_predictor_revert(mut predictor: Box, name: &str) { + let history = [true, false, true, true, false, false, true]; + + // Update all + for &sym in &history { + predictor.update(sym); + } + let prob_after_updates = predictor.predict_prob(true); + + // Revert all + for _ in &history { + predictor.revert(); + } + + // Should be back to initial state (approx 0.5 for uniform prior) + let prob_reverted = predictor.predict_prob(true); + + println!("{name}: After full revert, p(1) = {prob_reverted}"); + assert!( + (prob_reverted - 0.5).abs() < 0.1, + "{name}: Reverted predictor should be roughly uninformed (0.5), got {prob_reverted}" + ); + + // Re-apply and check we get same result as before + for &sym in &history { + predictor.update(sym); + } + let prob_redo = predictor.predict_prob(true); + assert!( + (prob_redo - prob_after_updates).abs() < 1e-9, + "{name}: Deterministic replay failed. {prob_redo} != {prob_after_updates}" + ); +} + +#[test] +fn ctw_update_revert_consistency() { + test_predictor_revert(Box::new(CtwPredictor::new(8)), "CTW"); +} + +#[test] +fn rosa_update_revert_consistency() { + test_predictor_revert(Box::new(RosaPredictor::new(8)), "ROSA"); +} + +#[test] +fn default_agent_config_validates() { + AgentConfig::default() + .validate() + .expect("default MC-AIXI config should satisfy its own contract"); +} + +fn nested_generic_backend() -> RateBackend { + let inner = MixtureSpec::new( + MixtureKind::Bayes, + vec![ + { + let mut expert = MixtureExpertSpec::new(RateBackend::Ctw { depth: 6 }); + expert.name = Some("ctw".to_string()); + expert.log_prior = 0.0; + expert + }, + { + let mut expert = MixtureExpertSpec::new(RateBackend::Match { + hash_bits: 18, + min_len: 2, + max_len: 32, + base_mix: 0.05, + confidence_scale: 1.0, + }); + expert.name = Some("match".to_string()); + expert.log_prior = 0.0; + expert + }, + ], + ) + .with_alpha(0.03); + let outer = MixtureSpec::new( + MixtureKind::Convex, + vec![ + { + let mut expert = MixtureExpertSpec::new(RateBackend::Mixture { + spec: Arc::new(inner), + }); + expert.name = Some("nested".to_string()); + expert.log_prior = 0.0; + expert + }, + { + let mut expert = MixtureExpertSpec::new(RateBackend::Ppmd { + order: 4, + memory_mb: 8, + }); + expert.name = Some("ppmd".to_string()); + expert.log_prior = 0.0; + expert + }, + ], + ) + .with_alpha(1.25); + RateBackend::Mixture { + spec: Arc::new(outer), + } +} + +fn predictor_snapshot(predictor: &mut dyn Predictor) -> (f64, f64) { + (predictor.predict_prob(false), predictor.predict_prob(true)) +} + +fn assert_snapshot_eq(actual: (f64, f64), expected: (f64, f64), label: &str) { + assert!( + (actual.0 - expected.0).abs() < 1e-12 && (actual.1 - expected.1).abs() < 1e-12, + "{label}: expected {:?}, got {:?}", + expected, + actual + ); +} + +#[test] +fn rate_backend_bit_predictor_roundtrips_nested_mixtures() { + let config = RateBackendBitPredictorConfig::compile_with_semantics( + nested_generic_backend(), + 1e-12, + BitStreamSemantics::BinaryTokens, + ) + .expect("config"); + let mut predictor = RateBackendBitPredictor::new(config).expect("valid predictor"); + + let initial = predictor_snapshot(&mut predictor); + + predictor.update(true); + let after_update = predictor_snapshot(&mut predictor); + predictor.revert(); + assert_snapshot_eq( + predictor_snapshot(&mut predictor), + initial, + "revert after update", + ); + + predictor.update(true); + assert_snapshot_eq( + predictor_snapshot(&mut predictor), + after_update, + "redo after update", + ); + + predictor.update_history(false); + let after_frozen = predictor_snapshot(&mut predictor); + predictor.pop_history(); + assert_snapshot_eq( + predictor_snapshot(&mut predictor), + after_update, + "pop_history after frozen update", + ); + + predictor.update_history(false); + assert_snapshot_eq( + predictor_snapshot(&mut predictor), + after_frozen, + "redo after frozen update", + ); +} + +#[test] +fn rate_backend_bit_predictor_roundtrips_sequitur_backend() { + let config = RateBackendBitPredictorConfig::compile_with_semantics( + RateBackend::Sequitur { context_bytes: 32 }, + 1e-12, + BitStreamSemantics::BinaryTokens, + ) + .expect("config"); + let mut predictor = RateBackendBitPredictor::new(config).expect("valid sequitur predictor"); + + let initial = predictor_snapshot(&mut predictor); + + predictor.update(true); + let after_update = predictor_snapshot(&mut predictor); + predictor.revert(); + assert_snapshot_eq( + predictor_snapshot(&mut predictor), + initial, + "sequitur revert after update", + ); + + predictor.update(true); + assert_snapshot_eq( + predictor_snapshot(&mut predictor), + after_update, + "sequitur redo after update", + ); + + predictor.update_history(false); + let after_frozen = predictor_snapshot(&mut predictor); + predictor.pop_history(); + assert_snapshot_eq( + predictor_snapshot(&mut predictor), + after_update, + "sequitur pop_history after frozen update", + ); + + predictor.update_history(false); + assert_snapshot_eq( + predictor_snapshot(&mut predictor), + after_frozen, + "sequitur redo after frozen update", + ); +} + +#[test] +fn rate_backend_bit_predictor_respects_custom_min_prob_floor() { + let config = RateBackendBitPredictorConfig::compile_with_semantics( + RateBackend::Ctw { depth: 8 }, + 0.49, + BitStreamSemantics::BinaryTokens, + ) + .expect("config"); + let mut predictor = RateBackendBitPredictor::new(config).expect("valid predictor"); + + for _ in 0..64 { + predictor.update(true); + } + + let p1 = predictor.predict_prob(true); + let p0 = predictor.predict_prob(false); + assert!( + (0.49..=0.51).contains(&p1), + "custom min_prob floor must clamp P(1); got {p1}" + ); + assert!( + (0.49..=0.51).contains(&p0), + "custom min_prob floor must clamp P(0); got {p0}" + ); +} + +#[test] +fn rate_backend_bit_predictor_bytepacked_ctw_rewinds_adversarial_prefixes() { + let config = RateBackendBitPredictorConfig::compile_with_semantics( + RateBackend::Ctw { depth: 8 }, + 1e-12, + BitStreamSemantics::BytePacked { + order: BitOrder::MsbFirst, + }, + ) + .expect("byte-packed CTW config"); + let mut predictor = RateBackendBitPredictor::new(config).expect("byte-packed CTW predictor"); + + for &bit in &[ + true, false, true, false, false, true, true, false, false, true, false, true, true, false, + false, true, + ] { + predictor.commit_update(bit); + } + + let baseline = predictor_snapshot(&mut predictor); + + for &bit in &[true, false, true, true, false, false, true] { + predictor.update_history(bit); + } + let after_partial_frozen_prefix = predictor_snapshot(&mut predictor); + for _ in 0..7 { + predictor.pop_history(); + } + assert_snapshot_eq( + predictor_snapshot(&mut predictor), + baseline, + "byte-packed frozen partial-byte rewind", + ); + + for &bit in &[false, true, false, false, true] { + predictor.update(bit); + } + let after_partial_adaptive_prefix = predictor_snapshot(&mut predictor); + for _ in 0..5 { + predictor.revert(); + } + assert_snapshot_eq( + predictor_snapshot(&mut predictor), + baseline, + "byte-packed adaptive partial-byte rewind", + ); + + assert!( + (after_partial_frozen_prefix.0 - baseline.0).abs() > 1e-15 + || (after_partial_frozen_prefix.1 - baseline.1).abs() > 1e-15, + "adversarial frozen prefix should affect the prediction before rewind" + ); + assert!( + (after_partial_adaptive_prefix.0 - baseline.0).abs() > 1e-15 + || (after_partial_adaptive_prefix.1 - baseline.1).abs() > 1e-15, + "adversarial adaptive prefix should affect the prediction before rewind" + ); + + predictor.begin_rollback_scope(); + for &bit in &[true, false, false, true, true, false, true, false] { + predictor.update_history(bit); + } + for &bit in &[false, true, true, false, false, true, false, true] { + predictor.update(bit); + } + assert!( + predictor.rollback_scope(), + "byte-packed simulation scope should restore through one frozen action byte and one adaptive percept byte", + ); + assert_snapshot_eq( + predictor_snapshot(&mut predictor), + baseline, + "byte-packed scoped planner rollback", + ); +} + +// ============================================================================ +// Environment Tests +// ============================================================================ + +#[test] +fn ctw_test_env_is_deterministic() { + let mut env1 = DeterministicBinaryEnv::new(); + let mut env2 = DeterministicBinaryEnv::new(); + + for i in 0..50 { + let action = (i % 2) as Action; + env1.perform_action(action); + env2.perform_action(action); + + assert_eq!( + env1.get_observation(), + env2.get_observation(), + "Obs mismatch at step {i}" + ); + assert_eq!( + env1.get_reward(), + env2.get_reward(), + "Reward mismatch at step {i}" + ); + } +} + +// ============================================================================ +// Agent / MCTS Tests +// ============================================================================ + +fn run_agent_env(agent: &mut Agent, mut env: T, cycles: usize) -> f64 { + let mut total_reward = 0.0; + let mut obs_stream = env.drain_observations(); + let mut prev_rew = env.get_reward(); + let mut prev_act = 0; + + for _ in 0..cycles { + agent.model_update_percept_stream(&obs_stream, prev_rew); + let action = agent.get_planned_action(&obs_stream, prev_rew, prev_act); + + // Update model with chosen action (so model sees: ...p a p a p a...) + agent.model_update_action_external(action); + + env.perform_action(action); + + obs_stream = env.drain_observations(); + let rew = env.get_reward(); + + // Update model with observed percept stream + agent.model_update_percept_stream(&obs_stream, rew); + + total_reward += rew as f64; + prev_rew = rew; + prev_act = action; + + if env.is_finished() { + break; + } + } + total_reward +} + +fn agent_action_trace_on_deterministic_env(mut agent: Agent, steps: usize) -> Vec { + let mut env = DeterministicBinaryEnv::new(); + let mut obs_stream = env.drain_observations(); + let mut prev_rew = env.get_reward(); + let mut prev_act = 0u64; + let mut actions = Vec::with_capacity(steps); + + for _ in 0..steps { + agent.model_update_percept_stream(&obs_stream, prev_rew); + let action = agent.get_planned_action(&obs_stream, prev_rew, prev_act); + actions.push(action); + + agent.model_update_action_external(action); + env.perform_action(action); + obs_stream = env.drain_observations(); + prev_rew = env.get_reward(); + prev_act = action; + } + + actions +} + +fn generic_agent_config(rate_backend: RateBackend) -> AgentConfig { + let mut cfg = AgentConfig::default(); + cfg.rate_backend = rate_backend; + cfg.bit_stream_semantics = infotheory::api::BitStreamSemantics::BinaryTokens; + cfg.agent_horizon = 5; + cfg.observation_bits = 1; + cfg.observation_stream_len = 1; + cfg.observation_key_mode = ObservationKeyMode::FullStream; + cfg.reward_bits = 1; + cfg.agent_actions = + ActionAlphabet::try_from_usize(2).expect("test fixture action alphabet must be valid"); + cfg.num_simulations = 60; + cfg.exploration_exploitation_ratio = 1.4; + cfg.discount_gamma = 1.0; + cfg.min_reward = 0; + cfg.max_reward = 1; + cfg.reward_offset = 0; + cfg.random_seed = Some(2026); + cfg +} + +fn mixture_backend(kind: MixtureKind) -> RateBackend { + let experts = vec![ + { + let mut expert = MixtureExpertSpec::new(RateBackend::Ctw { depth: 8 }); + expert.name = Some("ctw".to_string()); + expert.log_prior = 0.0; + expert + }, + { + let mut expert = MixtureExpertSpec::new(RateBackend::RosaPlus { max_order: 8 }); + expert.name = Some("rosa".to_string()); + expert.log_prior = 0.0; + expert + }, + ]; + let alpha = match kind { + MixtureKind::Switching => 0.05, + MixtureKind::Convex => 1.25, + _ => 0.03, + }; + RateBackend::Mixture { + spec: Arc::new(MixtureSpec::new(kind, experts).with_alpha(alpha)), + } +} + +fn deeply_nested_bayes_backend(depth: usize) -> RateBackend { + let mut backend = RateBackend::Ctw { depth: 4 }; + for level in 0..depth { + backend = RateBackend::Mixture { + spec: Arc::new(MixtureSpec::new( + MixtureKind::Bayes, + vec![{ + let mut expert = MixtureExpertSpec::new(backend); + expert.name = Some(format!("level-{level}")); + expert.log_prior = 0.0; + expert + }], + )), + }; + } + backend +} + +#[test] +fn agent_solves_ctw_test_environment() { + let mut config = AgentConfig::default(); + config.bit_stream_semantics = infotheory::api::BitStreamSemantics::BinaryTokens; + config.rate_backend = RateBackend::FacCtw { + base_depth: 8, + num_percept_bits: 2, + encoding_bits: 1, + msb_first: None, + }; + config.agent_horizon = 8; + config.observation_bits = 1; + config.observation_stream_len = 1; + config.observation_key_mode = infotheory::aixi::common::ObservationKeyMode::FullStream; + config.reward_bits = 1; + config.agent_actions = + ActionAlphabet::try_from_usize(2).expect("test fixture action alphabet must be valid"); + config.num_simulations = 200; + config.exploration_exploitation_ratio = 2.0; + config.discount_gamma = 1.0; + config.min_reward = 0; + config.max_reward = 1; + config.reward_offset = 0; + config.random_seed = Some(17); + + let mut agent = Agent::new(config); + let env = DeterministicBinaryEnv::new(); + + let cycles = 100; + let total_reward = run_agent_env(&mut agent, env, cycles); + + println!( + "Agent Total Reward on DeterministicBinaryEnv (100 cycles): {}", + total_reward + ); + + // Agent should learn pattern and get reasonable reward + assert!( + total_reward > 50.0, + "Agent failed to learn DeterministicBinaryEnv pattern. Reward: {total_reward}" + ); +} + +#[test] +fn agent_regret_sublinear_coinflip() { + let mut config = AgentConfig::default(); + config.bit_stream_semantics = infotheory::api::BitStreamSemantics::BinaryTokens; + config.rate_backend = RateBackend::FacCtw { + base_depth: 4, + num_percept_bits: 2, + encoding_bits: 1, + msb_first: None, + }; + config.agent_horizon = 4; + config.observation_bits = 1; + config.observation_stream_len = 1; + config.observation_key_mode = infotheory::aixi::common::ObservationKeyMode::FullStream; + config.reward_bits = 1; + config.agent_actions = + ActionAlphabet::try_from_usize(2).expect("test fixture action alphabet must be valid"); + config.num_simulations = 100; + config.exploration_exploitation_ratio = 1.0; + config.discount_gamma = 1.0; + config.min_reward = 0; + config.max_reward = 1; + config.reward_offset = 0; + config.random_seed = Some(23); + + let mut agent = Agent::new(config); + let env = SeededCoinFlipEnv::new(0.8); + + let cycles = 500; + let total_reward = run_agent_env(&mut agent, env, cycles); + + let expected_optimal = 0.8 * cycles as f64; + let regret = expected_optimal - total_reward; + let regret_per_step = regret / cycles as f64; + + println!( + "SeededCoinFlipEnv(0.8): Reward={total_reward}, Opt={expected_optimal}, Regret/step={regret_per_step:.4}" + ); + + // Regret should be reasonable (< 0.25 per step) + assert!(regret_per_step < 0.25, "Regret too high: {regret_per_step}"); +} + +#[test] +fn agent_seeded_policy_is_reproducible_on_deterministic_env() { + let mut config = AgentConfig::default(); + config.bit_stream_semantics = infotheory::api::BitStreamSemantics::BinaryTokens; + config.rate_backend = RateBackend::FacCtw { + base_depth: 8, + num_percept_bits: 2, + encoding_bits: 1, + msb_first: None, + }; + config.agent_horizon = 6; + config.observation_bits = 1; + config.observation_stream_len = 1; + config.observation_key_mode = infotheory::aixi::common::ObservationKeyMode::FullStream; + config.reward_bits = 1; + config.agent_actions = + ActionAlphabet::try_from_usize(2).expect("test fixture action alphabet must be valid"); + config.num_simulations = 80; + config.exploration_exploitation_ratio = 1.4; + config.discount_gamma = 1.0; + config.min_reward = 0; + config.max_reward = 1; + config.reward_offset = 0; + config.random_seed = Some(12345); + + let mut a = Agent::new(config.clone()); + let mut b = Agent::new(config); + let mut env_a = DeterministicBinaryEnv::new(); + let mut env_b = DeterministicBinaryEnv::new(); + + let mut obs_a = env_a.drain_observations(); + let mut obs_b = env_b.drain_observations(); + let mut rew_a = env_a.get_reward(); + let mut rew_b = env_b.get_reward(); + let mut prev_a = 0u64; + let mut prev_b = 0u64; + + for step in 0..64usize { + assert_eq!(obs_a, obs_b, "observation mismatch at step {step}"); + assert_eq!(rew_a, rew_b, "reward mismatch at step {step}"); + + a.model_update_percept_stream(&obs_a, rew_a); + b.model_update_percept_stream(&obs_b, rew_b); + + let act_a = a.get_planned_action(&obs_a, rew_a, prev_a); + let act_b = b.get_planned_action(&obs_b, rew_b, prev_b); + assert_eq!(act_a, act_b, "action mismatch at step {step}"); + + a.model_update_action_external(act_a); + b.model_update_action_external(act_b); + + env_a.perform_action(act_a); + env_b.perform_action(act_b); + obs_a = env_a.drain_observations(); + obs_b = env_b.drain_observations(); + rew_a = env_a.get_reward(); + rew_b = env_b.get_reward(); + prev_a = act_a; + prev_b = act_b; + } +} + +#[test] +fn agent_omitted_seed_matches_explicit_default_seed() { + let mut cfg_omitted = generic_agent_config(RateBackend::Ctw { depth: 8 }); + cfg_omitted.random_seed = None; + let mut cfg_explicit = cfg_omitted.clone(); + cfg_explicit.random_seed = Some(DEFAULT_RANDOM_SEED); + + let mut a = Agent::try_new(cfg_omitted).expect("agent with omitted seed"); + let mut b = Agent::try_new(cfg_explicit).expect("agent with explicit default seed"); + + assert_eq!(a.resolved_random_seed(), DEFAULT_RANDOM_SEED); + assert_eq!(b.resolved_random_seed(), DEFAULT_RANDOM_SEED); + + let mut env_a = DeterministicBinaryEnv::new(); + let mut env_b = DeterministicBinaryEnv::new(); + + let mut obs_a = env_a.drain_observations(); + let mut obs_b = env_b.drain_observations(); + let mut rew_a = env_a.get_reward(); + let mut rew_b = env_b.get_reward(); + let mut prev_a = 0u64; + let mut prev_b = 0u64; + + for step in 0..64usize { + a.model_update_percept_stream(&obs_a, rew_a); + b.model_update_percept_stream(&obs_b, rew_b); + + let act_a = a.get_planned_action(&obs_a, rew_a, prev_a); + let act_b = b.get_planned_action(&obs_b, rew_b, prev_b); + assert_eq!(act_a, act_b, "action mismatch at step {step}"); + + a.model_update_action_external(act_a); + b.model_update_action_external(act_b); + + env_a.perform_action(act_a); + env_b.perform_action(act_b); + obs_a = env_a.drain_observations(); + obs_b = env_b.drain_observations(); + rew_a = env_a.get_reward(); + rew_b = env_b.get_reward(); + prev_a = act_a; + prev_b = act_b; + } +} + +#[test] +fn agent_different_seeds_can_change_stochastic_trace() { + let mut seed_pair = None; + for left in 1u64..256 { + let left_draw = infotheory::aixi::common::RandomGenerator::from_seed(left).gen_range(2); + for right in (left + 1)..256 { + let right_draw = + infotheory::aixi::common::RandomGenerator::from_seed(right).gen_range(2); + if left_draw != right_draw { + seed_pair = Some((left, right, left_draw, right_draw)); + break; + } + } + if seed_pair.is_some() { + break; + } + } + let (left_seed, right_seed, left_draw, right_draw) = + seed_pair.expect("expected to find a seed pair with different first draws"); + assert_ne!(left_draw, right_draw); + + let mut left_rng = infotheory::aixi::common::RandomGenerator::from_seed(left_seed); + let mut right_rng = infotheory::aixi::common::RandomGenerator::from_seed(right_seed); + assert_ne!(left_rng.gen_range(2), right_rng.gen_range(2)); +} + +#[test] +fn agent_config_accepts_explicit_programmatic_rate_backend() { + let cfg = generic_agent_config(RateBackend::Ppmd { + order: 4, + memory_mb: 8, + }); + assert!(cfg.validate().is_ok()); + let mut agent = Agent::try_new(cfg).expect("explicit rate_backend should be valid"); + let action = agent.get_planned_action(&[0], 0, 0); + assert!(action < 2); +} + +#[test] +fn agent_config_rejects_zpaq_rate_backend_in_strict_mode() { + let cfg = generic_agent_config(RateBackend::Mixture { + spec: Arc::new(MixtureSpec::new( + MixtureKind::Bayes, + vec![{ + let mut expert = MixtureExpertSpec::new(RateBackend::Zpaq { + method: infotheory::api::ZpaqMethodSpec::literal("1"), + }); + expert.name = Some("bad-zpaq".to_string()); + expert.log_prior = 0.0; + expert + }], + )), + }); + let err = cfg + .validate() + .expect_err("zpaq-backed generic MC-AIXI should be rejected"); + assert!(matches!(err, AgentError::UnsupportedRateBackend { .. })); +} + +#[test] +fn agent_config_rejects_invalid_programmatic_mixture_rate_backend() { + let cfg = generic_agent_config(RateBackend::Mixture { + spec: Arc::new(MixtureSpec::new(MixtureKind::Bayes, vec![])), + }); + let err = cfg + .validate() + .expect_err("empty mixture backend should be rejected"); + assert!(matches!(err, AgentError::InvalidRateBackend(_))); +} + +#[test] +fn agent_config_rejects_programmatic_mixture_nesting_overflow() { + let cfg = generic_agent_config(deeply_nested_bayes_backend(MAX_MIXTURE_NESTING + 1)); + let err = cfg + .validate() + .expect_err("overly deep nested mixture should be rejected"); + assert!(matches!(err, AgentError::InvalidRateBackend(_))); +} + +#[test] +fn agent_with_generic_mixture_backends_smoke_runs() { + for (kind, label) in [ + (MixtureKind::Bayes, "bayes"), + (MixtureKind::Switching, "switching"), + (MixtureKind::Convex, "convex"), + ] { + let mut agent = + Agent::try_new(generic_agent_config(mixture_backend(kind))).expect("valid mixture"); + let total_reward = run_agent_env(&mut agent, DeterministicBinaryEnv::new(), 48); + assert!( + total_reward > 16.0, + "{label} mixture backend reward too low on DeterministicBinaryEnv: {total_reward}" + ); + } +} + +#[test] +fn agent_solves_deterministic_env_with_generic_ctw_backend() { + let mut cfg = generic_agent_config(RateBackend::Ctw { depth: 10 }); + cfg.agent_horizon = 8; + cfg.num_simulations = 140; + + let mut agent = Agent::try_new(cfg).expect("valid generic CTW configuration"); + let total_reward = run_agent_env(&mut agent, DeterministicBinaryEnv::new(), 120); + + assert!( + total_reward > 70.0, + "generic CTW MC-AIXI should learn DeterministicBinaryEnv, got total_reward={total_reward}" + ); +} + +#[test] +fn agent_solves_deterministic_env_with_generic_fac_ctw_backend() { + let mut cfg = generic_agent_config(RateBackend::FacCtw { + base_depth: 10, + num_percept_bits: 2, + encoding_bits: 1, + msb_first: None, + }); + cfg.agent_horizon = 8; + cfg.num_simulations = 140; + + let mut agent = Agent::try_new(cfg).expect("valid generic FAC-CTW configuration"); + let total_reward = run_agent_env(&mut agent, DeterministicBinaryEnv::new(), 120); + + assert!( + total_reward > 70.0, + "generic FAC-CTW MC-AIXI should learn DeterministicBinaryEnv, got total_reward={total_reward}" + ); +} + +#[test] +fn generic_mixture_world_models_are_seed_deterministic() { + for (kind, label) in [ + (MixtureKind::Bayes, "bayes"), + (MixtureKind::Switching, "switching"), + (MixtureKind::Convex, "convex"), + ] { + let mut cfg = generic_agent_config(mixture_backend(kind)); + cfg.random_seed = Some(88172645463393265); + cfg.num_simulations = 80; + cfg.agent_horizon = 6; + + let trace_a = agent_action_trace_on_deterministic_env( + Agent::try_new(cfg.clone()).expect("valid mixture agent A"), + 64, + ); + let trace_b = agent_action_trace_on_deterministic_env( + Agent::try_new(cfg).expect("valid mixture agent B"), + 64, + ); + assert_eq!( + trace_a, trace_b, + "{label} mixture world model should be deterministic under identical seed and history" + ); + } +} + +/// Exercises `BitStreamSemantics::BytePacked` with native-capable CTW inside a +/// real planner/percept loop, proving that byte-packed bit sessions work beyond +/// the public API surface tests. +#[test] +fn agent_bytepacked_ctw_native_planner_integration() { + let mut config = AgentConfig::default(); + // BytePacked (the key new surface under test) with byte-aligned action and + // percept segments. The 3-bit observation plus 5-bit reward intentionally + // share a single byte, matching the validator's actual segment contract. + config.bit_stream_semantics = infotheory::api::BitStreamSemantics::BytePacked { + order: infotheory::api::BitOrder::MsbFirst, + }; + // Native-capable Ctw (full MSB byte-prefix support + reversible) — the "native-capable + // backend" required by the criteria. + config.rate_backend = RateBackend::Ctw { depth: 8 }; + config.agent_horizon = 4; + config.observation_bits = 3; + config.observation_stream_len = 1; + config.observation_key_mode = infotheory::aixi::common::ObservationKeyMode::FullStream; + config.reward_bits = 5; + config.agent_actions = + ActionAlphabet::try_from_usize(129).expect("test fixture action alphabet must be valid"); + config.num_simulations = 50; // modest budget that still exercises the planner loop many times; byte-aligned 129-action exploration makes consistent positive reward noisy (see assertion comment) + config.exploration_exploitation_ratio = 1.0; + config.discount_gamma = 0.95; + config.min_reward = 0; + config.max_reward = 31; + config.reward_offset = 0; + config.random_seed = Some(42); + + let cycles = 20; // enough steps to drive multiple percept updates + planner decisions + let total_reward = run_agent_env( + &mut Agent::new(config.clone()), + DeterministicBinaryEnv::new(), + cycles, + ); + let total_reward_replay = run_agent_env( + &mut Agent::new(config.clone()), + DeterministicBinaryEnv::new(), + cycles, + ); + let trace_a = agent_action_trace_on_deterministic_env(Agent::new(config.clone()), cycles); + let trace_b = agent_action_trace_on_deterministic_env(Agent::new(config), cycles); + + println!( + "Agent Total Reward on DeterministicBinaryEnv (BytePacked + native Ctw, {} cycles): {}", + cycles, total_reward + ); + + // Strong contract check: fixed seed + deterministic environment must produce + // identical reward and action traces across repeated full planner runs. + assert_eq!( + total_reward, total_reward_replay, + "BytePacked + native Ctw planner reward must be deterministic under identical seed" + ); + assert_eq!( + trace_a, trace_b, + "BytePacked + native Ctw planner action trace must be deterministic under identical seed" + ); + assert!( + total_reward.is_finite(), + "BytePacked + native Ctw planner reward must remain finite. Reward: {total_reward}" + ); +} + +/// Mirrors the native CTW byte-packed planner integration test but with +/// generic FAC-CTW to cover the byte-packed + FAC-CTW planner path directly. +#[test] +fn agent_bytepacked_fac_ctw_planner_integration() { + let mut config = AgentConfig::default(); + config.bit_stream_semantics = BitStreamSemantics::BytePacked { + order: BitOrder::MsbFirst, + }; + config.rate_backend = RateBackend::FacCtw { + base_depth: 8, + // 3-bit observation + 5-bit reward share one percept byte in this setup. + num_percept_bits: 8, + encoding_bits: 1, + msb_first: Some(true), + }; + config.agent_horizon = 4; + config.observation_bits = 3; + config.observation_stream_len = 1; + config.observation_key_mode = ObservationKeyMode::FullStream; + config.reward_bits = 5; + config.agent_actions = + ActionAlphabet::try_from_usize(129).expect("test fixture action alphabet must be valid"); + config.num_simulations = 60; + config.exploration_exploitation_ratio = 1.0; + config.discount_gamma = 0.95; + config.min_reward = 0; + config.max_reward = 31; + config.reward_offset = 0; + config.random_seed = Some(1337); + + let cycles = 20; + let total_reward = run_agent_env( + &mut Agent::new(config.clone()), + DeterministicBinaryEnv::new(), + cycles, + ); + let total_reward_replay = run_agent_env( + &mut Agent::new(config.clone()), + DeterministicBinaryEnv::new(), + cycles, + ); + let trace_a = agent_action_trace_on_deterministic_env(Agent::new(config.clone()), cycles); + let trace_b = agent_action_trace_on_deterministic_env(Agent::new(config), cycles); + + assert_eq!( + total_reward, total_reward_replay, + "BytePacked + FAC-CTW planner reward must be deterministic under identical seed" + ); + assert_eq!( + trace_a, trace_b, + "BytePacked + FAC-CTW planner action trace must be deterministic under identical seed" + ); + assert!( + total_reward.is_finite(), + "BytePacked + FAC-CTW planner reward must remain finite. Reward: {total_reward}" + ); +} + +/// `BitStreamSemantics::BinaryTokens` with canonical 8-bit MSB FacCtw exercises the +/// planner [`FacCtwPredictor`] fast path (per-bit lanes via `percept_bits`), not the +/// byte-level `RateBackendPredictor` MSB byte-prefix machinery. +#[test] +fn agent_binarytokens_fac_ctw_native_planner_integration() { + let compiled = RateBackend::FacCtw { + base_depth: 8, + num_percept_bits: 8, + encoding_bits: 8, + msb_first: Some(true), + } + .compile() + .expect("compile fac-ctw backend"); + let caps = compiled.capabilities(); + assert!(caps.supports_native_bit_prediction); + assert!(caps.supports_reversible_bit_updates); + + let mut config = AgentConfig::default(); + config.bit_stream_semantics = BitStreamSemantics::BinaryTokens; + config.rate_backend = RateBackend::FacCtw { + base_depth: 8, + num_percept_bits: 8, + encoding_bits: 8, + msb_first: Some(true), + }; + config.agent_horizon = 4; + config.observation_bits = 3; + config.observation_stream_len = 1; + config.observation_key_mode = ObservationKeyMode::FullStream; + config.reward_bits = 5; + config.agent_actions = + ActionAlphabet::try_from_usize(129).expect("129 actions require one byte of action bits"); + config.num_simulations = 50; + config.exploration_exploitation_ratio = 1.0; + config.discount_gamma = 0.95; + config.min_reward = 0; + config.max_reward = 31; + config.reward_offset = 0; + config.random_seed = Some(4242); + + let cycles = 20; + let total_reward = run_agent_env( + &mut Agent::new(config.clone()), + DeterministicBinaryEnv::new(), + cycles, + ); + let total_reward_replay = run_agent_env( + &mut Agent::new(config.clone()), + DeterministicBinaryEnv::new(), + cycles, + ); + let trace_a = agent_action_trace_on_deterministic_env(Agent::new(config.clone()), cycles); + let trace_b = agent_action_trace_on_deterministic_env(Agent::new(config), cycles); + + assert_eq!( + total_reward, total_reward_replay, + "BinaryTokens + FAC-CTW planner reward must be deterministic under identical seed" + ); + assert_eq!( + trace_a, trace_b, + "BinaryTokens + FAC-CTW planner action trace must be deterministic under identical seed" + ); + assert!( + total_reward.is_finite(), + "BinaryTokens + FAC-CTW planner reward must remain finite. Reward: {total_reward}" + ); +} diff --git a/crates/infotheory/tests/api_surface.rs b/crates/infotheory/tests/api_surface.rs new file mode 100644 index 00000000..b4898ae1 --- /dev/null +++ b/crates/infotheory/tests/api_surface.rs @@ -0,0 +1,1367 @@ +#[cfg(feature = "backend-ctw")] +use infotheory::api::BinaryPrediction; +#[cfg(any(feature = "backend-ctw", feature = "backend-zpaq"))] +use infotheory::api::BitOrder; +#[cfg(all(feature = "backend-mixture", feature = "backend-ctw"))] +use infotheory::api::MixtureExpertSpec; +#[cfg(any(feature = "backend-ctw", feature = "backend-zpaq"))] +use infotheory::api::OnlineBitPredictor; +#[cfg(feature = "backend-particle")] +use infotheory::api::ParticleSpec; +#[cfg(any( + feature = "backend-ctw", + feature = "backend-zpaq", + feature = "backend-match" +))] +use infotheory::api::{BitStreamSemantics, RateBackendBitSession}; +#[cfg(feature = "backend-calibrated")] +use infotheory::api::{CalibratedSpec, CalibrationContextKind}; +#[cfg(feature = "backend-zpaq")] +use infotheory::api::{CompressionBackend, try_compress_bytes_backend}; +use infotheory::api::{MixtureKind, MixtureSpec, RateBackend, RateBackendSession}; +#[cfg(any( + feature = "backend-ctw", + feature = "backend-particle", + all(feature = "backend-mixture", feature = "backend-ctw") +))] +use infotheory::spec::CanonicalJson; +use std::sync::Arc; + +#[cfg(feature = "backend-ctw")] +const CHECKPOINT_RESTORE_PREDICTION_ULPS: u64 = 4; + +#[cfg(feature = "backend-ctw")] +fn ordered_f64_bits(value: f64) -> u64 { + let bits: u64 = value.to_bits(); + if bits & (1u64 << 63) == 0 { + bits | (1u64 << 63) + } else { + !bits + } +} + +#[cfg(feature = "backend-ctw")] +fn assert_f64_within_ulps(label: &str, actual: f64, expected: f64, max_ulps: u64) { + assert!( + actual.is_finite() && expected.is_finite(), + "{label} must be finite: actual={actual}, expected={expected}" + ); + let ulps: u64 = ordered_f64_bits(actual).abs_diff(ordered_f64_bits(expected)); + assert!( + ulps <= max_ulps, + "{label} differs by {ulps} ULPs, expected at most {max_ulps}: actual={actual}, expected={expected}" + ); +} + +#[cfg(feature = "backend-ctw")] +fn assert_valid_binary_prediction(label: &str, prediction: BinaryPrediction) { + assert!( + prediction.p0.is_finite() && prediction.p1.is_finite(), + "{label} probabilities must be finite: prediction={prediction:?}" + ); + assert!( + (0.0..=1.0).contains(&prediction.p0) && (0.0..=1.0).contains(&prediction.p1), + "{label} probabilities must stay in [0, 1]: prediction={prediction:?}" + ); + assert_f64_within_ulps( + &format!("{label} normalization"), + prediction.p0 + prediction.p1, + 1.0, + CHECKPOINT_RESTORE_PREDICTION_ULPS, + ); +} + +#[cfg(feature = "backend-ctw")] +fn assert_checkpoint_prediction_within_roundoff( + label: &str, + actual: BinaryPrediction, + expected: BinaryPrediction, +) { + // Checkpoint restore reuses the same prediction code after replaying a short + // reversible journal; this bound admits only tiny inverse-update round-off. + assert_valid_binary_prediction(&format!("{label} actual"), actual); + assert_valid_binary_prediction(&format!("{label} expected"), expected); + assert_f64_within_ulps( + &format!("{label} p0"), + actual.p0, + expected.p0, + CHECKPOINT_RESTORE_PREDICTION_ULPS, + ); + assert_f64_within_ulps( + &format!("{label} p1"), + actual.p1, + expected.p1, + CHECKPOINT_RESTORE_PREDICTION_ULPS, + ); +} + +#[test] +fn api_surface_rate_backend_session_rejects_invalid_programmatic_mixture() { + let backend = RateBackend::Mixture { + spec: Arc::new(MixtureSpec::new(MixtureKind::Bayes, vec![])), + }; + let err = match RateBackendSession::from_spec(backend, None) { + Ok(_) => panic!("invalid mixture backend should be rejected before runtime construction"), + Err(err) => err, + }; + let message = err.to_string(); + if cfg!(feature = "backend-mixture") { + assert!(message.contains("must include at least one expert")); + } else { + assert!(message.contains("requires infotheory feature 'backend-mixture'")); + } +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn api_surface_rate_backend_serializes_canonically() { + let backend = RateBackend::Ctw { depth: 9 }; + let backend_json = backend.to_canonical_json().expect("backend json"); + assert!(backend_json.contains("\"kind\": \"ctw\"")); +} + +#[cfg(all(feature = "backend-mixture", feature = "backend-ctw"))] +#[test] +fn api_surface_mixture_spec_serializes_canonically() { + let backend = RateBackend::Ctw { depth: 9 }; + let mixture = MixtureSpec::new( + MixtureKind::Bayes, + vec![{ + let mut expert = MixtureExpertSpec::new(backend.clone()); + expert.name = Some("ctw".to_string()); + expert + }], + ); + let mix_json = mixture.to_canonical_json().expect("mixture json"); + assert!(mix_json.contains("\"kind\": \"bayes\"")); +} + +#[cfg(feature = "backend-particle")] +#[test] +fn api_surface_particle_spec_serializes_canonically() { + let particle_json = ParticleSpec::default().to_canonical_json(); + assert!( + particle_json + .expect("particle json") + .contains("\"num_particles\"") + ); +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn api_surface_byte_packed_bit_session_matches_byte_prediction_chain() { + let backend = RateBackend::Ctw { depth: 6 }; + let mut byte_session = + RateBackendSession::from_spec(backend.clone(), Some(16)).expect("byte session"); + let mut bit_session = RateBackendBitSession::from_spec( + backend, + Some(16 * 8), + BitStreamSemantics::BytePacked { + order: BitOrder::MsbFirst, + }, + ) + .expect("bit session"); + + for &symbol in b"bit-session" { + let mut row = [0.0f64; 256]; + byte_session.fill_log_probs(&mut row); + let expected = row[symbol as usize].exp(); + let mut product = 1.0f64; + for bit_idx in 0..8u8 { + let bit = ((symbol >> (7 - bit_idx)) & 1) == 1; + product *= bit_session + .try_step_bit(bit) + .expect("byte-packed step should remain in one update mode") + .prob(bit); + } + byte_session.observe(&[symbol]); + assert!( + (product - expected).abs() < 1e-9, + "symbol={symbol} product={product} expected={expected}" + ); + } + + byte_session.finish().expect("byte finish"); + bit_session.finish().expect("bit finish"); +} + +#[cfg(all(feature = "backend-ctw", feature = "backend-mixture"))] +#[test] +fn api_surface_byte_packed_mixture_observe_only_matches_step_learning_outcome() { + let backend = RateBackend::Mixture { + spec: Arc::new(MixtureSpec::new( + MixtureKind::Bayes, + vec![ + MixtureExpertSpec::new(RateBackend::Ctw { depth: 4 }), + MixtureExpertSpec::new(RateBackend::Ctw { depth: 12 }), + ], + )), + }; + let semantics = BitStreamSemantics::BytePacked { + order: BitOrder::MsbFirst, + }; + let data = b"native-byte-prefix observe-only parity check"; + let total_bits = Some((data.len() * 8) as u64); + + let mut step_session = + RateBackendBitSession::from_spec(backend.clone(), total_bits, semantics).expect("step"); + let mut observe_session = + RateBackendBitSession::from_spec(backend, total_bits, semantics).expect("observe"); + + for &symbol in data { + for bit_idx in 0..8u8 { + let bit = ((symbol >> (7 - bit_idx)) & 1) == 1; + let prediction = step_session.try_step_bit(bit).expect("step bit"); + let sum = prediction.p0 + prediction.p1; + assert!( + (sum - 1.0).abs() < 1e-12, + "step-bit prediction must stay normalized, got {sum}" + ); + observe_session + .try_observe_bit(bit) + .expect("observe-only bit"); + } + } + + let step_pred = step_session.predict_bit(); + let observe_pred = observe_session.predict_bit(); + assert!( + (step_pred.p1 - observe_pred.p1).abs() < 1e-12, + "observe-only training should match predict+observe training: step={} observe={}", + step_pred.p1, + observe_pred.p1 + ); + assert!( + (observe_pred.p0 + observe_pred.p1 - 1.0).abs() < 1e-12, + "observe-only prediction must remain normalized" + ); +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn api_surface_bit_session_checkpoint_restores_byte_packed_prefix() { + let semantics = BitStreamSemantics::BytePacked { + order: BitOrder::LsbFirst, + }; + let mut session = + RateBackendBitSession::from_spec(RateBackend::Ctw { depth: 6 }, Some(16), semantics) + .expect("bit session"); + + let initial = session.predict_bit(); + let checkpoint = session.checkpoint(); + session.try_observe_bit(true).expect("partial byte bit"); + session.try_observe_bit(false).expect("partial byte bit"); + let _partial = session.predict_bit(); + + session + .restore_checkpoint(&checkpoint) + .expect("checkpoint restore"); + assert_checkpoint_prediction_within_roundoff( + "restore must recover the pre-prefix prediction", + session.predict_bit(), + initial, + ); + + for bit in [true, false, true, false, true, false, true, false] { + session.try_observe_bit(bit).expect("complete byte"); + } + let _after_byte = session.predict_bit(); + session + .restore_checkpoint(&checkpoint) + .expect("restore after completed byte"); + assert_checkpoint_prediction_within_roundoff( + "restore after completed byte", + session.predict_bit(), + initial, + ); +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn api_surface_bit_session_checkpoint_rejects_mismatched_session() { + let mut a = RateBackendBitSession::from_spec( + RateBackend::Ctw { depth: 6 }, + Some(8), + BitStreamSemantics::BinaryTokens, + ) + .expect("session a"); + let mut b = RateBackendBitSession::from_spec( + RateBackend::Ctw { depth: 7 }, + Some(8), + BitStreamSemantics::BinaryTokens, + ) + .expect("session b"); + let checkpoint = a.checkpoint(); + let err = b + .restore_checkpoint(&checkpoint) + .expect_err("checkpoint must be tied to its backend"); + assert!(err.to_string().contains("different backend")); +} + +#[cfg(all(feature = "backend-ctw", feature = "backend-mixture"))] +#[test] +fn api_surface_bit_session_checkpoint_restores_native_reversible_mixture() { + let backend = RateBackend::Mixture { + spec: Arc::new(MixtureSpec::new( + MixtureKind::Bayes, + vec![ + MixtureExpertSpec::new(RateBackend::Ctw { depth: 6 }), + MixtureExpertSpec::new(RateBackend::FacCtw { + base_depth: 6, + num_percept_bits: 1, + encoding_bits: 1, + msb_first: None, + }), + ], + )), + }; + let mut session = + RateBackendBitSession::from_spec(backend, Some(64), BitStreamSemantics::BinaryTokens) + .expect("native reversible mixture bit session"); + + for bit in [true, false, true, true, false] { + session.try_observe_bit(bit).expect("training bit"); + } + let checkpoint = session.checkpoint(); + let expected = session.predict_bit(); + + for bit in [false, false, true, false, true, true] { + session.try_observe_bit(bit).expect("speculative bit"); + } + session + .restore_checkpoint(&checkpoint) + .expect("restore mixture checkpoint"); + assert_checkpoint_prediction_within_roundoff( + "restore native reversible mixture checkpoint", + session.predict_bit(), + expected, + ); +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn api_surface_byte_packed_try_methods_reject_mixed_update_modes() { + let semantics = BitStreamSemantics::BytePacked { + order: BitOrder::MsbFirst, + }; + let mut frozen_then_adaptive = + RateBackendBitSession::from_spec(RateBackend::Ctw { depth: 6 }, Some(8), semantics) + .expect("bit session"); + for &bit in &[true, false, true] { + frozen_then_adaptive + .try_condition_bit(bit) + .expect("conditioning prefix"); + } + let err = frozen_then_adaptive + .try_observe_bit(false) + .expect_err("mixed-mode byte updates must be rejected"); + let message = err.to_string(); + assert!(message.contains("cannot mix")); + assert!(message.contains("BinaryTokens")); + + let mut adaptive_then_frozen = + RateBackendBitSession::from_spec(RateBackend::Ctw { depth: 6 }, Some(8), semantics) + .expect("bit session"); + for &bit in &[true, false, true] { + adaptive_then_frozen + .try_observe_bit(bit) + .expect("adaptive prefix"); + } + let err = adaptive_then_frozen + .try_condition_bit(false) + .expect_err("mixed-mode byte updates must be rejected"); + assert!(err.to_string().contains("cannot mix")); +} + +#[cfg(feature = "backend-ctw")] +#[test] +#[should_panic(expected = "cannot mix")] +fn api_surface_byte_packed_strict_methods_panic_on_mixed_update_modes() { + let mut bit_session = RateBackendBitSession::from_spec( + RateBackend::Ctw { depth: 6 }, + Some(8), + BitStreamSemantics::BytePacked { + order: BitOrder::MsbFirst, + }, + ) + .expect("bit session"); + for &bit in &[true, false, true] { + bit_session.condition_bit(bit); + } + bit_session.observe_bit(false); +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn api_surface_bit_session_semantics_are_fixed() { + let backend = RateBackend::Ctw { depth: 6 }; + let mut bit_session = RateBackendBitSession::from_spec( + backend, + Some(8), + BitStreamSemantics::BytePacked { + order: BitOrder::MsbFirst, + }, + ) + .expect("bit session"); + + bit_session + .begin_bit_stream( + Some(8), + BitStreamSemantics::BytePacked { + order: BitOrder::MsbFirst, + }, + ) + .expect("same semantics reset"); + let err = bit_session + .begin_bit_stream(Some(8), BitStreamSemantics::BinaryTokens) + .expect_err("semantic switches need a freshly adapted session"); + assert!(err.contains("semantics are fixed")); +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn api_surface_byte_packed_bit_session_rejects_non_byte_aligned_lengths() { + let semantics = BitStreamSemantics::BytePacked { + order: BitOrder::MsbFirst, + }; + let err = + match RateBackendBitSession::from_spec(RateBackend::Ctw { depth: 6 }, Some(9), semantics) { + Ok(_) => panic!("byte-packed streams require whole bytes"), + Err(err) => err, + }; + assert!(err.to_string().contains("whole number of bytes")); + assert!(err.to_string().contains("BinaryTokens")); + + let mut bit_session = + RateBackendBitSession::from_spec(RateBackend::Ctw { depth: 6 }, Some(8), semantics) + .expect("bit session"); + let reset_err = bit_session + .reset_frozen(Some(9)) + .expect_err("reset should reject non-byte-aligned total_bits"); + assert!(reset_err.to_string().contains("whole number of bytes")); + + let begin_err = bit_session + .begin_bit_stream(Some(9), semantics) + .expect_err("begin should reject non-byte-aligned total_bits"); + assert!(begin_err.contains("whole number of bytes")); +} + +#[cfg(feature = "backend-zpaq")] +#[test] +fn api_surface_zpaq_bit_session_begin_stream_does_not_require_frozen_reset() { + let semantics = BitStreamSemantics::BinaryTokens; + let mut bit_session = RateBackendBitSession::from_spec( + RateBackend::Zpaq { + method: infotheory::api::ZpaqMethodSpec::literal("1"), + }, + Some(9), + semantics, + ) + .expect("zpaq bit session"); + + let reset_err = bit_session + .reset_frozen(Some(9)) + .expect_err("zpaq must continue to reject frozen-reset semantics"); + assert!(reset_err.to_string().contains("plugin entropy")); + + bit_session + .begin_bit_stream(Some(9), semantics) + .expect("zpaq stream restarts should use begin/finish lifecycle hooks"); + + for bit in [true, false, true, true, false, false, true, false, true] { + let prediction = bit_session.step_bit(bit); + let sum = prediction.p0 + prediction.p1; + assert!( + (sum - 1.0).abs() < 1e-12, + "zpaq binary-token prediction must stay normalized, got {sum}" + ); + } + + bit_session.finish().expect("zpaq bit finish"); +} + +#[cfg(feature = "backend-zpaq")] +#[test] +fn api_surface_zpaq_byte_packed_bit_session_is_rejected() { + let err = match RateBackendBitSession::from_spec( + RateBackend::Zpaq { + method: infotheory::api::ZpaqMethodSpec::literal("1"), + }, + Some(8), + BitStreamSemantics::BytePacked { + order: BitOrder::MsbFirst, + }, + ) { + Ok(_) => panic!("zpaq byte-packed session should be rejected"), + Err(err) => err, + }; + let message = err.to_string(); + assert!(message.contains("does not support efficient BitStreamSemantics::BytePacked")); +} + +#[cfg(feature = "backend-zpaq")] +#[test] +fn api_surface_zpaq_rate_backend_session_begin_stream_does_not_require_frozen_reset() { + let mut session = RateBackendSession::from_spec( + RateBackend::Zpaq { + method: infotheory::api::ZpaqMethodSpec::literal("1"), + }, + Some(9), + ) + .expect("zpaq rate session"); + + let reset_err = session + .reset_frozen(Some(9)) + .expect_err("zpaq must continue to reject frozen-reset semantics"); + assert!(reset_err.to_string().contains("plugin entropy")); + + session + .begin_stream(Some(9)) + .expect("zpaq stream restarts should use begin/finish lifecycle hooks"); + + let mut row = [0.0f64; 256]; + session.fill_log_probs(&mut row); + assert!(row.iter().all(|lp| lp.is_finite())); + let first_row = row; + + session.observe(&[0, 1, 0, 1, 1, 0, 1, 0, 1]); + // Exercise ordinary ZPAQ compression in-process before stream restart; + // restarted session probabilities must still match a fresh session. + let zpaq_compression = CompressionBackend::zpaq("1") + .compile() + .expect("compile zpaq compression backend"); + let _ = try_compress_bytes_backend(b"zpaq helper warmup", &zpaq_compression) + .expect("zpaq helper compression"); + session + .begin_stream(Some(9)) + .expect("zpaq stream should be restartable repeatedly"); + let mut restarted_row = [0.0f64; 256]; + session.fill_log_probs(&mut restarted_row); + let mut fresh = RateBackendSession::from_spec( + RateBackend::Zpaq { + method: infotheory::api::ZpaqMethodSpec::literal("1"), + }, + Some(9), + ) + .expect("fresh zpaq rate session"); + let mut fresh_row = [0.0f64; 256]; + fresh.fill_log_probs(&mut fresh_row); + for idx in 0..256usize { + assert!( + (restarted_row[idx] - fresh_row[idx]).abs() < 1e-12, + "zpaq restart must match fresh-session state at symbol {idx}: restarted={} fresh={}", + restarted_row[idx], + fresh_row[idx] + ); + } + // Sanity check: this test should fail if restart preserves post-observation history. + let changed = (0..256usize).any(|idx| (first_row[idx] - restarted_row[idx]).abs() > 1e-12); + assert!( + !changed, + "zpaq restart should return to the initial stream state" + ); + session.finish().expect("zpaq rate finish"); + fresh.finish().expect("fresh zpaq rate finish"); +} + +#[cfg(all(feature = "backend-mixture", feature = "backend-ctw"))] +#[test] +fn api_surface_mixture_begin_stream_resets_wrapper_priors_after_expert_restart() { + let backend = RateBackend::Mixture { + spec: Arc::new(MixtureSpec::new( + MixtureKind::Bayes, + vec![ + MixtureExpertSpec::new(RateBackend::Ctw { depth: 0 }), + MixtureExpertSpec::new(RateBackend::Ctw { depth: 6 }), + ], + )), + }; + let train = b"ABABABABABABABABABABABABABABABAB"; + + let mut reset_session = + RateBackendSession::from_spec(backend.clone(), None).expect("reset session"); + let mut restarted_session = + RateBackendSession::from_spec(backend.clone(), None).expect("restarted session"); + let mut fresh_session = RateBackendSession::from_spec(backend, None).expect("fresh session"); + + reset_session.observe(train); + restarted_session.observe(train); + + reset_session.reset_frozen(None).expect("reset_frozen"); + restarted_session.begin_stream(None).expect("begin_stream"); + + let mut reset_row = [0.0f64; 256]; + let mut restarted_row = [0.0f64; 256]; + let mut fresh_row = [0.0f64; 256]; + reset_session.fill_log_probs(&mut reset_row); + restarted_session.fill_log_probs(&mut restarted_row); + fresh_session.fill_log_probs(&mut fresh_row); + + let diverged_from_reset = + (0..256usize).any(|byte| (reset_row[byte] - restarted_row[byte]).abs() > 1e-12); + assert!( + diverged_from_reset, + "mixture begin_stream should reset sequence-local wrapper weights instead of preserving reset_frozen posterior state" + ); + + let diverged_from_fresh = + (0..256usize).any(|byte| (restarted_row[byte] - fresh_row[byte]).abs() > 1e-12); + assert!( + diverged_from_fresh, + "mixture begin_stream should preserve restarted expert state instead of rebuilding fresh experts" + ); +} + +#[cfg(all( + feature = "backend-mixture", + feature = "backend-zpaq", + feature = "backend-ctw" +))] +#[test] +fn api_surface_mixture_with_zpaq_expert_can_restart_bit_streams() { + let backend = RateBackend::Mixture { + spec: Arc::new(MixtureSpec::new( + MixtureKind::Bayes, + vec![ + MixtureExpertSpec::new(RateBackend::Ctw { depth: 6 }), + MixtureExpertSpec::new(RateBackend::Zpaq { + method: infotheory::api::ZpaqMethodSpec::literal("1"), + }), + ], + )), + }; + let mut bit_session = + RateBackendBitSession::from_spec(backend, Some(9), BitStreamSemantics::BinaryTokens) + .expect("mixture bit session"); + + let reset_err = bit_session + .reset_frozen(Some(9)) + .expect_err("mixtures containing zpaq experts cannot satisfy frozen-reset semantics"); + assert!(reset_err.to_string().contains("plugin entropy")); + + bit_session + .begin_bit_stream(Some(9), BitStreamSemantics::BinaryTokens) + .expect("mixture stream restarts should fall back to lifecycle hooks"); + + for bit in [true, false, true, false, true, true, false, false, true] { + let prediction = bit_session.step_bit(bit); + let sum = prediction.p0 + prediction.p1; + assert!( + (sum - 1.0).abs() < 1e-12, + "mixture-zpaq binary-token prediction must stay normalized, got {sum}" + ); + } + + bit_session.finish().expect("mixture-zpaq bit finish"); +} + +#[cfg(all( + feature = "backend-mixture", + feature = "backend-zpaq", + feature = "backend-ctw" +))] +#[test] +fn api_surface_mixture_with_zpaq_rate_backend_session_can_restart_streams() { + let backend = RateBackend::Mixture { + spec: Arc::new(MixtureSpec::new( + MixtureKind::Bayes, + vec![ + MixtureExpertSpec::new(RateBackend::Ctw { depth: 6 }), + MixtureExpertSpec::new(RateBackend::Zpaq { + method: infotheory::api::ZpaqMethodSpec::literal("1"), + }), + ], + )), + }; + let mut session = RateBackendSession::from_spec(backend, Some(9)).expect("mixture session"); + + let reset_err = session + .reset_frozen(Some(9)) + .expect_err("mixtures containing zpaq experts cannot satisfy frozen-reset semantics"); + assert!(reset_err.to_string().contains("plugin entropy")); + + session + .begin_stream(Some(9)) + .expect("mixture stream restarts should use begin/finish lifecycle hooks"); + + let mut row = [0.0f64; 256]; + session.fill_log_probs(&mut row); + assert!(row.iter().all(|lp| lp.is_finite())); + let first_row = row; + + session.observe(&[1, 0, 1, 0, 1, 1, 0, 0, 1]); + session + .begin_stream(Some(9)) + .expect("mixture stream should be restartable repeatedly"); + let mut restarted_row = [0.0f64; 256]; + session.fill_log_probs(&mut restarted_row); + assert!(restarted_row.iter().all(|lp| lp.is_finite())); + let changed = (0..256usize).any(|idx| (first_row[idx] - restarted_row[idx]).abs() > 1e-12); + assert!( + changed, + "mixture+zpaq restart should preserve fitted state from resettable experts" + ); + session.finish().expect("mixture rate finish"); +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn api_surface_byte_packed_finish_rejects_dangling_partial_byte() { + let mut bit_session = RateBackendBitSession::from_spec( + RateBackend::Ctw { depth: 6 }, + None, + BitStreamSemantics::BytePacked { + order: BitOrder::MsbFirst, + }, + ) + .expect("bit session"); + + for bit in [true, false, true] { + bit_session.observe_bit(bit); + } + + let err = bit_session + .finish() + .expect_err("dangling partial byte must not be discarded"); + assert!(err.to_string().contains("whole-byte boundary")); +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn api_surface_byte_packed_finish_allows_prediction_without_observe() { + let mut bit_session = RateBackendBitSession::from_spec( + RateBackend::Ctw { depth: 6 }, + None, + BitStreamSemantics::BytePacked { + order: BitOrder::MsbFirst, + }, + ) + .expect("bit session"); + + let prediction = bit_session.predict_bit(); + let sum = prediction.p0 + prediction.p1; + assert!( + (sum - 1.0).abs() < 1e-12, + "byte-packed prediction must stay normalized, got {sum}" + ); + + bit_session + .finish() + .expect("prediction-only byte-packed sessions must finish cleanly"); +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn api_surface_binary_tokens_accept_arbitrary_length_streams() { + let mut bit_session = RateBackendBitSession::from_spec( + RateBackend::Ctw { depth: 6 }, + Some(9), + BitStreamSemantics::BinaryTokens, + ) + .expect("bit session"); + + for bit in [true, false, true, true, false, false, true, false, true] { + let prediction = bit_session.step_bit(bit); + let sum = prediction.p0 + prediction.p1; + assert!( + (sum - 1.0).abs() < 1e-12, + "binary-token prediction must stay normalized, got {sum}" + ); + } + + bit_session.finish().expect("bit finish"); +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn api_surface_fac_ctw_binary_tokens_accept_arbitrary_length_streams() { + let compiled = RateBackend::FacCtw { + base_depth: 6, + num_percept_bits: 8, + encoding_bits: 8, + msb_first: None, + } + .compile() + .expect("compiled fac-ctw"); + assert!(compiled.capabilities().supports_native_bit_prediction); + assert!(compiled.capabilities().supports_byte_prefix_mass); + assert!(compiled.supports_efficient_byte_packed_bit_sessions()); + assert!(compiled.capabilities().supports_reversible_bit_updates); + + let mut bit_session = + RateBackendBitSession::from_backend(compiled, Some(9), BitStreamSemantics::BinaryTokens) + .expect("fac-ctw binary-token session"); + + for bit in [true, false, true, false, true, true, false, false, true] { + let prediction = bit_session.step_bit(bit); + let sum = prediction.p0 + prediction.p1; + assert!( + (sum - 1.0).abs() < 1e-12, + "binary-token prediction must stay normalized, got {sum}" + ); + } + + bit_session.finish().expect("bit finish"); +} + +#[cfg(all(feature = "backend-mixture", feature = "backend-ctw"))] +#[test] +fn api_surface_mixture_over_native_bit_backend_preserves_binary_tokens() { + let backend = RateBackend::Mixture { + spec: Arc::new(MixtureSpec::new( + MixtureKind::Bayes, + vec![MixtureExpertSpec::new(RateBackend::Ctw { depth: 6 })], + )), + }; + let compiled = backend.clone().compile().expect("compiled mixture"); + assert!(compiled.capabilities().supports_native_bit_prediction); + assert!(compiled.capabilities().supports_byte_prefix_mass); + assert!(compiled.supports_efficient_byte_packed_bit_sessions()); + assert!(compiled.capabilities().supports_reversible_bit_updates); + + let mut bit_session = + RateBackendBitSession::from_spec(backend, Some(9), BitStreamSemantics::BinaryTokens) + .expect("mixture binary-token session"); + + for bit in [true, false, false, true, true, false, true, false, true] { + let prediction = bit_session.step_bit(bit); + let sum = prediction.p0 + prediction.p1; + assert!( + (sum - 1.0).abs() < 1e-12, + "binary-token prediction must stay normalized, got {sum}" + ); + } + + bit_session.finish().expect("bit finish"); +} + +#[cfg(feature = "backend-match")] +#[test] +fn api_surface_binary_tokens_adapt_byte_native_backends() { + let backend = RateBackend::Match { + hash_bits: 20, + min_len: 4, + max_len: 255, + base_mix: 0.02, + confidence_scale: 1.0, + }; + let compiled = backend.clone().compile().expect("compiled match"); + assert!(!compiled.capabilities().supports_native_bit_prediction); + + let mut byte_session = + RateBackendSession::from_spec(backend.clone(), Some(9)).expect("byte session"); + let mut bit_session = + RateBackendBitSession::from_spec(backend, Some(9), BitStreamSemantics::BinaryTokens) + .expect("binary-token session"); + + for &bit in &[true, false, true, true, false, false, true, false, true] { + let mut row = [0.0f64; 256]; + byte_session.fill_log_probs(&mut row); + let p0 = row[0].exp(); + let p1 = row[1].exp(); + let total = p0 + p1; + let expected_p0 = if total.is_finite() && total > 0.0 { + p0 / total + } else { + 0.5 + }; + let expected_p1 = if total.is_finite() && total > 0.0 { + p1 / total + } else { + 0.5 + }; + + let prediction = bit_session.step_bit(bit); + assert!( + (prediction.p0 + prediction.p1 - 1.0).abs() < 1e-12, + "binary-token adaptation must stay normalized, got p0={} p1={}", + prediction.p0, + prediction.p1 + ); + assert!( + (prediction.p0 - expected_p0).abs() < 1e-12, + "adapted p0 drifted: got {} expected {}", + prediction.p0, + expected_p0 + ); + assert!( + (prediction.p1 - expected_p1).abs() < 1e-12, + "adapted p1 drifted: got {} expected {}", + prediction.p1, + expected_p1 + ); + + byte_session.observe(&[u8::from(bit)]); + } + + byte_session.finish().expect("byte finish"); + bit_session.finish().expect("bit finish"); +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn api_surface_rate_backend_bit_capabilities_are_explicit() { + let ctw = RateBackend::Ctw { depth: 6 } + .compile() + .expect("compiled ctw"); + assert!(ctw.capabilities().supports_native_bit_prediction); + assert!(ctw.capabilities().supports_byte_prefix_mass); + assert!(ctw.supports_efficient_byte_packed_bit_sessions()); + assert!(ctw.capabilities().supports_reversible_bit_updates); + + #[cfg(feature = "backend-match")] + { + let match_backend = RateBackend::Match { + hash_bits: 20, + min_len: 4, + max_len: 255, + base_mix: 0.02, + confidence_scale: 1.0, + } + .compile() + .expect("compiled match"); + assert!(!match_backend.capabilities().supports_native_bit_prediction); + assert!(match_backend.capabilities().supports_byte_prefix_mass); + assert!(match_backend.supports_efficient_byte_packed_bit_sessions()); + assert!(!match_backend.capabilities().supports_reversible_bit_updates); + } + + #[cfg(feature = "backend-zpaq")] + { + use infotheory::api::ZpaqMethodSpec; + + let zpaq = RateBackend::Zpaq { + method: ZpaqMethodSpec::Literal { + value: "1".to_string(), + }, + } + .compile() + .expect("compiled zpaq"); + assert!(!zpaq.capabilities().supports_native_bit_prediction); + assert!(zpaq.capabilities().supports_byte_prefix_mass); + assert!(!zpaq.supports_efficient_byte_packed_bit_sessions()); + assert!(!zpaq.capabilities().supports_reversible_bit_updates); + } + + #[cfg(feature = "backend-mixture")] + { + let mixture = RateBackend::Mixture { + spec: Arc::new(MixtureSpec::new( + MixtureKind::Bayes, + vec![MixtureExpertSpec::new(RateBackend::Ctw { depth: 6 })], + )), + } + .compile() + .expect("compiled mixture"); + assert!(mixture.capabilities().supports_native_bit_prediction); + assert!(mixture.capabilities().supports_byte_prefix_mass); + assert!(mixture.supports_efficient_byte_packed_bit_sessions()); + assert!(mixture.capabilities().supports_reversible_bit_updates); + } + + #[cfg(all(feature = "backend-mixture", feature = "backend-zpaq"))] + { + let mixture = RateBackend::Mixture { + spec: Arc::new(MixtureSpec::new( + MixtureKind::Bayes, + vec![MixtureExpertSpec::new(RateBackend::Zpaq { + method: infotheory::api::ZpaqMethodSpec::literal("1"), + })], + )), + } + .compile() + .expect("compiled zpaq mixture"); + assert!(!mixture.capabilities().supports_native_bit_prediction); + assert!(mixture.capabilities().supports_byte_prefix_mass); + assert!(!mixture.supports_efficient_byte_packed_bit_sessions()); + assert!(!mixture.capabilities().supports_reversible_bit_updates); + } + + #[cfg(all(feature = "backend-mixture", feature = "backend-match"))] + { + let mixture = RateBackend::Mixture { + spec: Arc::new(MixtureSpec::new( + MixtureKind::Bayes, + vec![MixtureExpertSpec::new(RateBackend::Match { + hash_bits: 20, + min_len: 4, + max_len: 255, + base_mix: 0.02, + confidence_scale: 1.0, + })], + )), + } + .compile() + .expect("compiled byte-native mixture"); + assert!(!mixture.capabilities().supports_native_bit_prediction); + assert!(mixture.capabilities().supports_byte_prefix_mass); + assert!(mixture.supports_efficient_byte_packed_bit_sessions()); + assert!(!mixture.capabilities().supports_reversible_bit_updates); + } + + #[cfg(feature = "backend-calibrated")] + { + let calibrated = RateBackend::Calibrated { + spec: Arc::new(CalibratedSpec::new( + RateBackend::Ctw { depth: 6 }, + CalibrationContextKind::Global, + )), + } + .compile() + .expect("compiled calibrated"); + assert!(calibrated.capabilities().supports_native_bit_prediction); + assert!(calibrated.capabilities().supports_byte_prefix_mass); + assert!(calibrated.supports_efficient_byte_packed_bit_sessions()); + assert!(calibrated.capabilities().supports_reversible_bit_updates); + } + + #[cfg(all(feature = "backend-calibrated", feature = "backend-zpaq"))] + { + let calibrated = RateBackend::Calibrated { + spec: Arc::new(CalibratedSpec::new( + RateBackend::Zpaq { + method: infotheory::api::ZpaqMethodSpec::literal("1"), + }, + CalibrationContextKind::Global, + )), + } + .compile() + .expect("compiled zpaq calibrated"); + assert!(!calibrated.capabilities().supports_native_bit_prediction); + assert!(calibrated.capabilities().supports_byte_prefix_mass); + assert!(!calibrated.supports_efficient_byte_packed_bit_sessions()); + assert!(!calibrated.capabilities().supports_reversible_bit_updates); + } + + #[cfg(all(feature = "backend-calibrated", feature = "backend-match"))] + { + let calibrated = RateBackend::Calibrated { + spec: Arc::new(CalibratedSpec::new( + RateBackend::Match { + hash_bits: 20, + min_len: 4, + max_len: 255, + base_mix: 0.02, + confidence_scale: 1.0, + }, + CalibrationContextKind::Global, + )), + } + .compile() + .expect("compiled byte-native calibrated"); + assert!(!calibrated.capabilities().supports_native_bit_prediction); + assert!(calibrated.capabilities().supports_byte_prefix_mass); + assert!(calibrated.supports_efficient_byte_packed_bit_sessions()); + assert!(!calibrated.capabilities().supports_reversible_bit_updates); + } +} + +#[cfg(feature = "backend-ctw")] +mod ctw_surface { + use infotheory::api::{ + CompressionBackend, InfotheoryCtx, RateBackend, d_kl_bytes, empirical_cross_entropy_bytes, + empirical_entropy_bytes, empirical_joint_entropy_bytes, empirical_mutual_information_bytes, + empirical_ned_bytes, empirical_ned_cons_bytes, empirical_nte_bytes, get_default_ctx, + js_div_bytes, nhd_bytes, set_default_ctx, try_biased_entropy_rate_backend, + try_biased_entropy_rate_bytes, try_conditional_entropy_bytes, + try_conditional_entropy_rate_bytes, try_cross_entropy_bytes, + try_cross_entropy_rate_backend, try_cross_entropy_rate_bytes, try_entropy_rate_backend, + try_entropy_rate_bytes, try_intrinsic_dependence_bytes, try_joint_entropy_rate_backend, + try_joint_entropy_rate_bytes, try_mutual_information_bytes, + try_mutual_information_rate_backend, try_mutual_information_rate_bytes, try_ned_bytes, + try_ned_cons_bytes, try_ned_cons_rate_bytes, try_ned_rate_backend, try_ned_rate_bytes, + try_nte_bytes, try_nte_rate_backend, try_nte_rate_bytes, + try_resistance_to_transformation_bytes, tvd_bytes, + }; + + #[test] + fn api_surface_entropy_and_distance_wrappers_are_callable() { + let x = b"alpha beta alpha beta alpha"; + let y = b"alpha gamma alpha gamma alpha"; + let backend = RateBackend::Ctw { depth: 8 }; + let compiled = backend.compile().expect("compiled ctw backend"); + + let prev = get_default_ctx().expect("default ctx"); + set_default_ctx( + InfotheoryCtx::from_specs( + backend.clone(), + CompressionBackend::try_default().expect("default compression backend"), + ) + .expect("ctw context"), + ); + + assert!(try_entropy_rate_backend(x, &compiled).expect("entropy rate") >= 0.0); + assert!(try_biased_entropy_rate_backend(x, &compiled).expect("biased entropy rate") >= 0.0); + assert!( + try_cross_entropy_rate_backend(x, y, &compiled).expect("cross entropy rate") >= 0.0 + ); + assert!( + try_joint_entropy_rate_backend(x, y, &compiled).expect("joint entropy rate") >= 0.0 + ); + assert!(try_mutual_information_rate_backend(x, y, &compiled).expect("mi rate") >= 0.0); + assert!((0.0..=1.0).contains(&try_ned_rate_backend(x, y, &compiled).expect("ned rate"))); + assert!((0.0..=2.0).contains(&try_nte_rate_backend(x, y, &compiled).expect("nte rate"))); + + assert!(empirical_entropy_bytes(x) >= 0.0); + assert!(empirical_joint_entropy_bytes(x, y) >= 0.0); + assert!(try_entropy_rate_bytes(x).expect("entropy rate bytes") >= 0.0); + assert!(try_biased_entropy_rate_bytes(x).expect("biased entropy rate bytes") >= 0.0); + assert!(try_joint_entropy_rate_bytes(x, y).expect("joint entropy rate bytes") >= 0.0); + assert!( + try_conditional_entropy_rate_bytes(x, y).expect("conditional entropy rate bytes") + >= 0.0 + ); + assert!(try_conditional_entropy_bytes(x, y).expect("conditional entropy bytes") >= 0.0); + assert!(try_mutual_information_bytes(x, y).expect("mutual information bytes") >= 0.0); + assert!(empirical_mutual_information_bytes(x, y) >= 0.0); + assert!( + try_mutual_information_rate_bytes(x, y).expect("mutual information rate bytes") >= 0.0 + ); + assert!((0.0..=1.0).contains(&try_ned_bytes(x, y).expect("ned bytes"))); + assert!((0.0..=1.0).contains(&empirical_ned_bytes(x, y))); + assert!((0.0..=1.0).contains(&try_ned_rate_bytes(x, y).expect("ned rate bytes"))); + assert!((0.0..=1.0).contains(&try_ned_cons_bytes(x, y).expect("ned cons bytes"))); + assert!((0.0..=1.0).contains(&empirical_ned_cons_bytes(x, y))); + assert!((0.0..=1.0).contains(&try_ned_cons_rate_bytes(x, y).expect("ned cons rate bytes"))); + assert!((0.0..=2.0).contains(&try_nte_bytes(x, y).expect("nte bytes"))); + assert!((0.0..=2.0).contains(&empirical_nte_bytes(x, y))); + assert!((0.0..=2.0).contains(&try_nte_rate_bytes(x, y).expect("nte rate bytes"))); + assert!((0.0..=1.0).contains(&tvd_bytes(x, y))); + assert!((0.0..=1.0).contains(&nhd_bytes(x, y))); + assert!(try_cross_entropy_bytes(x, y).expect("cross entropy bytes") >= 0.0); + assert!(empirical_cross_entropy_bytes(x, y) >= 0.0); + assert!(try_cross_entropy_rate_bytes(x, y).expect("cross entropy rate bytes") >= 0.0); + assert!(d_kl_bytes(x, y) >= 0.0); + assert!(js_div_bytes(x, y) >= 0.0); + assert!( + (0.0..=1.0).contains(&try_intrinsic_dependence_bytes(x).expect("intrinsic dependence")) + ); + assert!((0.0..=1.0).contains( + &try_resistance_to_transformation_bytes(x, y).expect("resistance to transformation") + )); + + set_default_ctx(prev); + } +} + +#[cfg(feature = "backend-rosa")] +mod rosa_surface { + use infotheory::api::{ + CompressionBackend, GenerationConfig, InfotheoryCtx, RateBackend, RateBackendSession, + }; + + #[test] + fn api_surface_generation_session_and_config_are_callable() { + let prompt = b"If a frog is green, dogs are red.\nIf a toad is green, cats are red.\nIf a dog is green, frogs are red.\nIf a cat is green, toads are red.\nIf a frog is red, dogs are green.\nIf a toad is red, cats are green.\nIf a dog is red, frogs are green.\nIf a cat is red, toads are "; + let backend = RateBackend::RosaPlus { max_order: -1 }; + let ctx = InfotheoryCtx::from_specs( + backend.clone(), + CompressionBackend::try_default().expect("default compression backend"), + ) + .expect("ctx"); + let cfg = GenerationConfig::sampled_frozen(42); + + let direct = ctx + .try_generate_bytes_with_config(prompt, 8, cfg) + .expect("direct generation"); + assert_eq!(direct.len(), 8); + + let mut session = + RateBackendSession::from_spec(backend, Some((prompt.len() + direct.len()) as u64)) + .expect("session init"); + session.observe(prompt); + let from_session = session.generate_bytes(8, cfg); + session.finish().expect("session finish"); + + assert_eq!(from_session, direct); + } +} + +#[cfg(feature = "backend-zpaq")] +mod zpaq_surface { + use infotheory::api::{ + CompressionBackend, CompressionPathBatchOptions, NcdVariant, OperationParallelism, + try_compress_bytes_backend, try_compress_size_backend, try_compress_size_chain_backend, + try_conditional_entropy_paths, try_cross_entropy_paths, try_decompress_bytes_backend, + try_get_bytes_from_paths, try_get_compressed_size_path_backend, + try_get_compressed_sizes_from_paths_backend, + try_get_compressed_sizes_from_paths_backend_with_options, try_js_divergence_paths, + try_kl_divergence_paths, try_mutual_information_paths, try_ncd_bytes_backend, + try_ncd_bytes_default, try_ncd_matrix_bytes_backend, try_ncd_matrix_paths_backend, + try_ncd_paths_backend, try_ncd_paths_compiled_backend, try_ned_paths, try_nhd_paths, + try_nte_paths, try_tvd_paths, + }; + use std::fs; + use std::path::PathBuf; + use std::time::{SystemTime, UNIX_EPOCH}; + + fn has_not_found_io_error(err: &(dyn std::error::Error + 'static)) -> bool { + let mut current = Some(err); + while let Some(err) = current { + if let Some(io_err) = err.downcast_ref::() + && io_err.kind() == std::io::ErrorKind::NotFound + { + return true; + } + current = err.source(); + } + false + } + + fn temp_file(name: &str, contents: &[u8]) -> PathBuf { + let ts = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("clock should be monotonic") + .as_nanos(); + let path = std::env::temp_dir().join(format!("infotheory_api_{name}_{ts}.bin")); + fs::write(&path, contents).expect("temp fixture write should succeed"); + path + } + + #[cfg(not(target_env = "musl"))] + #[test] + fn api_surface_path_and_compression_helpers_are_callable() { + let x = b"lorem ipsum dolor sit amet"; + let y = b"lorem ipsum dolor"; + let px = temp_file("x", x); + let py = temp_file("y", y); + let sx = px.to_string_lossy().to_string(); + let sy = py.to_string_lossy().to_string(); + let paths = [sx.as_str(), sy.as_str()]; + + let backend = CompressionBackend::zpaq("1"); + let compiled = backend.compile().expect("compiled zpaq backend"); + + assert!(try_compress_size_backend(x, &compiled).expect("fallible zpaq size") > 0); + assert!( + try_compress_size_chain_backend(&[x.as_slice(), y.as_slice()], &compiled) + .expect("fallible chain size") + > 0 + ); + let c = try_compress_bytes_backend(x, &compiled).expect("zpaq compress"); + let d = try_decompress_bytes_backend(&c, &compiled).expect("zpaq decompress"); + assert_eq!(d, x); + + assert!( + try_get_compressed_size_path_backend(&sx, &compiled).expect("fallible file size") > 0 + ); + + let bytes_try = try_get_bytes_from_paths(&paths).expect("fallible bytes from paths"); + assert_eq!(bytes_try.len(), 2); + assert_eq!(bytes_try[0], x); + assert_eq!(bytes_try[1], y); + + let s_serial = try_get_compressed_sizes_from_paths_backend_with_options( + &paths, + &compiled, + CompressionPathBatchOptions { + parallelism: OperationParallelism::Serial, + }, + ) + .expect("serial sizes"); + let s_auto = + try_get_compressed_sizes_from_paths_backend(&paths, &compiled).expect("auto sizes"); + let s_pool = try_get_compressed_sizes_from_paths_backend_with_options( + &paths, + &compiled, + CompressionPathBatchOptions { + parallelism: OperationParallelism::Threads(2), + }, + ) + .expect("pool sizes"); + for sizes in [s_serial, s_auto, s_pool] { + assert_eq!(sizes.len(), 2); + assert!(sizes[0] > 0); + assert!(sizes[1] > 0); + } + + assert!( + try_ncd_bytes_backend(x, y, &compiled, NcdVariant::Vitanyi).expect("fallible ncd") + >= 0.0 + ); + assert!( + try_ncd_bytes_default(x, y, NcdVariant::SymVitanyi).expect("ncd bytes default") >= 0.0 + ); + assert!( + try_ncd_bytes_backend(x, y, &compiled, NcdVariant::Cons).expect("ncd bytes backend") + >= 0.0 + ); + assert!( + try_ncd_paths_backend(&sx, &sy, &backend, NcdVariant::Vitanyi) + .expect("fallible file ncd") + >= 0.0 + ); + assert!( + try_ncd_paths_compiled_backend(&sx, &sy, &compiled, NcdVariant::SymCons) + .expect("ncd paths compiled") + >= 0.0 + ); + let m = + try_ncd_matrix_bytes_backend(&[x.to_vec(), y.to_vec()], &compiled, NcdVariant::Vitanyi) + .expect("matrix ncd bytes"); + assert_eq!(m.len(), 4); + let mp = try_ncd_matrix_paths_backend(&paths, &compiled, NcdVariant::Cons) + .expect("matrix ncd paths"); + assert_eq!(mp.len(), 4); + + assert!(try_ned_paths(&sx, &sy).expect("ned paths") >= 0.0); + assert!(try_nte_paths(&sx, &sy).expect("nte paths") >= 0.0); + assert!(try_tvd_paths(&sx, &sy).expect("tvd paths") >= 0.0); + assert!(try_nhd_paths(&sx, &sy).expect("nhd paths") >= 0.0); + assert!(try_mutual_information_paths(&sx, &sy).expect("mi paths") >= 0.0); + assert!(try_conditional_entropy_paths(&sx, &sy).expect("conditional entropy paths") >= 0.0); + assert!(try_cross_entropy_paths(&sx, &sy).expect("cross entropy paths") >= 0.0); + assert!(try_kl_divergence_paths(&sx, &sy).expect("kl paths") >= 0.0); + assert!(try_js_divergence_paths(&sx, &sy).expect("js paths") >= 0.0); + + let _ = fs::remove_file(px); + let _ = fs::remove_file(py); + } + + #[test] + fn api_surface_fallible_path_helpers_report_missing_files() { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("clock should be monotonic") + .as_nanos(); + let missing_path = std::env::temp_dir().join(format!( + "infotheory_api_missing_does_not_exist_{unique}.bin" + )); + let missing = missing_path.to_string_lossy().to_string(); + let compiled = CompressionBackend::zpaq("1") + .compile() + .expect("compile zpaq backend"); + + let err = try_get_compressed_size_path_backend(&missing, &compiled) + .expect_err("missing file should error"); + assert!( + has_not_found_io_error(&err), + "expected not-found io error, got: {err}" + ); + + let err = + try_get_bytes_from_paths(&[&missing]).expect_err("missing bytes path should error"); + assert!( + has_not_found_io_error(&err), + "expected not-found io error, got: {err}" + ); + + for err in [ + try_ned_paths(&missing, &missing).expect_err("ned paths should error"), + try_nte_paths(&missing, &missing).expect_err("nte paths should error"), + try_nhd_paths(&missing, &missing).expect_err("nhd paths should error"), + try_mutual_information_paths(&missing, &missing).expect_err("mi paths should error"), + try_conditional_entropy_paths(&missing, &missing) + .expect_err("conditional entropy paths should error"), + try_cross_entropy_paths(&missing, &missing) + .expect_err("cross entropy paths should error"), + try_kl_divergence_paths(&missing, &missing).expect_err("kl paths should error"), + try_js_divergence_paths(&missing, &missing).expect_err("jsd paths should error"), + ] { + assert!( + has_not_found_io_error(&err), + "expected not-found io error, got: {err}" + ); + } + } +} diff --git a/crates/infotheory/tests/benchmark_suite_specs.rs b/crates/infotheory/tests/benchmark_suite_specs.rs new file mode 100644 index 00000000..ef9508a4 --- /dev/null +++ b/crates/infotheory/tests/benchmark_suite_specs.rs @@ -0,0 +1,139 @@ +use serde_json::Value; +use std::fs; +use std::path::PathBuf; + +fn repo_root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("..") + .join("..") +} + +fn read_json(path: &PathBuf) -> Value { + let raw = fs::read_to_string(path) + .unwrap_or_else(|e| panic!("failed to read {}: {e}", path.display())); + serde_json::from_str(&raw) + .unwrap_or_else(|e| panic!("failed to parse {} as JSON: {e}", path.display())) +} + +fn load_example(name: &str) -> Value { + let path = repo_root().join("configs").join("bench").join(name); + read_json(&path) +} + +#[test] +fn two_json_benchmark_specs_are_pinned_and_canonical() { + let root = repo_root(); + let config_path = root.join("configs").join("bench").join("two.json"); + let example_path = root.join("examples").join("two.json"); + let config_raw = fs::read_to_string(&config_path) + .unwrap_or_else(|e| panic!("failed to read {}: {e}", config_path.display())); + let example_raw = fs::read_to_string(&example_path) + .unwrap_or_else(|e| panic!("failed to read {}: {e}", example_path.display())); + + assert_eq!( + config_raw, example_raw, + "configs/bench/two.json and examples/two.json must stay byte-identical" + ); + + let v: Value = serde_json::from_str(&config_raw) + .unwrap_or_else(|e| panic!("failed to parse {} as JSON: {e}", config_path.display())); + assert_eq!(v["kind"], "neural"); + assert_eq!( + v["alpha"].as_f64(), + Some(0.03), + "two.json preserves the historical alpha used by benchmark baselines" + ); + + let experts = v["experts"] + .as_array() + .expect("two.json must contain experts array"); + assert!( + experts + .iter() + .any(|expert| expert["kind"] == "rwkv7" && expert["name"] == "rwkv7"), + "two.json must use canonical rwkv7 kind/name spelling" + ); + assert!( + experts.iter().all(|expert| expert["name"] != "rwkv"), + "two.json must not retain stale rwkv expert labels" + ); + + let fac_ctw = experts + .iter() + .find(|expert| expert["kind"] == "fac-ctw") + .expect("two.json must include the canonical factorized CTW subject"); + assert_eq!( + fac_ctw["name"], "fac-ctw", + "the canonical CTW benchmark slot must be named fac-ctw, not stale ctw" + ); + assert_eq!( + fac_ctw["encoding_bits"].as_u64(), + Some(8), + "canonical fac-ctw benchmark subject must be byte-width" + ); + assert_eq!( + fac_ctw["num_percept_bits"].as_u64(), + Some(8), + "canonical fac-ctw benchmark subject must expose the 8-bit percept width" + ); + assert_eq!( + fac_ctw["msb_first"].as_bool(), + Some(true), + "canonical fac-ctw benchmark subject must explicitly select MSB-first byte order" + ); +} + +#[test] +fn extra_suite_includes_expected_uncovered_backends() { + let v = load_example("extra.json"); + assert_eq!(v["kind"], "neural"); + let experts = v["experts"] + .as_array() + .expect("extra.json must contain experts array"); + assert!( + !experts.is_empty(), + "extra.json experts array must be non-empty" + ); + + let mut saw_mamba = false; + let mut saw_particle_fast = false; + let mut saw_sparse_match = false; + + for expert in experts { + let kind = expert["kind"].as_str().unwrap_or_default(); + match kind { + "mamba" => { + saw_mamba = true; + let method = expert["method"] + .as_str() + .expect("mamba expert must define method"); + assert!( + method.contains("policy:schedule="), + "mamba method should include an explicit schedule policy" + ); + } + "particle" => { + let spec_path = expert["spec_path"] + .as_str() + .expect("particle expert must use spec_path"); + if spec_path == "particle_fast.json" { + saw_particle_fast = true; + } + } + "sparse-match" => { + saw_sparse_match = true; + } + _ => {} + } + } + + assert!(saw_mamba, "extra.json must include a mamba expert"); + assert!( + saw_particle_fast, + "extra.json must include particle_fast.json-backed particle expert" + ); + assert!( + saw_sparse_match, + "extra.json must include a sparse-match expert" + ); +} diff --git a/tests/cli_api_parity.rs b/crates/infotheory/tests/cli_api_parity.rs similarity index 81% rename from tests/cli_api_parity.rs rename to crates/infotheory/tests/cli_api_parity.rs index ddc3ae4e..2064e2c6 100644 --- a/tests/cli_api_parity.rs +++ b/crates/infotheory/tests/cli_api_parity.rs @@ -3,9 +3,10 @@ use std::io::Write; use std::process::{Command, Stdio}; -use infotheory::{ - NcdVariant, biased_entropy_rate_bytes, cross_entropy_rate_bytes, entropy_rate_bytes, - marginal_entropy_bytes, ncd_matrix_bytes, ncd_paths, +use infotheory::api::{ + CompressionBackend, NcdVariant, empirical_entropy_bytes, try_biased_entropy_rate_bytes, + try_cross_entropy_rate_bytes, try_entropy_rate_bytes, try_ncd_matrix_bytes_backend, + try_ncd_paths_backend, }; use serde_json::Value; @@ -51,11 +52,11 @@ fn assert_close(actual: f64, expected: f64, tol: f64, label: &str) { ); } -fn rosa_distance_like_cli(x: &[u8], y: &[u8], max_order: i64) -> f64 { - let h_x_x = biased_entropy_rate_bytes(x, max_order); - let h_y_y = biased_entropy_rate_bytes(y, max_order); - let h_y_x = cross_entropy_rate_bytes(x, y, max_order); - let h_x_y = cross_entropy_rate_bytes(y, x, max_order); +fn rosa_distance_like_cli(x: &[u8], y: &[u8]) -> f64 { + let h_x_x = try_biased_entropy_rate_bytes(x).expect("biased entropy x"); + let h_y_y = try_biased_entropy_rate_bytes(y).expect("biased entropy y"); + let h_y_x = try_cross_entropy_rate_bytes(x, y).expect("cross entropy y|x"); + let h_x_y = try_cross_entropy_rate_bytes(y, x).expect("cross entropy x|y"); if h_x_x < 1e-9 || h_y_y < 1e-9 { return 1.0; } @@ -65,16 +66,14 @@ fn rosa_distance_like_cli(x: &[u8], y: &[u8], max_order: i64) -> f64 { #[test] fn metrics_text_parity_with_library() { let text = "entropy parity text"; - let max_order = 5; let out = run_batch(&serde_json::json!({ "op": "metrics", "text": text, - "max_order": max_order, })); let data = text.as_bytes(); - let h0 = marginal_entropy_bytes(data); - let h_rate = entropy_rate_bytes(data, max_order); + let h0 = empirical_entropy_bytes(data); + let h_rate = try_entropy_rate_bytes(data).expect("h_rate"); let id = ((h0 - h_rate) / h0).clamp(0.0, 1.0); assert_close(as_f64(&out, "h0"), h0, 1e-6, "h0"); @@ -94,11 +93,10 @@ fn metrics_file_parity_with_library() { let out = run_batch(&serde_json::json!({ "op": "metrics_file", "path": fixture, - "max_order": 3, })); - let h0 = marginal_entropy_bytes(&bytes); - let h_rate = entropy_rate_bytes(&bytes, 3); + let h0 = empirical_entropy_bytes(&bytes); + let h_rate = try_entropy_rate_bytes(&bytes).expect("h_rate"); let id = if h0 < 1e-9 { 0.0 } else { @@ -126,7 +124,8 @@ fn ncd_file_parity_with_library() { "method": "5", "variant": "vitanyi", })); - let rust_val = ncd_paths(a, b, "5", NcdVariant::Vitanyi); + let backend = CompressionBackend::zpaq("5"); + let rust_val = try_ncd_paths_backend(a, b, &backend, NcdVariant::Vitanyi).expect("ncd"); assert_close(as_f64(&out, "ncd"), rust_val, 1e-6, "ncd"); } @@ -134,14 +133,12 @@ fn ncd_file_parity_with_library() { fn cross_entropy_parity_with_library() { let x = "abracadabra"; let y = "alakazam"; - let max_order = 3; let out = run_batch(&serde_json::json!({ "op": "cross_entropy", "text_x": x, "text_y": y, - "max_order": max_order, })); - let rust_val = cross_entropy_rate_bytes(x.as_bytes(), y.as_bytes(), max_order); + let rust_val = try_cross_entropy_rate_bytes(x.as_bytes(), y.as_bytes()).expect("cross entropy"); assert_close( as_f64(&out, "cross_entropy"), rust_val, @@ -153,11 +150,9 @@ fn cross_entropy_parity_with_library() { #[test] fn batch_metrics_parity_with_library() { let texts = vec!["abracadabra", "alakazam", "xyzxyz"]; - let max_order = 4; let out = run_batch(&serde_json::json!({ "op": "batch_metrics", "texts": texts, - "max_order": max_order, })); let rows = out .get("results") @@ -167,8 +162,8 @@ fn batch_metrics_parity_with_library() { for (idx, text) in texts.iter().enumerate() { let data = text.as_bytes(); - let h0 = marginal_entropy_bytes(data); - let h_rate = entropy_rate_bytes(data, max_order); + let h0 = empirical_entropy_bytes(data); + let h_rate = try_entropy_rate_bytes(data).expect("h_rate"); let id = if h0 < 1e-9 { 0.0 } else { @@ -200,7 +195,11 @@ fn ncd_matrix_parity_with_library() { .and_then(Value::as_array) .expect("missing matrix"); assert_eq!(matrix.len(), n); - let rust_flat = ncd_matrix_bytes(&datas, "5", NcdVariant::SymVitanyi); + let backend = CompressionBackend::zpaq("5") + .compile() + .expect("compile zpaq backend"); + let rust_flat = try_ncd_matrix_bytes_backend(&datas, &backend, NcdVariant::SymVitanyi) + .expect("ncd matrix bytes"); for i in 0..n { let row = matrix[i].as_array().expect("row must be array"); for j in 0..n { @@ -213,11 +212,9 @@ fn ncd_matrix_parity_with_library() { #[test] fn rosa_matrix_parity_with_library_formula() { let texts = vec!["abracadabra", "alakazam", "xyzxyz"]; - let max_order = 3; let out = run_batch(&serde_json::json!({ "op": "rosa_matrix", "texts": texts, - "max_order": max_order, })); let n = out.get("n").and_then(Value::as_u64).expect("missing n") as usize; let matrix = out @@ -233,7 +230,7 @@ fn rosa_matrix_parity_with_library_formula() { let expected = if i == j { 0.0 } else { - rosa_distance_like_cli(texts[i].as_bytes(), texts[j].as_bytes(), max_order) + rosa_distance_like_cli(texts[i].as_bytes(), texts[j].as_bytes()) }; assert_close(val, expected, 1e-6, "rosa_matrix"); } diff --git a/crates/infotheory/tests/cli_commands.rs b/crates/infotheory/tests/cli_commands.rs new file mode 100644 index 00000000..07e656b6 --- /dev/null +++ b/crates/infotheory/tests/cli_commands.rs @@ -0,0 +1,1481 @@ +#![cfg(all(feature = "cli", feature = "all-backends"))] + +use infotheory::api::{empirical_entropy_bytes, try_entropy_rate_bytes}; +use serde_json::{Value, json}; +use std::fs; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::process::{Command, Output, Stdio}; +use std::time::{SystemTime, UNIX_EPOCH}; + +fn temp_path(name: &str, ext: &str) -> PathBuf { + let ts = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("clock") + .as_nanos(); + std::env::temp_dir().join(format!("infotheory_cli_{name}_{ts}.{ext}")) +} + +fn write_temp_file(path: &Path, bytes: &[u8]) { + fs::write(path, bytes).expect("write temp file"); +} + +fn run_cli(args: &[&str], stdin_bytes: Option<&[u8]>) -> Output { + let bin = env!("CARGO_BIN_EXE_infotheory"); + let mut child = Command::new(bin) + .args(args) + .stdin(if stdin_bytes.is_some() { + Stdio::piped() + } else { + Stdio::null() + }) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("spawn cli"); + + if let Some(bytes) = stdin_bytes { + child + .stdin + .as_mut() + .expect("stdin") + .write_all(bytes) + .expect("write stdin"); + } + + child.wait_with_output().expect("wait cli") +} + +fn stdout_string(output: &Output) -> String { + String::from_utf8(output.stdout.clone()).expect("stdout utf8") +} + +fn stderr_string(output: &Output) -> String { + String::from_utf8(output.stderr.clone()).expect("stderr utf8") +} + +fn parse_stdout_f64(output: &Output) -> f64 { + stdout_string(output) + .trim() + .parse::() + .expect("stdout should be numeric") +} + +fn parse_stdout_json_lines(output: &Output) -> Vec { + stdout_string(output) + .lines() + .map(|line| serde_json::from_str::(line).expect("stdout line should be JSON")) + .collect() +} + +fn assert_close(actual: f64, expected: f64, tol: f64, label: &str) { + let delta = (actual - expected).abs(); + assert!( + delta <= tol, + "{label} mismatch: actual={actual}, expected={expected}, delta={delta}, tol={tol}" + ); +} + +#[test] +fn cli_usage_and_unknown_primitive_paths_are_stable() { + let no_args = run_cli(&[], None); + assert!(no_args.status.success()); + assert!(stderr_string(&no_args).contains("InfoTheory CLI")); + + let help = run_cli(&["--help"], None); + assert!(help.status.success()); + let help_text = stderr_string(&help); + assert!(help_text.contains("Usage:")); + assert!(help_text.contains("infotheory [args...] [options]")); + assert!(help_text.contains("infotheory help [topic]")); + assert!(help_text.contains("Common backend options")); + assert!(help_text.contains("Topics:")); + assert!(!help_text.contains("--exec-config")); + assert!(!help_text.contains("--cpu-affinity")); + assert!(!help_text.contains("--rss-mode")); + assert!(!help_text.contains("--determinism-deadline-certificate")); + assert!(!help_text.contains("--rwkv-export")); + + let tune_help = run_cli(&["help", "tune"], None); + assert!(tune_help.status.success()); + let tune_help_text = stderr_string(&tune_help); + assert!(tune_help_text.contains("InfoTheory tuner")); + assert!(tune_help_text.contains("--exec-config")); + assert!(tune_help_text.contains("--cpu-affinity")); + assert!(tune_help_text.contains("--rss-mode")); + assert!(tune_help_text.contains("--determinism-deadline-certificate")); + + let ncd_help = run_cli(&["ncd", "--help"], None); + assert!(ncd_help.status.success()); + let ncd_help_text = stderr_string(&ncd_help); + assert!(ncd_help_text.contains("InfoTheory compression and NCD")); + assert!(ncd_help_text.contains("rate-ac")); + + let nested_help = run_cli(&["warmstart", "teacher", "--help"], None); + assert!(nested_help.status.success()); + assert!(stderr_string(&nested_help).contains("InfoTheory warm-start teacher tools")); + + let a_path = temp_path("unknown_a", "txt"); + let b_path = temp_path("unknown_b", "txt"); + write_temp_file(&a_path, b"a"); + write_temp_file(&b_path, b"b"); + let unknown = run_cli( + &[ + "definitely-unknown-primitive", + a_path.to_string_lossy().as_ref(), + b_path.to_string_lossy().as_ref(), + ], + None, + ); + assert!(unknown.status.success()); + let stderr = stderr_string(&unknown); + assert!(stderr.contains("Unknown primitive")); + assert!(stderr.contains("InfoTheory CLI")); + + let _ = fs::remove_file(a_path); + let _ = fs::remove_file(b_path); +} + +#[test] +fn cli_rejects_removed_rwkv_export_flag() { + let input_path = temp_path("removed_rwkv_export", "txt"); + write_temp_file(&input_path, b"abracadabra"); + let input = input_path.to_string_lossy().to_string(); + let out = run_cli( + &[ + "h", + input.as_str(), + "--rwkv-export", + "/tmp/model.safetensors", + ], + None, + ); + assert!( + !out.status.success(), + "removed flag should fail: stderr={}", + stderr_string(&out) + ); + assert!( + stderr_string(&out).contains("--rwkv-export has been removed; use --model-export instead") + ); + + let _ = fs::remove_file(input_path); +} + +#[test] +fn direct_cli_primitives_cover_empirical_and_backend_paths() { + let a_path = temp_path("a", "txt"); + let b_path = temp_path("b", "txt"); + write_temp_file(&a_path, b"abracadabra abracadabra abracadabra"); + write_temp_file(&b_path, b"alakazam alakazam alakazam"); + let a = a_path.to_string_lossy().to_string(); + let b = b_path.to_string_lossy().to_string(); + + let single_file_cases = [ + vec!["h", a.as_str()], + vec![ + "h_rate", + a.as_str(), + "--rate-backend", + "ctw", + "--method", + "8", + ], + vec!["id", a.as_str(), "--rate-backend", "ctw", "--method", "8"], + ]; + for args in single_file_cases { + let output = run_cli(&args, None); + assert!( + output.status.success(), + "single-file cli failed: args={args:?}, stderr={}", + stderr_string(&output) + ); + let value = parse_stdout_f64(&output); + assert!(value.is_finite(), "args={args:?}"); + } + + let pair_cases = [ + vec!["mi", a.as_str(), b.as_str()], + vec![ + "mi", + a.as_str(), + b.as_str(), + "--rate-backend", + "ctw", + "--method", + "8", + ], + vec!["xe", a.as_str(), b.as_str()], + vec![ + "xe", + a.as_str(), + b.as_str(), + "--rate-backend", + "ctw", + "--method", + "8", + ], + vec!["ce", a.as_str(), b.as_str()], + vec!["joint_entropy", a.as_str(), b.as_str()], + vec![ + "joint_entropy", + a.as_str(), + b.as_str(), + "--rate-backend", + "ctw", + "--method", + "8", + ], + vec!["ned", a.as_str(), b.as_str()], + vec![ + "ned", + a.as_str(), + b.as_str(), + "--rate-backend", + "ctw", + "--method", + "8", + ], + vec!["ned_cons", a.as_str(), b.as_str()], + vec!["nte", a.as_str(), b.as_str()], + vec![ + "rt", + a.as_str(), + b.as_str(), + "--rate-backend", + "ctw", + "--method", + "8", + ], + vec!["tvd", a.as_str(), b.as_str()], + vec!["nhd", a.as_str(), b.as_str()], + vec!["kl", a.as_str(), b.as_str()], + vec!["js", a.as_str(), b.as_str()], + vec![ + "ncd", + a.as_str(), + b.as_str(), + "--compression-backend", + "rate-ac", + "--rate-backend", + "ctw", + "--method", + "8", + ], + vec!["ncd_sym_cons", a.as_str(), b.as_str(), "5"], + ]; + for args in pair_cases { + let output = run_cli(&args, None); + assert!( + output.status.success(), + "pair cli failed: args={args:?}, stderr={}", + stderr_string(&output) + ); + let value = parse_stdout_f64(&output); + assert!(value.is_finite(), "args={args:?}"); + } + + let _ = fs::remove_file(a_path); + let _ = fs::remove_file(b_path); +} + +#[test] +fn search_cli_supports_prior_modes_and_granularity_flags() { + let target_root = temp_path("search_target", "dir"); + let prior_root = temp_path("search_prior", "dir"); + fs::create_dir_all(&target_root).expect("create target dir"); + fs::create_dir_all(&prior_root).expect("create prior dir"); + + write_temp_file( + &target_root.join("match.txt"), + b"needle exact search phrase\nneedle exact search phrase\n", + ); + write_temp_file( + &target_root.join("other.txt"), + b"unrelated background text without the exact phrase\n", + ); + write_temp_file( + &prior_root.join("prior.txt"), + b"needle prior corpus bytes\nexact search phrase prior bytes\n", + ); + + let target = target_root.to_string_lossy().to_string(); + let prior = prior_root.to_string_lossy().to_string(); + for mode in ["use", "summarize", "none"] { + let output = run_cli( + &[ + "search", + "needle exact search phrase", + target.as_str(), + "--level", + "file", + "--prior", + prior.as_str(), + "--stage2-prior-mode", + mode, + "--top-k", + "2", + "--rate-backend", + "ctw", + "--method", + "8", + ], + None, + ); + assert!( + output.status.success(), + "search failed for mode {mode}: {}", + stderr_string(&output) + ); + let stdout = stdout_string(&output); + assert!(stdout.contains("sed -n")); + assert!(stdout.contains("match.txt")); + } + + let _ = fs::remove_dir_all(target_root); + let _ = fs::remove_dir_all(prior_root); +} + +#[test] +fn compress_and_decompress_roundtrip_for_zpaq_and_rate_backends() { + let input_path = temp_path("compress_input", "bin"); + let zpaq_out = temp_path("compress_zpaq", "itc"); + let zpaq_roundtrip = temp_path("compress_zpaq_roundtrip", "bin"); + let rate_out = temp_path("compress_rate", "itc"); + let rate_roundtrip = temp_path("compress_rate_roundtrip", "bin"); + let payload = b"roundtrip payload for compress/decompress command coverage"; + write_temp_file(&input_path, payload); + let input = input_path.to_string_lossy().to_string(); + + let zpaq_compress = run_cli( + &[ + "compress", + input.as_str(), + zpaq_out.to_string_lossy().as_ref(), + "--compression-backend", + "zpaq", + "--method", + "5", + ], + None, + ); + assert!( + zpaq_compress.status.success(), + "{}", + stderr_string(&zpaq_compress) + ); + assert!(stdout_string(&zpaq_compress).contains("compressed")); + + let zpaq_decompress = run_cli( + &[ + "decompress", + zpaq_out.to_string_lossy().as_ref(), + zpaq_roundtrip.to_string_lossy().as_ref(), + "--compression-backend", + "zpaq", + "--method", + "5", + ], + None, + ); + assert!( + zpaq_decompress.status.success(), + "{}", + stderr_string(&zpaq_decompress) + ); + assert_eq!(fs::read(&zpaq_roundtrip).expect("zpaq roundtrip"), payload); + + let rate_compress = run_cli( + &[ + "compress", + input.as_str(), + rate_out.to_string_lossy().as_ref(), + "--compression-backend", + "rate-ac", + "--rate-backend", + "ctw", + "--method", + "8", + ], + None, + ); + assert!( + rate_compress.status.success(), + "{}", + stderr_string(&rate_compress) + ); + + let rate_decompress = run_cli( + &[ + "decompress", + rate_out.to_string_lossy().as_ref(), + rate_roundtrip.to_string_lossy().as_ref(), + "--compression-backend", + "rate-ac", + "--rate-backend", + "ctw", + "--method", + "8", + ], + None, + ); + assert!( + rate_decompress.status.success(), + "{}", + stderr_string(&rate_decompress) + ); + assert_eq!(fs::read(&rate_roundtrip).expect("rate roundtrip"), payload); + + let _ = fs::remove_file(input_path); + let _ = fs::remove_file(zpaq_out); + let _ = fs::remove_file(zpaq_roundtrip); + let _ = fs::remove_file(rate_out); + let _ = fs::remove_file(rate_roundtrip); +} + +#[test] +fn sequitur_debug_accepts_hex_and_file_input() { + let output = run_cli( + &[ + "sequitur-debug", + "--hex", + "616263616263", + "--context-bytes", + "32", + "--alphabet-prefix", + "8", + ], + None, + ); + assert!(output.status.success(), "{}", stderr_string(&output)); + let parsed: Value = serde_json::from_slice(&output.stdout).expect("sequitur json"); + assert_eq!( + parsed.get("context_bytes").and_then(Value::as_u64), + Some(32) + ); + assert_eq!( + parsed.get("alphabet_prefix").and_then(Value::as_u64), + Some(8) + ); + assert_eq!( + parsed + .get("cases") + .and_then(Value::as_array) + .expect("cases") + .len(), + 1 + ); + + let input_path = temp_path("sequitur_input", "bin"); + write_temp_file(&input_path, b"abcabc"); + let file_output = run_cli( + &[ + "sequitur-debug", + input_path.to_string_lossy().as_ref(), + "--context-bytes", + "16", + ], + None, + ); + assert!( + file_output.status.success(), + "{}", + stderr_string(&file_output) + ); + let parsed: Value = serde_json::from_slice(&file_output.stdout).expect("sequitur file json"); + assert_eq!( + parsed.get("context_bytes").and_then(Value::as_u64), + Some(16) + ); + + let _ = fs::remove_file(input_path); +} + +#[test] +fn batch_cli_stream_handles_mixed_lines_with_error_isolation() { + let fixture_path = temp_path("batch_fixture", "txt"); + let fixture_bytes = b"batch fixture entropy text"; + write_temp_file(&fixture_path, fixture_bytes); + + let missing_path = temp_path("batch_missing", "txt"); + let fixture_str = fixture_path.to_string_lossy().to_string(); + let missing_str = missing_path.to_string_lossy().to_string(); + + let line1 = serde_json::json!({ + "op": "metrics", + "text": "batch fixture entropy text" + }) + .to_string(); + let line3 = serde_json::json!({ + "op": "metrics_file", + "path": fixture_str + }) + .to_string(); + let line4 = serde_json::json!({ + "op": "metrics_file", + "path": missing_str + }) + .to_string(); + let line6 = serde_json::json!({ + "op": "batch_metrics", + "texts": ["abcabc", ""] + }) + .to_string(); + + let input = [ + line1, + "{ invalid".to_string(), + line3, + line4, + serde_json::json!({ "op": "nope" }).to_string(), + line6, + "".to_string(), + ] + .join("\n") + + "\n"; + + let output = run_cli(&["batch"], Some(input.as_bytes())); + assert!(output.status.success(), "{}", stderr_string(&output)); + + let lines = parse_stdout_json_lines(&output); + assert_eq!( + lines.len(), + 7, + "batch should emit one JSON row per input line" + ); + + let h0 = empirical_entropy_bytes(fixture_bytes); + let h_rate = try_entropy_rate_bytes(fixture_bytes).expect("h_rate"); + let id = ((h0 - h_rate) / h0).clamp(0.0, 1.0); + + assert_close( + lines[0]["h0"].as_f64().expect("line 1 h0"), + h0, + 1e-5, + "line 1 h0", + ); + assert_close( + lines[0]["h_rate"].as_f64().expect("line 1 h_rate"), + h_rate, + 1e-5, + "line 1 h_rate", + ); + assert_close( + lines[0]["id"].as_f64().expect("line 1 id"), + id, + 1e-5, + "line 1 id", + ); + assert_eq!(lines[0]["len"].as_u64(), Some(fixture_bytes.len() as u64)); + + assert!( + lines[1]["error"] + .as_str() + .expect("line 2 error") + .contains("invalid json"), + "line 2 should report malformed JSON" + ); + + assert_close( + lines[2]["h0"].as_f64().expect("line 3 h0"), + h0, + 1e-5, + "line 3 h0", + ); + assert_close( + lines[2]["h_rate"].as_f64().expect("line 3 h_rate"), + h_rate, + 1e-5, + "line 3 h_rate", + ); + assert_close( + lines[2]["id"].as_f64().expect("line 3 id"), + id, + 1e-5, + "line 3 id", + ); + assert_eq!(lines[2]["len"].as_u64(), Some(fixture_bytes.len() as u64)); + + assert!( + lines[3]["error"] + .as_str() + .expect("line 4 error") + .contains("failed to read file"), + "line 4 should report read failure for missing path" + ); + + assert_eq!(lines[4]["error"], "unknown op: nope"); + + let batch_results = lines[5]["results"] + .as_array() + .expect("line 6 results array"); + assert_eq!(batch_results.len(), 2); + assert_eq!(batch_results[1]["len"], 0); + assert_eq!(batch_results[1]["h_rate"], 0); + + assert_eq!(lines[6]["error"], "empty input"); + + let _ = fs::remove_file(fixture_path); +} + +#[test] +fn search_cli_default_and_topk_paths_return_relevant_hit() { + let target_root = temp_path("search_default_target", "dir"); + fs::create_dir_all(&target_root).expect("create target dir"); + + write_temp_file( + &target_root.join("match.txt"), + b"needle exact CLI search phrase\nneedle exact CLI search phrase\n", + ); + write_temp_file( + &target_root.join("other.txt"), + b"background text without the exact phrase\n", + ); + + let target = target_root.to_string_lossy().to_string(); + + let default_output = run_cli( + &["search", "needle exact CLI search phrase", target.as_str()], + None, + ); + assert!( + default_output.status.success(), + "{}", + stderr_string(&default_output) + ); + let default_stdout = stdout_string(&default_output); + assert!(default_stdout.contains("sed -n")); + assert!(default_stdout.contains("match.txt")); + + let topk_output = run_cli( + &[ + "search", + "needle exact CLI search phrase", + target.as_str(), + "--level", + "file", + "--top-k", + "1", + "--rate-backend", + "ctw", + "--method", + "8", + ], + None, + ); + assert!( + topk_output.status.success(), + "{}", + stderr_string(&topk_output) + ); + let topk_stdout = stdout_string(&topk_output); + let topk_lines: Vec<&str> = topk_stdout + .lines() + .filter(|line| !line.trim().is_empty()) + .collect(); + assert_eq!( + topk_lines.len(), + 1, + "--top-k 1 should emit one search command" + ); + assert!(topk_lines[0].contains("match.txt")); + + let _ = fs::remove_dir_all(target_root); +} + +#[test] +fn search_cli_reports_argument_and_target_failures() { + let missing_target = run_cli(&["search", "needle only"], None); + assert!(!missing_target.status.success()); + assert!( + stderr_string(&missing_target).contains("Error: 'search' requires query and target path") + ); + + let nonexistent = temp_path("missing_search_target", "dir"); + let nonexistent_str = nonexistent.to_string_lossy().to_string(); + let missing_target_path = run_cli( + &[ + "search", + "needle exact phrase", + nonexistent_str.as_str(), + "--level", + "file", + "--rate-backend", + "ctw", + "--method", + "8", + ], + None, + ); + assert!(!missing_target_path.status.success()); + assert!(stderr_string(&missing_target_path).contains("Error: search failed")); +} + +#[test] +fn search_cli_handles_expert_spec_unknown_flags_and_invalid_stage2_mode() { + let target_root = temp_path("search_expert_target", "dir"); + fs::create_dir_all(&target_root).expect("create target dir"); + write_temp_file( + &target_root.join("match.txt"), + b"needle exact expert-spec query phrase\n", + ); + + let expert_spec_path = temp_path("search_expert_spec", "json"); + write_temp_file( + &expert_spec_path, + br#"{"name":"leaf","kind":"ctw","depth":8,"log_prior":0.0}"#, + ); + + let target = target_root.to_string_lossy().to_string(); + let expert = expert_spec_path.to_string_lossy().to_string(); + let output = run_cli( + &[ + "search", + "needle exact expert-spec query phrase", + target.as_str(), + "--stage2-prior-mode", + "unknown-mode", + "--bogus-flag", + "ignored", + "--expert-spec", + expert.as_str(), + "--level", + "file", + "--top-k", + "1", + ], + None, + ); + assert!(output.status.success()); + let stdout = stdout_string(&output); + assert!(stdout.contains("sed -n")); + assert!(stdout.contains("match.txt")); + + let _ = fs::remove_file(expert_spec_path); + let _ = fs::remove_dir_all(target_root); +} + +#[test] +fn ac_log_loss_reports_mixture_load_failures() { + let input_path = temp_path("ac_log_loss_input", "txt"); + write_temp_file(&input_path, b"ac log loss fixture bytes"); + let input = input_path.to_string_lossy().to_string(); + let missing_spec = temp_path("ac_log_loss_missing", "json"); + let missing = missing_spec.to_string_lossy().to_string(); + let out_prefix = temp_path("ac_log_loss_out", "prefix"); + let out = out_prefix.to_string_lossy().to_string(); + + let output = run_cli( + &[ + "ac-log-loss", + input.as_str(), + "--mixture", + missing.as_str(), + "--out-prefix", + out.as_str(), + ], + None, + ); + assert!(!output.status.success()); + assert!( + stderr_string(&output).contains("Error: failed to load mixture spec"), + "stderr={}", + stderr_string(&output) + ); + + let _ = fs::remove_file(input_path); +} + +#[test] +fn generate_cli_rejects_invalid_numeric_sampling_flags() { + let prompt_path = temp_path("generate_invalid_flags", "txt"); + write_temp_file(&prompt_path, b"prompt bytes for invalid flag validation"); + let prompt = prompt_path.to_string_lossy().to_string(); + + let invalid_temperature = run_cli( + &[ + "generate", + prompt.as_str(), + "--rate-backend", + "ctw", + "--method", + "8", + "--sample", + "--temperature", + "-1", + ], + None, + ); + assert!(!invalid_temperature.status.success()); + assert!( + stderr_string(&invalid_temperature) + .contains("Error: --temperature must be finite and non-negative") + ); + + let invalid_top_p = run_cli( + &[ + "generate", + prompt.as_str(), + "--rate-backend", + "ctw", + "--method", + "8", + "--sample", + "--top-p", + "1.5", + ], + None, + ); + assert!(!invalid_top_p.status.success()); + assert!(stderr_string(&invalid_top_p).contains("Error: --top-p must be in (0, 1]")); + + let invalid_temperature_parse = run_cli( + &[ + "generate", + prompt.as_str(), + "--rate-backend", + "ctw", + "--method", + "8", + "--sample", + "--temperature", + "nan-not-a-number", + ], + None, + ); + assert!(!invalid_temperature_parse.status.success()); + assert!( + stderr_string(&invalid_temperature_parse) + .contains("Error: --temperature must be a finite number") + ); + + let invalid_seed = run_cli( + &[ + "generate", + prompt.as_str(), + "--rate-backend", + "ctw", + "--method", + "8", + "--seed", + "not-a-u64", + ], + None, + ); + assert!(!invalid_seed.status.success()); + assert!(stderr_string(&invalid_seed).contains("Error: --seed must be an unsigned integer")); + + let invalid_top_k = run_cli( + &[ + "generate", + prompt.as_str(), + "--rate-backend", + "ctw", + "--method", + "8", + "--sample", + "--top-k", + "not-a-usize", + ], + None, + ); + assert!(!invalid_top_k.status.success()); + assert!( + stderr_string(&invalid_top_k).contains("Error: --top-k must be a non-negative integer") + ); + + let invalid_top_p_parse = run_cli( + &[ + "generate", + prompt.as_str(), + "--rate-backend", + "ctw", + "--method", + "8", + "--sample", + "--top-p", + "not-a-float", + ], + None, + ); + assert!(!invalid_top_p_parse.status.success()); + assert!( + stderr_string(&invalid_top_p_parse).contains("Error: --top-p must be a number in (0, 1]") + ); + + let adaptive = run_cli( + &[ + "generate", + prompt.as_str(), + "--rate-backend", + "ctw", + "--method", + "8", + "--bytes", + "4", + "--adaptive", + ], + None, + ); + assert!(adaptive.status.success()); + assert_eq!(adaptive.stdout.len(), 4); + + let invalid_bytes = run_cli( + &[ + "generate", + prompt.as_str(), + "--rate-backend", + "ctw", + "--method", + "8", + "--bytes", + "-1", + ], + None, + ); + assert!(!invalid_bytes.status.success()); + assert!( + stderr_string(&invalid_bytes).contains("Error: --bytes must be a non-negative integer") + ); + + let _ = fs::remove_file(prompt_path); +} + +#[test] +fn sequitur_debug_rejects_invalid_numeric_flags_and_hex_payloads() { + let invalid_context = run_cli( + &[ + "sequitur-debug", + "--hex", + "616263", + "--context-bytes", + "abc", + ], + None, + ); + assert!(!invalid_context.status.success()); + assert!( + stderr_string(&invalid_context) + .contains("Error: --context-bytes must be a positive integer") + ); + + let invalid_prefix = run_cli( + &[ + "sequitur-debug", + "--hex", + "616263", + "--alphabet-prefix", + "abc", + ], + None, + ); + assert!(!invalid_prefix.status.success()); + assert!( + stderr_string(&invalid_prefix) + .contains("Error: --alphabet-prefix must be a positive integer") + ); + + let invalid_hex = run_cli(&["sequitur-debug", "--hex", "xyz"], None); + assert!(!invalid_hex.status.success()); + assert!(stderr_string(&invalid_hex).contains("invalid --hex input")); +} + +#[test] +fn aixi_cli_requires_config_argument() { + let output = run_cli(&["aixi"], None); + assert!(!output.status.success()); + assert!(stderr_string(&output).contains("Error: 'aixi' requires config.json")); +} + +#[test] +fn aixi_cli_reports_missing_gameengine_feature_for_builtin_environments() { + let config_path = temp_path("aixi_builtin_config", "json"); + write_temp_file( + &config_path, + br#"{ + "schema_version": 1, + "kind": "planner_run", + "assets": [], + "environment": { "kind": "builtin", "name": "coin_flip" }, + "interface": { + "observation_bits": 1, + "observation_stream_len": 1, + "observation_key_mode": "full_stream", + "reward_bits": 1, + "agent_actions": 2 + }, + "controller": { + "kind": "aiqi_discounted", + "predictor": { "kind": "ctw", "depth": 8 }, + "bit_stream_semantics": { "kind": "binary_tokens" }, + "discount_gamma": 0.99, + "return_horizon": 2, + "return_bins": 8, + "augmentation_period": 2, + "baseline_exploration": 0.01 + }, + "runtime": { + "random_seed": 7, + "learn_cycles": 1, + "eval_cycles": 1, + "terminate_lifetime": 2, + "log_every": 1, + "perf": false, + "vm_perf_only": false, + "explore_epsilon": 0.0, + "explore_gamma": 1.0 + } +}"#, + ); + let config = config_path.to_string_lossy().to_string(); + + let output = run_cli(&["aixi", config.as_str()], None); + assert!(!output.status.success()); + assert!( + stderr_string(&output).contains("requires feature 'aixi-gameengine'"), + "stderr={}", + stderr_string(&output) + ); + + let _ = fs::remove_file(config_path); +} + +#[test] +fn decompress_cli_reports_corruption_instead_of_succeeding() { + let input_path = temp_path("corrupt_input", "bin"); + let compressed_path = temp_path("corrupt_compressed", "itc"); + let corrupt_path = temp_path("corrupt_truncated", "itc"); + let output_path = temp_path("corrupt_output", "bin"); + + let payload = b"payload for corruption failure contract"; + write_temp_file(&input_path, payload); + + let compress = run_cli( + &[ + "compress", + input_path.to_string_lossy().as_ref(), + compressed_path.to_string_lossy().as_ref(), + "--compression-backend", + "zpaq", + "--method", + "5", + ], + None, + ); + assert!(compress.status.success(), "{}", stderr_string(&compress)); + + let mut compressed = fs::read(&compressed_path).expect("read compressed artifact"); + assert!( + compressed.len() > 8, + "compressed payload too small for truncation test" + ); + compressed.truncate(compressed.len() / 2); + write_temp_file(&corrupt_path, &compressed); + + let decompress = run_cli( + &[ + "decompress", + corrupt_path.to_string_lossy().as_ref(), + output_path.to_string_lossy().as_ref(), + "--compression-backend", + "zpaq", + "--method", + "5", + ], + None, + ); + assert!( + !decompress.status.success(), + "corrupted stream should fail decompression" + ); + assert!( + stderr_string(&decompress).contains("decompression failed"), + "stderr should expose decompression failure" + ); + assert!( + !output_path.exists(), + "failed decompression must not materialize output file" + ); + + let _ = fs::remove_file(input_path); + let _ = fs::remove_file(compressed_path); + let _ = fs::remove_file(corrupt_path); +} + +#[test] +fn cli_compression_backend_json_roundtrips_rate_ac_ctw() { + let cb_path = temp_path("cli_cb_spec", "json"); + let input_path = temp_path("cli_cb_in", "bin"); + let comp_path = temp_path("cli_cb_comp", "itc"); + let out_path = temp_path("cli_cb_out", "bin"); + let spec = serde_json::json!({ + "kind": "rate-ac", + "framing": "framed", + "rate_backend": { "kind": "ctw", "depth": 8 } + }); + fs::write(&cb_path, serde_json::to_vec(&spec).expect("spec json")) + .expect("write compression spec"); + write_temp_file(&input_path, b"hello world json backend roundtrip"); + + let compress = run_cli( + &[ + "compress", + input_path.to_string_lossy().as_ref(), + comp_path.to_string_lossy().as_ref(), + "--compression-backend-json", + cb_path.to_string_lossy().as_ref(), + ], + None, + ); + assert!(compress.status.success(), "{}", stderr_string(&compress)); + + let decompress = run_cli( + &[ + "decompress", + comp_path.to_string_lossy().as_ref(), + out_path.to_string_lossy().as_ref(), + "--compression-backend-json", + cb_path.to_string_lossy().as_ref(), + ], + None, + ); + assert!( + decompress.status.success(), + "{}", + stderr_string(&decompress) + ); + assert_eq!( + fs::read(&out_path).expect("read restored"), + b"hello world json backend roundtrip" + ); + + let _ = fs::remove_file(cb_path); + let _ = fs::remove_file(input_path); + let _ = fs::remove_file(comp_path); + let _ = fs::remove_file(out_path); +} + +#[test] +fn cli_rate_backend_json_with_rate_ac_compression_name_runs_h_rate() { + let rb_path = temp_path("cli_rb_spec", "json"); + let data_path = temp_path("cli_rb_data", "txt"); + let spec = serde_json::json!({ "kind": "ctw", "depth": 7 }); + fs::write(&rb_path, serde_json::to_vec(&spec).expect("rate json")).expect("write rate spec"); + write_temp_file(&data_path, b"rate json path smoke"); + + let out = run_cli( + &[ + "h_rate", + data_path.to_string_lossy().as_ref(), + "--rate-backend-json", + rb_path.to_string_lossy().as_ref(), + "--compression-backend", + "rate-ac", + ], + None, + ); + assert!(out.status.success(), "{}", stderr_string(&out)); + let v = parse_stdout_f64(&out); + assert!(v.is_finite() && v >= 0.0); + + let _ = fs::remove_file(rb_path); + let _ = fs::remove_file(data_path); +} + +#[test] +fn cli_rate_backend_json_can_be_combined_with_method_for_zpaq() { + let rb_path = temp_path("cli_rb_zpaq_rate", "json"); + let data_path = temp_path("cli_rb_zpaq_data", "txt"); + let spec = serde_json::json!({ "kind": "ctw", "depth": 7 }); + fs::write(&rb_path, serde_json::to_vec(&spec).expect("rate json")).expect("write rate spec"); + write_temp_file(&data_path, b"rate json with zpaq method"); + + let comp_path = temp_path("cli_rb_zpaq_comp", "itc"); + let out_path = temp_path("cli_rb_zpaq_out", "txt"); + + let compress = run_cli( + &[ + "compress", + data_path.to_string_lossy().as_ref(), + comp_path.to_string_lossy().as_ref(), + "--rate-backend-json", + rb_path.to_string_lossy().as_ref(), + "--compression-backend", + "zpaq", + "--method", + "5", + ], + None, + ); + assert!(compress.status.success(), "{}", stderr_string(&compress)); + + let decompress = run_cli( + &[ + "decompress", + comp_path.to_string_lossy().as_ref(), + out_path.to_string_lossy().as_ref(), + "--rate-backend-json", + rb_path.to_string_lossy().as_ref(), + "--compression-backend", + "zpaq", + "--method", + "5", + ], + None, + ); + assert!( + decompress.status.success(), + "{}", + stderr_string(&decompress) + ); + assert_eq!( + fs::read(&out_path).expect("read restored"), + b"rate json with zpaq method" + ); + + let _ = fs::remove_file(rb_path); + let _ = fs::remove_file(data_path); + let _ = fs::remove_file(comp_path); + let _ = fs::remove_file(out_path); +} + +#[test] +fn cli_rejects_compression_backend_json_with_method() { + let cb_path = temp_path("cli_cb_conflict", "json"); + let spec = serde_json::json!({ + "kind": "rate-ac", + "rate_backend": { "kind": "ctw", "depth": 4 } + }); + fs::write(&cb_path, serde_json::to_vec(&spec).expect("json")).expect("write spec"); + let data_path = temp_path("cli_cb_conflict_data", "txt"); + write_temp_file(&data_path, b"x"); + + let out = run_cli( + &[ + "h_rate", + data_path.to_string_lossy().as_ref(), + "--compression-backend-json", + cb_path.to_string_lossy().as_ref(), + "--method", + "5", + ], + None, + ); + assert!(!out.status.success()); + let err = stderr_string(&out); + assert!( + err.contains("compression-backend-json") && err.contains("--method"), + "stderr={err}" + ); + + let _ = fs::remove_file(cb_path); + let _ = fs::remove_file(data_path); +} + +#[test] +fn cli_rejects_mismatched_rate_backend_json_with_embedded_rate() { + let cb_path = temp_path("cli_cb_embed", "json"); + let rb_path = temp_path("cli_rb_mismatch", "json"); + let data_path = temp_path("cli_mismatch_data", "txt"); + let cb = serde_json::json!({ + "kind": "rate-ac", + "rate_backend": { "kind": "ctw", "depth": 8 } + }); + let rb = serde_json::json!({ "kind": "ctw", "depth": 4 }); + fs::write(&cb_path, serde_json::to_vec(&cb).unwrap()).expect("cb"); + fs::write(&rb_path, serde_json::to_vec(&rb).unwrap()).expect("rb"); + write_temp_file(&data_path, b"zzz"); + + let out = run_cli( + &[ + "h_rate", + data_path.to_string_lossy().as_ref(), + "--compression-backend-json", + cb_path.to_string_lossy().as_ref(), + "--rate-backend-json", + rb_path.to_string_lossy().as_ref(), + ], + None, + ); + assert!(!out.status.success()); + let err = stderr_string(&out); + assert!( + err.contains("does not match") || err.contains("rate-backend-json"), + "stderr={err}" + ); + + let _ = fs::remove_file(cb_path); + let _ = fs::remove_file(rb_path); + let _ = fs::remove_file(data_path); +} + +#[test] +fn cli_rejects_unknown_rate_backend_name_in_flag() { + let input_path = temp_path("cli_unknown_rate_backend_input", "txt"); + let out_path = temp_path("cli_unknown_rate_backend_output", "itc"); + write_temp_file(&input_path, b"unknown rate backend must fail"); + + let output = run_cli( + &[ + "compress", + input_path.to_string_lossy().as_ref(), + out_path.to_string_lossy().as_ref(), + "--compression-backend", + "rate-ac", + "--rate-backend", + "/tmp/not-a-backend-name.json", + ], + None, + ); + assert!(!output.status.success(), "command should fail"); + let err = stderr_string(&output); + assert!( + err.contains("--rate-backend") && err.contains("--rate-backend-json"), + "stderr={err}" + ); + + let _ = fs::remove_file(input_path); + let _ = fs::remove_file(out_path); +} + +#[test] +fn cli_rejects_unknown_compression_backend_name_in_flag() { + let input_path = temp_path("cli_unknown_compression_backend_input", "txt"); + let out_path = temp_path("cli_unknown_compression_backend_output", "itc"); + write_temp_file(&input_path, b"unknown compression backend must fail"); + + let output = run_cli( + &[ + "compress", + input_path.to_string_lossy().as_ref(), + out_path.to_string_lossy().as_ref(), + "--compression-backend", + "/tmp/not-a-compression-backend-name.json", + ], + None, + ); + assert!(!output.status.success(), "command should fail"); + let err = stderr_string(&output); + assert!( + err.contains("--compression-backend") && err.contains("--compression-backend-json"), + "stderr={err}" + ); + + let _ = fs::remove_file(input_path); + let _ = fs::remove_file(out_path); +} + +#[test] +fn cli_help_documents_warmstart_teacher_paths() { + let help = run_cli(&["help", "warmstart"], None); + assert!(help.status.success()); + let help_text = stderr_string(&help); + assert!(help_text.contains("warmstart teacher planner-run")); + assert!(help_text.contains("warmstart teacher from-jsonl")); + assert!(help_text.contains("warmstart teacher merge")); +} + +#[test] +fn cli_warmstart_subcommands_reject_missing_required_options() { + let missing_merge_teacher = run_cli( + &[ + "warmstart", + "teacher", + "merge", + "--target", + "target.json", + "--out", + "out.json", + ], + None, + ); + assert!(!missing_merge_teacher.status.success()); + let merge_err = stderr_string(&missing_merge_teacher); + assert!( + merge_err.contains("missing required --teacher"), + "stderr={merge_err}" + ); +} + +#[test] +fn cli_warmstart_from_jsonl_failure_message_is_stable() { + let target_path = temp_path("warmstart_from_jsonl_target", "json"); + let teacher_path = temp_path("warmstart_from_jsonl_teacher", "json"); + let jsonl_path = temp_path("warmstart_from_jsonl_trace", "jsonl"); + let out_path = temp_path("warmstart_from_jsonl_out", "json"); + let target = json!({ + "schema_version": 1, + "kind": "planner_run", + "assets": [{ + "id": "teacher", + "path": teacher_path.to_string_lossy(), + }], + "environment": { + "kind": "builtin", + "name": "coin_flip", + }, + "interface": { + "observation_bits": 2, + "observation_stream_len": 1, + "observation_key_mode": "full_stream", + "reward_bits": 2, + "agent_actions": 2, + }, + "controller": { + "kind": "aiqi_warmstart_exact_jh", + "predictor": { + "kind": "ctw", + "depth": 4, + }, + "return_horizon": 1, + "return_bins": 4, + "label_phase_period": 1, + "teacher_dataset_asset": "teacher", + "planner_simulations_per_step": 1, + }, + "runtime": { + "random_seed": 7, + "learn_cycles": 1, + "eval_cycles": 0, + "terminate_lifetime": 1, + "log_every": 1, + "perf": false, + "vm_perf_only": false, + "explore_epsilon": 0.0, + "explore_gamma": 1.0, + } + }); + write_temp_file( + &target_path, + &serde_json::to_vec(&target).expect("serialize planner_run target"), + ); + write_temp_file(&jsonl_path, br#"{"kind":"action","t":0,"action":0}"#); + + let output = run_cli( + &[ + "warmstart", + "teacher", + "from-jsonl", + "--target", + target_path.to_string_lossy().as_ref(), + "--jsonl", + jsonl_path.to_string_lossy().as_ref(), + "--out", + out_path.to_string_lossy().as_ref(), + ], + None, + ); + + assert!(!output.status.success()); + let err = stderr_string(&output); + assert!( + err.contains("Error: warmstart failed: invalid telemetry: cannot infer JSONL action/percept convention"), + "stderr={err}" + ); + assert!(!out_path.exists()); + + let _ = fs::remove_file(target_path); + let _ = fs::remove_file(teacher_path); + let _ = fs::remove_file(jsonl_path); + let _ = fs::remove_file(out_path); +} diff --git a/tests/cli_golden.rs b/crates/infotheory/tests/cli_golden.rs similarity index 87% rename from tests/cli_golden.rs rename to crates/infotheory/tests/cli_golden.rs index 2af3d20c..7a1eadb7 100644 --- a/tests/cli_golden.rs +++ b/crates/infotheory/tests/cli_golden.rs @@ -106,3 +106,20 @@ fn cli_batch_golden_outputs_match() { assert_json_close(&case.expected, &actual, tolerance, &case.name); } } + +#[test] +fn cli_warmstart_usage_error_is_stable() { + let bin = env!("CARGO_BIN_EXE_infotheory"); + let output = Command::new(bin) + .args(["warmstart", "teacher", "merge"]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .output() + .expect("spawn warmstart merge without options"); + assert!(!output.status.success()); + let stderr = String::from_utf8(output.stderr).expect("stderr utf8"); + assert!( + stderr.contains("Error: warmstart failed: missing required --teacher"), + "stderr={stderr}" + ); +} diff --git a/tests/cli_golden_cases.json b/crates/infotheory/tests/cli_golden_cases.json similarity index 88% rename from tests/cli_golden_cases.json rename to crates/infotheory/tests/cli_golden_cases.json index 523083c1..d9444424 100644 --- a/tests/cli_golden_cases.json +++ b/crates/infotheory/tests/cli_golden_cases.json @@ -1,7 +1,7 @@ [ { "name": "metrics abracadabra", - "input": {"op":"metrics","text":"abracadabra","max_order":3}, + "input": {"op":"metrics","text":"abracadabra"}, "expected": {"h0":2.040373,"h_rate":1.763318,"id":0.135787,"len":11}, "float_tolerance": 1e-6 }, @@ -13,19 +13,19 @@ }, { "name": "rosa distance abracadabra alakazam", - "input": {"op":"rosa_dist","text1":"abracadabra","text2":"alakazam","max_order":3}, - "expected": {"rosa_dist":0.360834}, + "input": {"op":"rosa_dist","text1":"abracadabra","text2":"alakazam"}, + "expected": {"rosa_dist":1.000000}, "float_tolerance": 1e-6 }, { "name": "cross entropy abracadabra alakazam", - "input": {"op":"cross_entropy","text_x":"abracadabra","text_y":"alakazam","max_order":3}, + "input": {"op":"cross_entropy","text_x":"abracadabra","text_y":"alakazam"}, "expected": {"cross_entropy":1.778111}, "float_tolerance": 1e-6 }, { "name": "batch metrics triple", - "input": {"op":"batch_metrics","texts":["abracadabra","alakazam",""],"max_order":3}, + "input": {"op":"batch_metrics","texts":["abracadabra","alakazam",""]}, "expected": { "results": [ {"h0":2.040373,"h_rate":1.763318,"id":0.135787,"len":11}, @@ -50,11 +50,11 @@ }, { "name": "rosa matrix trio", - "input": {"op":"rosa_matrix","texts":["abracadabra","alakazam","xyzxyz"],"max_order":3}, + "input": {"op":"rosa_matrix","texts":["abracadabra","alakazam","xyzxyz"]}, "expected": { "matrix": [ - [0.000000,0.360834,1.000000], - [0.360834,0.000000,1.000000], + [0.000000,1.000000,1.000000], + [1.000000,0.000000,1.000000], [1.000000,1.000000,0.000000] ], "n": 3 diff --git a/crates/infotheory/tests/cli_warmstart_dispatch.rs b/crates/infotheory/tests/cli_warmstart_dispatch.rs new file mode 100644 index 00000000..7003eab9 --- /dev/null +++ b/crates/infotheory/tests/cli_warmstart_dispatch.rs @@ -0,0 +1,28 @@ +#![cfg(all(feature = "cli", feature = "backend-ctw"))] + +use std::process::{Command, Stdio}; + +#[test] +fn warmstart_dispatch_does_not_require_default_backends() { + let output = Command::new(env!("CARGO_BIN_EXE_infotheory")) + .args(["warmstart", "teacher", "merge"]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .output() + .expect("spawn warmstart command"); + + assert!(!output.status.success()); + let stderr = String::from_utf8(output.stderr).expect("stderr utf8"); + assert!( + stderr.contains("Error: warmstart failed: missing required --teacher"), + "stderr={stderr}" + ); + assert!( + !stderr.contains("requires infotheory feature 'backend-rosa'"), + "warmstart dispatch must not build the default rate backend before parsing: {stderr}" + ); + assert!( + !stderr.contains("requires infotheory feature 'backend-zpaq'"), + "warmstart dispatch must not build the default compression backend before parsing: {stderr}" + ); +} diff --git a/crates/infotheory/tests/compiled_plan_parity.rs b/crates/infotheory/tests/compiled_plan_parity.rs new file mode 100644 index 00000000..f23d4fff --- /dev/null +++ b/crates/infotheory/tests/compiled_plan_parity.rs @@ -0,0 +1,489 @@ +#[cfg(any( + feature = "backend-ctw", + feature = "backend-mixture", + feature = "backend-zpaq", + feature = "backend-calibrated", + feature = "backend-rwkv", + feature = "backend-mamba" +))] +use infotheory::api::{ + CompressionBackend, GenerationConfig, InfotheoryCtx, NcdVariant, RateBackend, + RateBackendSession, try_compress_bytes_backend, try_decompress_bytes_backend, +}; + +#[cfg(feature = "backend-ctw")] +use infotheory::api::{ + BinaryPrediction, BitOrder, BitStreamSemantics, OnlineBitPredictor, RateBackendBitSession, +}; + +#[cfg(any( + feature = "backend-ctw", + feature = "backend-mixture", + feature = "backend-zpaq", + feature = "backend-calibrated", + feature = "backend-rwkv", + feature = "backend-mamba" +))] +fn assert_close(label: &str, left: f64, right: f64) { + let diff = (left - right).abs(); + assert!( + diff <= 1e-12, + "{label} mismatch: left={left}, right={right}, diff={diff}" + ); +} + +#[cfg(feature = "backend-ctw")] +fn assert_bit_prediction_close(label: &str, left: BinaryPrediction, right: BinaryPrediction) { + assert_close(&format!("{label} p0"), left.p0, right.p0); + assert_close(&format!("{label} p1"), left.p1, right.p1); +} + +#[cfg(feature = "backend-ctw")] +fn assert_bit_sessions_predict_same( + label: &str, + compat_session: &mut RateBackendBitSession, + compiled_session: &mut RateBackendBitSession, +) { + let compat_prediction = compat_session.predict_bit(); + let compiled_prediction = compiled_session.predict_bit(); + assert_bit_prediction_close(label, compat_prediction, compiled_prediction); +} + +#[cfg(feature = "backend-ctw")] +#[derive(Clone, Copy)] +enum BitSessionOp { + Observe(bool), + Condition(bool), + Step(bool), +} + +#[cfg(feature = "backend-ctw")] +fn apply_bit_session_op( + label: &str, + op: BitSessionOp, + compat_session: &mut RateBackendBitSession, + compiled_session: &mut RateBackendBitSession, +) { + assert_bit_sessions_predict_same( + &format!("{label} pre-update"), + compat_session, + compiled_session, + ); + match op { + BitSessionOp::Observe(bit) => { + compat_session + .try_observe_bit(bit) + .unwrap_or_else(|err| panic!("{label} compat observe failed: {err}")); + compiled_session + .try_observe_bit(bit) + .unwrap_or_else(|err| panic!("{label} compiled observe failed: {err}")); + } + BitSessionOp::Condition(bit) => { + compat_session + .try_condition_bit(bit) + .unwrap_or_else(|err| panic!("{label} compat condition failed: {err}")); + compiled_session + .try_condition_bit(bit) + .unwrap_or_else(|err| panic!("{label} compiled condition failed: {err}")); + } + BitSessionOp::Step(bit) => { + let compat_prediction = compat_session + .try_step_bit(bit) + .unwrap_or_else(|err| panic!("{label} compat step failed: {err}")); + let compiled_prediction = compiled_session + .try_step_bit(bit) + .unwrap_or_else(|err| panic!("{label} compiled step failed: {err}")); + assert_bit_prediction_close(label, compat_prediction, compiled_prediction); + } + } + assert_bit_sessions_predict_same( + &format!("{label} post-update"), + compat_session, + compiled_session, + ); +} + +#[cfg(any( + feature = "backend-ctw", + feature = "backend-mixture", + feature = "backend-zpaq", + feature = "backend-calibrated", + feature = "backend-rwkv", + feature = "backend-mamba" +))] +fn assert_ctx_parity( + rate_backend: RateBackend, + compression_backend: CompressionBackend, + check_generation: bool, +) { + let compiled_rate = rate_backend + .compile() + .unwrap_or_else(|err| panic!("compile rate backend: {err}")); + let compiled_compression = compression_backend + .compile() + .unwrap_or_else(|err| panic!("compile compression backend: {err}")); + + let compiled_ctx = InfotheoryCtx::new(compiled_rate.clone(), compiled_compression.clone()); + let compat_ctx = InfotheoryCtx::from_specs(rate_backend.clone(), compression_backend.clone()) + .unwrap_or_else(|err| panic!("compat ctx: {err}")); + + let prompt = b"compiled plan parity prompt bytes"; + let train = b"compiled plan parity training data"; + let x = b"abcabc123compiled"; + let y = b"xyzxyz456compiled"; + + assert_close( + "entropy", + compat_ctx.try_entropy_rate_bytes(prompt).unwrap(), + compiled_ctx.try_entropy_rate_bytes(prompt).unwrap(), + ); + assert_close( + "cross-entropy", + compat_ctx + .try_cross_entropy_rate_bytes(prompt, train) + .unwrap(), + compiled_ctx + .try_cross_entropy_rate_bytes(prompt, train) + .unwrap(), + ); + assert_close( + "joint-entropy", + compat_ctx.try_joint_entropy_rate_bytes(x, y).unwrap(), + compiled_ctx.try_joint_entropy_rate_bytes(x, y).unwrap(), + ); + assert_close( + "ncd", + compat_ctx.try_ncd_bytes(x, y, NcdVariant::Vitanyi).unwrap(), + compiled_ctx + .try_ncd_bytes(x, y, NcdVariant::Vitanyi) + .unwrap(), + ); + + let enc_compat = try_compress_bytes_backend(prompt, &compat_ctx.compression_backend).unwrap(); + let enc_compiled = + try_compress_bytes_backend(prompt, &compiled_ctx.compression_backend).unwrap(); + assert_eq!(enc_compat, enc_compiled, "compressed payload drift"); + let dec = + try_decompress_bytes_backend(&enc_compiled, &compiled_ctx.compression_backend).unwrap(); + assert_eq!(dec, prompt, "decompressed payload mismatch"); + + let mut compat_session = + RateBackendSession::from_spec(rate_backend, Some(train.len() as u64)).unwrap(); + let mut compiled_session = + RateBackendSession::from_backend(compiled_rate, Some(train.len() as u64)).unwrap(); + compat_session.observe(train); + compiled_session.observe(train); + let mut compat_logps = [0.0; 256]; + let mut compiled_logps = [0.0; 256]; + compat_session.fill_log_probs(&mut compat_logps); + compiled_session.fill_log_probs(&mut compiled_logps); + for (idx, (&left, &right)) in compat_logps.iter().zip(compiled_logps.iter()).enumerate() { + let diff = (left - right).abs(); + assert!( + diff <= 1e-12, + "session log-prob drift at {idx}: left={left}, right={right}, diff={diff}" + ); + } + + if check_generation { + let mut cfg = GenerationConfig::default(); + cfg.seed = 7; + let compat = compat_ctx + .try_generate_bytes_with_config(prompt, 16, cfg) + .unwrap(); + let compiled = compiled_ctx + .try_generate_bytes_with_config(prompt, 16, cfg) + .unwrap(); + assert_eq!(compat, compiled, "generation drift"); + } +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn compiled_bit_session_matches_wrapper_for_binary_tokens_and_checkpoints() { + let rate = RateBackend::Ctw { depth: 6 }; + let compiled_rate = rate + .clone() + .compile() + .unwrap_or_else(|err| panic!("compile ctw rate backend: {err}")); + let mut compat_session = + RateBackendBitSession::from_spec(rate, Some(13), BitStreamSemantics::BinaryTokens) + .expect("compat binary-token bit session"); + let mut compiled_session = RateBackendBitSession::from_backend( + compiled_rate, + Some(13), + BitStreamSemantics::BinaryTokens, + ) + .expect("compiled binary-token bit session"); + + for (idx, op) in [ + BitSessionOp::Observe(true), + BitSessionOp::Condition(false), + BitSessionOp::Step(true), + BitSessionOp::Observe(true), + BitSessionOp::Condition(true), + BitSessionOp::Step(false), + ] + .into_iter() + .enumerate() + { + apply_bit_session_op( + &format!("binary-token prefix op {idx}"), + op, + &mut compat_session, + &mut compiled_session, + ); + } + + let compat_checkpoint = compat_session.checkpoint(); + let compiled_checkpoint = compiled_session.checkpoint(); + + for (idx, op) in [ + BitSessionOp::Step(true), + BitSessionOp::Observe(false), + BitSessionOp::Condition(false), + ] + .into_iter() + .enumerate() + { + apply_bit_session_op( + &format!("binary-token divergent op {idx}"), + op, + &mut compat_session, + &mut compiled_session, + ); + } + + compat_session + .restore_checkpoint(&compat_checkpoint) + .expect("restore compat binary-token checkpoint"); + compiled_session + .restore_checkpoint(&compiled_checkpoint) + .expect("restore compiled binary-token checkpoint"); + assert_bit_sessions_predict_same( + "binary-token restored checkpoint", + &mut compat_session, + &mut compiled_session, + ); + compat_session.clear_checkpoints_if_supported(); + compiled_session.clear_checkpoints_if_supported(); + + OnlineBitPredictor::begin_bit_stream( + &mut compat_session, + Some(5), + BitStreamSemantics::BinaryTokens, + ) + .expect("compat binary-token stream restart"); + OnlineBitPredictor::begin_bit_stream( + &mut compiled_session, + Some(5), + BitStreamSemantics::BinaryTokens, + ) + .expect("compiled binary-token stream restart"); + + for (idx, bit) in [true, false, true, true, false].into_iter().enumerate() { + apply_bit_session_op( + &format!("binary-token restarted op {idx}"), + BitSessionOp::Step(bit), + &mut compat_session, + &mut compiled_session, + ); + } +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn compiled_bit_session_matches_wrapper_for_byte_packed_checkpoint_restore() { + let rate = RateBackend::Ctw { depth: 6 }; + let compiled_rate = rate + .clone() + .compile() + .unwrap_or_else(|err| panic!("compile ctw rate backend: {err}")); + let semantics = BitStreamSemantics::BytePacked { + order: BitOrder::MsbFirst, + }; + let mut compat_session = + RateBackendBitSession::from_spec(rate, Some(24), semantics).expect("compat bit session"); + let mut compiled_session = + RateBackendBitSession::from_backend(compiled_rate, Some(24), semantics) + .expect("compiled bit session"); + + for (idx, bit) in [true, false, true, false, true, false, false, true] + .into_iter() + .enumerate() + { + apply_bit_session_op( + &format!("byte-packed first byte bit {idx}"), + BitSessionOp::Observe(bit), + &mut compat_session, + &mut compiled_session, + ); + } + + for (idx, bit) in [false, true, true].into_iter().enumerate() { + apply_bit_session_op( + &format!("byte-packed checkpoint prefix bit {idx}"), + BitSessionOp::Observe(bit), + &mut compat_session, + &mut compiled_session, + ); + } + + let compat_checkpoint = compat_session.checkpoint(); + let compiled_checkpoint = compiled_session.checkpoint(); + + for (idx, bit) in [true, true, false, false, false].into_iter().enumerate() { + apply_bit_session_op( + &format!("byte-packed divergent suffix bit {idx}"), + BitSessionOp::Observe(bit), + &mut compat_session, + &mut compiled_session, + ); + } + + compat_session + .restore_checkpoint(&compat_checkpoint) + .expect("restore compat byte-packed checkpoint"); + compiled_session + .restore_checkpoint(&compiled_checkpoint) + .expect("restore compiled byte-packed checkpoint"); + assert_bit_sessions_predict_same( + "byte-packed restored mid-byte checkpoint", + &mut compat_session, + &mut compiled_session, + ); + + for (idx, bit) in [false, false, true, true, true].into_iter().enumerate() { + apply_bit_session_op( + &format!("byte-packed restored suffix bit {idx}"), + BitSessionOp::Observe(bit), + &mut compat_session, + &mut compiled_session, + ); + } + + for (idx, bit) in [true, true, false, false, true, false, true, false] + .into_iter() + .enumerate() + { + apply_bit_session_op( + &format!("byte-packed frozen byte bit {idx}"), + BitSessionOp::Condition(bit), + &mut compat_session, + &mut compiled_session, + ); + } + + compat_session.finish().expect("compat byte-packed finish"); + compiled_session + .finish() + .expect("compiled byte-packed finish"); +} + +#[cfg(feature = "backend-ctw")] +#[test] +fn compiled_ctx_matches_wrapper_ctx_for_ctw_rate_and_rate_ac() { + let rate = RateBackend::Ctw { depth: 8 }; + let compression = CompressionBackend::Rate { + rate_backend: rate.clone(), + coder: infotheory::coders::CoderType::AC, + framing: infotheory::compression::FramingMode::Framed, + }; + assert_ctx_parity(rate, compression, true); +} + +#[cfg(all( + feature = "backend-mixture", + feature = "backend-ctw", + feature = "backend-match" +))] +#[test] +fn compiled_ctx_matches_wrapper_ctx_for_mixture_backend() { + let rate = RateBackend::Mixture { + spec: std::sync::Arc::new(infotheory::api::MixtureSpec::new( + infotheory::api::MixtureKind::Bayes, + vec![ + { + let mut expert = + infotheory::api::MixtureExpertSpec::new(RateBackend::Ctw { depth: 8 }); + expert.name = Some("ctw".to_string()); + expert + }, + { + let mut expert = infotheory::api::MixtureExpertSpec::new(RateBackend::Match { + hash_bits: 18, + min_len: 4, + max_len: 64, + base_mix: 0.02, + confidence_scale: 1.0, + }); + expert.name = Some("match".to_string()); + expert.log_prior = -0.2; + expert + }, + ], + )), + }; + let compression = CompressionBackend::Rate { + rate_backend: rate.clone(), + coder: infotheory::coders::CoderType::RANS, + framing: infotheory::compression::FramingMode::Framed, + }; + assert_ctx_parity(rate, compression, false); +} + +#[cfg(all(feature = "backend-calibrated", feature = "backend-ctw"))] +#[test] +fn compiled_ctx_matches_wrapper_ctx_for_calibrated_backend() { + let rate = RateBackend::Calibrated { + spec: std::sync::Arc::new(infotheory::api::CalibratedSpec::new( + RateBackend::Ctw { depth: 8 }, + infotheory::api::CalibrationContextKind::Text, + )), + }; + let compression = CompressionBackend::Rate { + rate_backend: rate.clone(), + coder: infotheory::coders::CoderType::AC, + framing: infotheory::compression::FramingMode::Framed, + }; + assert_ctx_parity(rate, compression, false); +} + +#[cfg(feature = "backend-zpaq")] +#[test] +fn compiled_ctx_matches_wrapper_ctx_for_zpaq_rate_and_compression() { + let rate = RateBackend::Zpaq { + method: infotheory::api::ZpaqMethodSpec::literal("1"), + }; + let compression = CompressionBackend::zpaq("1"); + assert_ctx_parity(rate, compression, false); +} + +#[cfg(feature = "backend-rwkv")] +#[test] +fn compiled_ctx_matches_wrapper_ctx_for_rwkv_backends() { + let method = "cfg:hidden=64,layers=1,intermediate=64,decay_rank=8,a_rank=8,v_rank=8,g_rank=8,seed=31,train=none,lr=0.0,stride=1;policy:schedule=0..100:infer"; + let rate = RateBackend::Rwkv7Method { + method: infotheory::rwkvzip::parse_method_spec(method).expect("rwkv method spec"), + }; + let compression = CompressionBackend::Rwkv7 { + method: infotheory::rwkvzip::parse_method_spec(method).expect("rwkv method spec"), + coder: infotheory::coders::CoderType::AC, + }; + assert_ctx_parity(rate, compression, false); +} + +#[cfg(feature = "backend-mamba")] +#[test] +fn compiled_ctx_matches_wrapper_ctx_for_mamba_rate_backend() { + let method = "cfg:hidden=64,layers=1,intermediate=96,state=16,conv=4,dt_rank=16,seed=26,train=none,lr=0.0,stride=1;policy:schedule=0..100:infer"; + let rate = RateBackend::MambaMethod { + method: infotheory::mambazip::parse_method_spec(method).expect("mamba method spec"), + }; + let compression = CompressionBackend::Rate { + rate_backend: rate.clone(), + coder: infotheory::coders::CoderType::AC, + framing: infotheory::compression::FramingMode::Framed, + }; + assert_ctx_parity(rate, compression, false); +} diff --git a/crates/infotheory/tests/compression_validation.rs b/crates/infotheory/tests/compression_validation.rs new file mode 100644 index 00000000..675a47f5 --- /dev/null +++ b/crates/infotheory/tests/compression_validation.rs @@ -0,0 +1,22 @@ +#![cfg(feature = "backend-rosa")] + +use infotheory::api::{CompressionBackend, RateBackend, validate_compression_backend}; +use infotheory::coders::CoderType; +use infotheory::compression::FramingMode; + +#[test] +fn validate_compression_backend_accepts_rate_ac_and_rate_rans_without_rwkv_feature() { + let ac = CompressionBackend::Rate { + rate_backend: RateBackend::RosaPlus { max_order: -1 }, + coder: CoderType::AC, + framing: FramingMode::Framed, + }; + validate_compression_backend(&ac).expect("rate-ac validation should not panic or fail"); + + let rans = CompressionBackend::Rate { + rate_backend: RateBackend::RosaPlus { max_order: -1 }, + coder: CoderType::RANS, + framing: FramingMode::Framed, + }; + validate_compression_backend(&rans).expect("rate-rans validation should not panic or fail"); +} diff --git a/tests/fixtures/fixture_a.txt b/crates/infotheory/tests/fixtures/fixture_a.txt similarity index 100% rename from tests/fixtures/fixture_a.txt rename to crates/infotheory/tests/fixtures/fixture_a.txt diff --git a/tests/fixtures/fixture_b.txt b/crates/infotheory/tests/fixtures/fixture_b.txt similarity index 100% rename from tests/fixtures/fixture_b.txt rename to crates/infotheory/tests/fixtures/fixture_b.txt diff --git a/tests/generate_cli.rs b/crates/infotheory/tests/generate_cli.rs similarity index 85% rename from tests/generate_cli.rs rename to crates/infotheory/tests/generate_cli.rs index 4091e5ee..9e4440da 100644 --- a/tests/generate_cli.rs +++ b/crates/infotheory/tests/generate_cli.rs @@ -1,4 +1,4 @@ -#![cfg(feature = "cli")] +#![cfg(all(feature = "cli", feature = "all-backends"))] use std::fs; use std::io::Write; @@ -91,11 +91,12 @@ fn generate_cli_rosaplus_predicts_green_from_file_and_stdin() { let _ = fs::remove_file(prompt_path); } -/// When stdin is piped and the sole positional parses as an integer, -/// the CLI should interpret it as `max_order` (not try to open it as a file). +/// When stdin is piped and the sole positional parses as an integer the CLI +/// must not treat it as a file path: with all `max_order` plumbing removed, +/// the integer positional is dropped and the prompt is read from stdin. #[test] -fn generate_cli_stdin_with_max_order_positional() { - let from_stdin_with_order = run_generate( +fn generate_cli_stdin_with_integer_positional_reads_prompt_from_stdin() { + let from_stdin = run_generate( &[ "generate", "8", @@ -110,9 +111,9 @@ fn generate_cli_stdin_with_max_order_positional() { Some(PROMPT), ); assert_eq!( - from_stdin_with_order.len(), + from_stdin.len(), 4, - "should interpret '8' as max_order and read prompt from stdin" + "integer positional with piped stdin must read prompt from stdin" ); } @@ -195,7 +196,7 @@ fn generate_cli_rwkv_emits_requested_bytes() { "generate", &path_str, "--rate-backend", - "rwkv", + "rwkv7", "--method", "cfg:hidden=64,layers=1,intermediate=64,decay_rank=8,a_rank=8,v_rank=8,g_rank=8,seed=31,train=none,lr=0.0,stride=1;policy:schedule=0..100:infer", "--bytes", @@ -210,6 +211,31 @@ fn generate_cli_rwkv_emits_requested_bytes() { let _ = fs::remove_file(prompt_path); } +#[cfg(feature = "backend-mamba")] +#[test] +fn generate_cli_mamba_emits_requested_bytes() { + let prompt_path = write_temp_file("mamba_prompt", "txt", PROMPT); + let path_str = prompt_path.to_string_lossy().to_string(); + let out = run_generate( + &[ + "generate", + &path_str, + "--rate-backend", + "mamba", + "--method", + "cfg:hidden=64,layers=1,intermediate=96,state=16,conv=4,dt_rank=16,seed=26,train=none,lr=0.0,stride=1;policy:schedule=0..100:infer", + "--bytes", + "8", + "--sample", + "--seed", + "42", + ], + None, + ); + assert_eq!(out.len(), 8); + let _ = fs::remove_file(prompt_path); +} + #[test] fn generate_cli_supports_expert_spec_and_mixture_spec() { let prompt_path = write_temp_file("spec_prompt", "txt", PROMPT); @@ -252,8 +278,8 @@ fn generate_cli_supports_expert_spec_and_mixture_spec() { let experts = { let mut experts = experts; experts.push(json!({ - "name": "rwkv", - "kind": "rwkv", + "name": "rwkv7", + "kind": "rwkv7", "method": "cfg:hidden=64,layers=1,intermediate=64,decay_rank=8,a_rank=8,v_rank=8,g_rank=8,seed=31,train=none,lr=0.0,stride=1;policy:schedule=0..100:infer", "log_prior": 0.0 })); diff --git a/crates/infotheory/tests/mixture_rate_backend.rs b/crates/infotheory/tests/mixture_rate_backend.rs new file mode 100644 index 00000000..ca59b0b1 --- /dev/null +++ b/crates/infotheory/tests/mixture_rate_backend.rs @@ -0,0 +1,344 @@ +#![cfg(feature = "all-backends")] + +use infotheory::api::{ + CalibratedSpec, CalibrationContextKind, MixtureExpertSpec, MixtureKind, MixtureScheduleMode, + MixtureSpec, RateBackend, try_entropy_rate_backend as try_entropy_rate_backend_compiled, +}; +use std::sync::Arc; + +fn try_entropy_rate_backend(data: &[u8], backend: &RateBackend) -> Result { + let compiled = backend.compile().map_err(|err| err.to_string())?; + try_entropy_rate_backend_compiled(data, &compiled).map_err(|err| err.to_string()) +} + +#[test] +fn mixture_single_expert_matches_backend() { + let data = b"abababababababababababababababab"; + let base = RateBackend::Ctw { depth: 8 }; + let base_rate = try_entropy_rate_backend(data, &base).expect("base rate"); + + let spec = MixtureSpec::new( + MixtureKind::Bayes, + vec![MixtureExpertSpec::new(base.clone())], + ); + let mix_backend = RateBackend::Mixture { + spec: Arc::new(spec), + }; + let mix_rate = try_entropy_rate_backend(data, &mix_backend).expect("mix rate"); + + assert!( + (mix_rate - base_rate).abs() < 1e-6, + "mix={mix_rate} base={base_rate}" + ); +} + +#[test] +fn mixture_single_sequitur_expert_matches_backend() { + let data = b"abcabcabcabcabcabc"; + let base = RateBackend::Sequitur { context_bytes: 32 }; + let base_rate = try_entropy_rate_backend(data, &base).expect("base rate"); + + let spec = MixtureSpec::new( + MixtureKind::Bayes, + vec![MixtureExpertSpec::new(base.clone()).with_name("sequitur")], + ); + let mix_backend = RateBackend::Mixture { + spec: Arc::new(spec), + }; + let mix_rate = try_entropy_rate_backend(data, &mix_backend).expect("mix rate"); + + assert!( + (mix_rate - base_rate).abs() < 1e-6, + "mix={mix_rate} base={base_rate}" + ); +} + +#[cfg(feature = "backend-rwkv")] +#[test] +fn rwkv7_mixture_single_expert_matches_backend_with_tbptt() { + let data = b"abcdefghij"; + let base = RateBackend::Rwkv7Method { + method: infotheory::rwkvzip::parse_method_spec("cfg:hidden=64,layers=1,intermediate=64,decay_rank=8,a_rank=8,v_rank=8,g_rank=8,seed=37,train=adam,lr=0.0008,stride=1;policy:schedule=0..100:train(scope=all,opt=adam,lr=0.0008,stride=1,bptt=8,clip=0,momentum=0.9)").expect("rwkv method spec"), + }; + let base_rate = try_entropy_rate_backend(data, &base).expect("base rate"); + + let spec = MixtureSpec::new( + MixtureKind::Bayes, + vec![MixtureExpertSpec::new(base.clone()).with_name("rwkv7")], + ); + let mix_backend = RateBackend::Mixture { + spec: Arc::new(spec), + }; + let mix_rate = try_entropy_rate_backend(data, &mix_backend).expect("mix rate"); + + assert!( + (mix_rate - base_rate).abs() < 1e-6, + "mix={mix_rate} base={base_rate}" + ); +} + +#[test] +fn mixture_recursive_expert_matches_backend() { + let data = b"01010101010101010101010101010101"; + let base = RateBackend::Ctw { depth: 8 }; + let base_rate = try_entropy_rate_backend(data, &base).expect("base rate"); + + let inner = MixtureSpec::new( + MixtureKind::Bayes, + vec![MixtureExpertSpec::new(base.clone()).with_name("ctw")], + ); + let outer = MixtureSpec::new( + MixtureKind::Bayes, + vec![ + MixtureExpertSpec::new(RateBackend::Mixture { + spec: Arc::new(inner), + }) + .with_name("inner"), + ], + ); + let mix_backend = RateBackend::Mixture { + spec: Arc::new(outer), + }; + let mix_rate = try_entropy_rate_backend(data, &mix_backend).expect("mix rate"); + + assert!( + (mix_rate - base_rate).abs() < 1e-6, + "mix={mix_rate} base={base_rate}" + ); +} + +#[test] +fn neural_mixture_single_expert_matches_backend() { + let data = b"abababababababababababababababab"; + let base = RateBackend::Ctw { depth: 8 }; + let base_rate = try_entropy_rate_backend(data, &base).expect("base rate"); + + let spec = MixtureSpec::new( + MixtureKind::Neural, + vec![MixtureExpertSpec::new(base.clone())], + ) + .with_alpha(0.05); + let mix_backend = RateBackend::Mixture { + spec: Arc::new(spec), + }; + let mix_rate = try_entropy_rate_backend(data, &mix_backend).expect("mix rate"); + + assert!( + (mix_rate - base_rate).abs() < 1e-6, + "mix={mix_rate} base={base_rate}" + ); +} + +#[test] +fn convex_mixture_single_expert_matches_backend() { + let data = b"abababababababababababababababab"; + let base = RateBackend::Ctw { depth: 8 }; + let base_rate = try_entropy_rate_backend(data, &base).expect("base rate"); + + let spec = MixtureSpec::new( + MixtureKind::Convex, + vec![MixtureExpertSpec::new(base.clone())], + ) + .with_alpha(1.25); + let mix_backend = RateBackend::Mixture { + spec: Arc::new(spec), + }; + let mix_rate = try_entropy_rate_backend(data, &mix_backend).expect("mix rate"); + + assert!( + (mix_rate - base_rate).abs() < 1e-6, + "mix={mix_rate} base={base_rate}" + ); +} + +#[test] +fn switching_theorem_schedule_backend_executes() { + let data = b"abababababababababababababababab"; + let spec = MixtureSpec::new( + MixtureKind::Switching, + vec![ + MixtureExpertSpec::new(RateBackend::Ctw { depth: 8 }).with_name("ctw"), + MixtureExpertSpec::new(RateBackend::Match { + hash_bits: 18, + min_len: 3, + max_len: 96, + base_mix: 0.03, + confidence_scale: 1.0, + }) + .with_name("match"), + ], + ) + .with_schedule(MixtureScheduleMode::Theorem) + .with_alpha(0.99); + let backend = RateBackend::Mixture { + spec: Arc::new(spec), + }; + let rate = try_entropy_rate_backend(data, &backend).expect("rate"); + assert!(rate.is_finite() && rate >= 0.0, "rate={rate}"); +} + +#[test] +fn convex_theorem_schedule_backend_executes() { + let data = b"abababababababababababababababab"; + let spec = MixtureSpec::new( + MixtureKind::Convex, + vec![ + MixtureExpertSpec::new(RateBackend::Ctw { depth: 8 }).with_name("ctw"), + MixtureExpertSpec::new(RateBackend::FacCtw { + base_depth: 8, + num_percept_bits: 8, + encoding_bits: 8, + msb_first: None, + }) + .with_name("fac"), + ], + ) + .with_schedule(MixtureScheduleMode::Theorem) + .with_alpha(7.5); + let backend = RateBackend::Mixture { + spec: Arc::new(spec), + }; + let rate = try_entropy_rate_backend(data, &backend).expect("rate"); + assert!(rate.is_finite() && rate >= 0.0, "rate={rate}"); +} + +#[test] +fn neural_mixture_supports_nested_mixture_expert() { + let data = b"abracadabra abracadabra abracadabra"; + let inner = MixtureSpec::new( + MixtureKind::Bayes, + vec![ + MixtureExpertSpec::new(RateBackend::Ctw { depth: 8 }).with_name("ctw"), + MixtureExpertSpec::new(RateBackend::FacCtw { + base_depth: 8, + num_percept_bits: 8, + encoding_bits: 8, + msb_first: None, + }) + .with_name("fac"), + ], + ); + + let outer = MixtureSpec::new( + MixtureKind::Neural, + vec![ + MixtureExpertSpec::new(RateBackend::Mixture { + spec: Arc::new(inner), + }) + .with_name("nested"), + MixtureExpertSpec::new(RateBackend::Zpaq { + method: infotheory::api::ZpaqMethodSpec::literal("1"), + }) + .with_name("zpaq"), + ], + ) + .with_alpha(0.03); + + let backend = RateBackend::Mixture { + spec: Arc::new(outer), + }; + let rate = try_entropy_rate_backend(data, &backend).expect("rate"); + assert!(rate.is_finite() && rate >= 0.0, "rate={rate}"); +} + +#[test] +fn convex_mixture_supports_nested_mixture_expert() { + let data = b"abracadabra abracadabra abracadabra"; + let inner = MixtureSpec::new( + MixtureKind::Bayes, + vec![ + MixtureExpertSpec::new(RateBackend::Ctw { depth: 8 }).with_name("ctw"), + MixtureExpertSpec::new(RateBackend::FacCtw { + base_depth: 8, + num_percept_bits: 8, + encoding_bits: 8, + msb_first: None, + }) + .with_name("fac"), + ], + ); + + let outer = MixtureSpec::new( + MixtureKind::Convex, + vec![ + MixtureExpertSpec::new(RateBackend::Mixture { + spec: Arc::new(inner), + }) + .with_name("nested"), + MixtureExpertSpec::new(RateBackend::Ppmd { + order: 6, + memory_mb: 8, + }) + .with_name("ppmd"), + ], + ) + .with_alpha(1.25); + + let backend = RateBackend::Mixture { + spec: Arc::new(outer), + }; + let rate = try_entropy_rate_backend(data, &backend).expect("rate"); + assert!(rate.is_finite() && rate >= 0.0, "rate={rate}"); +} + +#[test] +fn new_backends_have_finite_entropy_rates() { + let data = b"match match match sparse sparse sparse payload"; + let backends = [ + RateBackend::Match { + hash_bits: 20, + min_len: 4, + max_len: 255, + base_mix: 0.02, + confidence_scale: 1.0, + }, + RateBackend::SparseMatch { + hash_bits: 19, + min_len: 3, + max_len: 64, + gap_min: 1, + gap_max: 2, + base_mix: 0.05, + confidence_scale: 1.0, + }, + RateBackend::Ppmd { + order: 8, + memory_mb: 8, + }, + ]; + for backend in backends { + let rate = try_entropy_rate_backend(data, &backend).expect("rate"); + assert!(rate.is_finite() && rate >= 0.0, "rate={rate}"); + } +} + +#[test] +fn neural_mixture_supports_calibrated_expert() { + let data = b"calibrated ctw expert payload calibrated ctw expert payload"; + let spec = MixtureSpec::new( + MixtureKind::Neural, + vec![ + MixtureExpertSpec::new(RateBackend::Calibrated { + spec: Arc::new(CalibratedSpec::new( + RateBackend::Ctw { depth: 8 }, + CalibrationContextKind::Text, + )), + }) + .with_name("cal"), + MixtureExpertSpec::new(RateBackend::Match { + hash_bits: 20, + min_len: 4, + max_len: 255, + base_mix: 0.02, + confidence_scale: 1.0, + }) + .with_name("match"), + ], + ) + .with_alpha(0.03); + let backend = RateBackend::Mixture { + spec: Arc::new(spec), + }; + let rate = try_entropy_rate_backend(data, &backend).expect("rate"); + assert!(rate.is_finite() && rate >= 0.0, "rate={rate}"); +} diff --git a/tests/nyx_vm_tests.rs b/crates/infotheory/tests/nyx_vm_tests.rs similarity index 72% rename from tests/nyx_vm_tests.rs rename to crates/infotheory/tests/nyx_vm_tests.rs index d26d9c0b..940411ae 100644 --- a/tests/nyx_vm_tests.rs +++ b/crates/infotheory/tests/nyx_vm_tests.rs @@ -13,11 +13,16 @@ //! the appropriate guest image. Unit tests for non-VM components can //! run without the VM. -use infotheory::RateBackend; use infotheory::aixi::vm_nyx::*; +use infotheory::api::RateBackend; +use std::str::FromStr; use std::sync::Arc; use std::time::Duration; +fn vm_test_stats_backend() -> RateBackend { + RateBackend::Ctw { depth: 20 } +} + // ============================================================================ // Configuration Tests // ============================================================================ @@ -31,6 +36,10 @@ fn test_default_config() { assert_eq!(config.episode_steps, 100); assert_eq!(config.observation_stream_len, 64); assert!(config.firecracker_config.is_empty()); + assert!(matches!( + config.stats_backend, + RateBackend::Ctw { depth: 20 } + )); } #[test] @@ -101,14 +110,8 @@ fn test_hex_decode_error_invalid_char() { #[test] fn test_literal_action_source() { let actions = vec![ - NyxActionSpec { - name: Some("ping".to_string()), - payload: b"PING".to_vec(), - }, - NyxActionSpec { - name: Some("pong".to_string()), - payload: b"PONG".to_vec(), - }, + NyxActionSpec::named("ping", b"PING".to_vec()), + NyxActionSpec::named("pong", b"PONG".to_vec()), ]; let source = NyxActionSource::Literal(actions.clone()); @@ -124,14 +127,12 @@ fn test_literal_action_source() { #[test] fn test_fuzz_config() { - let config = NyxFuzzConfig { - seeds: vec![b"seed1".to_vec(), b"seed2".to_vec()], - mutators: vec![FuzzMutator::FlipBit, FuzzMutator::FlipByte], - min_len: 1, - max_len: 1024, - dictionary: vec![b"dict1".to_vec()], - rng_seed: 42, - }; + let mut config = NyxFuzzConfig::new(vec![b"seed1".to_vec(), b"seed2".to_vec()]); + config.mutators = vec![FuzzMutator::FlipBit, FuzzMutator::FlipByte]; + config.min_len = 1; + config.max_len = 1024; + config.dictionary = vec![b"dict1".to_vec()]; + config.rng_seed = 42; assert_eq!(config.seeds.len(), 2); assert_eq!(config.mutators.len(), 2); @@ -191,7 +192,6 @@ fn test_reward_policy_pattern() { fn test_reward_shaping_entropy() { let shaping = NyxRewardShaping::EntropyReduction { baseline_bytes: vec![0u8; 100], - max_order: 8, scale: 1.0, crash_bonus: None, timeout_bonus: None, @@ -203,7 +203,6 @@ fn test_reward_shaping_entropy() { #[test] fn test_reward_shaping_trace() { let shaping = NyxRewardShaping::TraceEntropy { - max_order: 4, scale: 2.0, normalize: true, }; @@ -227,15 +226,13 @@ fn test_reward_policy_custom() { #[test] fn test_action_filter() { - let filter = NyxActionFilter { - min_entropy: Some(1.0), - max_entropy: Some(7.5), - min_intrinsic_dependence: Some(0.1), - min_novelty: Some(0.5), - novelty_prior: Some(vec![0, 1, 2, 3]), - max_order: 8, - reject_reward: Some(-10), - }; + let mut filter = NyxActionFilter::new(); + filter.min_entropy = Some(1.0); + filter.max_entropy = Some(7.5); + filter.min_intrinsic_dependence = Some(0.1); + filter.min_novelty = Some(0.5); + filter.novelty_prior = Some(vec![0, 1, 2, 3]); + filter.reject_reward = Some(-10); assert_eq!(filter.min_entropy, Some(1.0)); assert_eq!(filter.max_entropy, Some(7.5)); @@ -309,11 +306,10 @@ fn test_hypercall_constants() { #[test] fn test_trace_config() { - let config = NyxTraceConfig { - shared_region_name: Some("trace_buffer".to_string()), - max_bytes: 4096, - reset_on_episode: true, - }; + let mut config = NyxTraceConfig::new(); + config.shared_region_name = Some("trace_buffer".to_string()); + config.max_bytes = 4096; + config.reset_on_episode = true; assert_eq!(config.shared_region_name, Some("trace_buffer".to_string())); assert_eq!(config.max_bytes, 4096); @@ -326,35 +322,17 @@ fn test_trace_config() { #[test] fn test_payload_encoding_from_str() { - // Inherent parser should work without importing `std::str::FromStr`. + // Canonical parser is `std::str::FromStr`. assert!(matches!( PayloadEncoding::from_str("utf8"), - Some(PayloadEncoding::Utf8) - )); - assert!(matches!( - PayloadEncoding::from_str("text"), - Some(PayloadEncoding::Utf8) + Ok(PayloadEncoding::Utf8) )); assert!(matches!( PayloadEncoding::from_str("hex"), - Some(PayloadEncoding::Hex) - )); - assert!(PayloadEncoding::from_str("unknown").is_none()); - - // `parse` remains equivalent aliasing behavior. - assert!(matches!( - PayloadEncoding::parse("utf8"), - Some(PayloadEncoding::Utf8) - )); - assert!(matches!( - PayloadEncoding::parse("text"), - Some(PayloadEncoding::Utf8) + Ok(PayloadEncoding::Hex) )); - assert!(matches!( - PayloadEncoding::parse("hex"), - Some(PayloadEncoding::Hex) - )); - assert!(PayloadEncoding::parse("unknown").is_none()); + assert!(PayloadEncoding::from_str("unknown").is_err()); + assert!(PayloadEncoding::from_str("text").is_err()); } // ============================================================================ @@ -376,7 +354,6 @@ fn test_fuzz_mutator_variants() { assert_eq!(mutators.len(), 7); } -// ============================================================================ // Information-Theoretic Properties Tests // ============================================================================ @@ -384,13 +361,15 @@ fn test_fuzz_mutator_variants() { mod info_theory_properties { #[allow(unused_imports)] use super::*; - use infotheory::{entropy_rate_bytes, marginal_entropy_bytes}; + use infotheory::api::empirical_entropy_bytes; + #[cfg(feature = "backend-rosa")] + use infotheory::api::try_entropy_rate_bytes; #[test] fn test_entropy_bounds() { // Maximum entropy for bytes is 8 bits let uniform_data: Vec = (0..=255).cycle().take(1024).collect(); - let h = marginal_entropy_bytes(&uniform_data); + let h = empirical_entropy_bytes(&uniform_data); assert!(h <= 8.0 + 1e-6, "Entropy should not exceed 8 bits per byte"); assert!(h >= 0.0, "Entropy should be non-negative"); } @@ -398,23 +377,24 @@ mod info_theory_properties { #[test] fn test_constant_data_low_entropy() { let constant_data = vec![42u8; 1000]; - let h = marginal_entropy_bytes(&constant_data); + let h = empirical_entropy_bytes(&constant_data); assert!( h < 0.01, - "Constant data should have near-zero marginal entropy" + "Constant data should have near-zero empirical entropy" ); } + #[cfg(feature = "backend-rosa")] #[test] - fn test_rate_entropy_less_than_marginal() { - // For structured data, H_rate <= H_marginal + fn test_rate_entropy_less_than_empirical() { + // For structured data, H_rate <= H_empirical let pattern = b"ABCABCABCABCABCABC"; - let h_marg = marginal_entropy_bytes(pattern); - let h_rate = entropy_rate_bytes(pattern, 8); + let h_empirical = empirical_entropy_bytes(pattern); + let h_rate = try_entropy_rate_bytes(pattern).expect("entropy rate"); assert!( - h_rate <= h_marg + 1e-6, - "Entropy rate should not exceed marginal entropy for patterned data" + h_rate <= h_empirical + 1e-6, + "Rate entropy should not exceed empirical entropy for structured data" ); } } @@ -426,7 +406,7 @@ mod info_theory_properties { /// These tests require a running Firecracker VM with proper setup. /// They are marked with #[ignore] and can be run with: /// cargo test -- --ignored -#[cfg(test)] +#[cfg(all(test, feature = "backend-ctw"))] mod vm_integration_tests { #[allow(unused_imports)] use super::*; @@ -437,6 +417,8 @@ mod vm_integration_tests { fn get_project_root() -> PathBuf { PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("..") + .join("..") } fn check_kvm_available() -> bool { @@ -477,10 +459,11 @@ mod vm_integration_tests { config_path } + #[cfg(feature = "backend-ctw")] fn get_test_vm_config(test_name: &str) -> Option { let root = get_project_root(); let kernel_path = root.join("vmlinux-6.1.58"); - let initrd_path = root.join("nyx-lite/guest/aixi_initramfs.cpio"); + let initrd_path = root.join("vendor/nyx-lite/guest/aixi_initramfs.cpio"); if !kernel_path.exists() { eprintln!("Skipping VM test: Kernel not found at {:?}", kernel_path); @@ -493,44 +476,38 @@ mod vm_integration_tests { let fc_config_path = create_firecracker_config(&kernel_path, &initrd_path, test_name); - Some(NyxVmConfig { - firecracker_config: fc_config_path.to_string_lossy().to_string(), - instance_id: format!("test-vm-{}-{}", std::process::id(), test_name), - shared_region_name: "shared".to_string(), - shared_region_size: 4096, - shared_memory_policy: SharedMemoryPolicy::Snapshot, - step_timeout: Duration::from_millis(500), - boot_timeout: Duration::from_secs(30), - episode_steps: 10, - step_cost: 1, - // With proper agent initrd, we can use SharedMemory policy - observation_policy: NyxObservationPolicy::SharedMemory, - observation_bits: 8, - observation_stream_len: 1, - observation_stream_mode: NyxObservationStreamMode::PadTruncate, - observation_pad_byte: 0, - reward_bits: 8, - reward_policy: NyxRewardPolicy::FromGuest, - reward_shaping: None, - action_source: NyxActionSource::Literal(vec![ - NyxActionSpec { - name: Some("nop".to_string()), - payload: vec![], - }, - NyxActionSpec { - name: Some("act".to_string()), - payload: vec![0x01], - }, - ]), - action_filter: None, - protocol: NyxProtocolConfig::default(), - stats_backend: RateBackend::default(), - trace: None, - debug_mode: true, - crash_log: None, - }) + let mut config = NyxVmConfig::default(); + config.firecracker_config = fc_config_path.to_string_lossy().to_string(); + config.instance_id = format!("test-vm-{}-{}", std::process::id(), test_name); + config.shared_region_name = "shared".to_string(); + config.shared_region_size = 4096; + config.shared_memory_policy = SharedMemoryPolicy::Snapshot; + config.step_timeout = Duration::from_millis(500); + config.boot_timeout = Duration::from_secs(30); + config.episode_steps = 10; + config.step_cost = 1; + config.observation_policy = NyxObservationPolicy::SharedMemory; + config.observation_bits = 8; + config.observation_stream_len = 1; + config.observation_stream_mode = NyxObservationStreamMode::PadTruncate; + config.observation_pad_byte = 0; + config.reward_bits = 8; + config.reward_policy = NyxRewardPolicy::FromGuest; + config.reward_shaping = None; + config.action_source = NyxActionSource::Literal(vec![ + NyxActionSpec::named("nop", vec![]), + NyxActionSpec::named("act", vec![0x01]), + ]); + config.action_filter = None; + config.protocol = NyxProtocolConfig::default(); + config.stats_backend = vm_test_stats_backend(); + config.trace = None; + config.debug_mode = true; + config.crash_log = None; + Some(config) } + #[cfg(feature = "backend-ctw")] #[test] fn test_vm_boot_and_snapshot() { if !check_kvm_available() { @@ -561,6 +538,7 @@ mod vm_integration_tests { } } + #[cfg(feature = "backend-ctw")] #[test] fn test_vm_action_execution() { if !check_kvm_available() { @@ -606,57 +584,44 @@ fn test_config_builder_pattern() { /// Test complete configuration for a typical experiment #[test] fn test_complete_experiment_config() { - let config = NyxVmConfig { - firecracker_config: "/path/to/config.json".to_string(), - instance_id: "experiment-1".to_string(), - shared_region_name: "shared".to_string(), - shared_region_size: 4096, - shared_memory_policy: SharedMemoryPolicy::Snapshot, - step_timeout: Duration::from_millis(100), - boot_timeout: Duration::from_secs(30), - episode_steps: 100, - step_cost: 1, - observation_policy: NyxObservationPolicy::SharedMemory, - observation_bits: 8, - observation_stream_len: 64, - observation_stream_mode: NyxObservationStreamMode::PadTruncate, - observation_pad_byte: 0, - reward_bits: 8, - reward_policy: NyxRewardPolicy::FromGuest, - reward_shaping: Some(NyxRewardShaping::TraceEntropy { - max_order: 8, - scale: 1.0, - normalize: true, - }), - action_source: NyxActionSource::Literal(vec![ - NyxActionSpec { - name: Some("nop".to_string()), - payload: vec![], - }, - NyxActionSpec { - name: Some("action1".to_string()), - payload: b"A".to_vec(), - }, - ]), - action_filter: Some(NyxActionFilter { - min_entropy: Some(0.5), - max_entropy: None, - min_intrinsic_dependence: None, - min_novelty: None, - novelty_prior: None, - max_order: 4, - reject_reward: Some(-1), - }), - protocol: NyxProtocolConfig::default(), - stats_backend: RateBackend::default(), - trace: Some(NyxTraceConfig { - shared_region_name: Some("trace".to_string()), - max_bytes: 1024, - reset_on_episode: true, - }), - debug_mode: false, - crash_log: None, - }; + let mut config = NyxVmConfig::default(); + config.firecracker_config = "/path/to/config.json".to_string(); + config.instance_id = "experiment-1".to_string(); + config.shared_region_name = "shared".to_string(); + config.shared_region_size = 4096; + config.shared_memory_policy = SharedMemoryPolicy::Snapshot; + config.step_timeout = Duration::from_millis(100); + config.boot_timeout = Duration::from_secs(30); + config.episode_steps = 100; + config.step_cost = 1; + config.observation_policy = NyxObservationPolicy::SharedMemory; + config.observation_bits = 8; + config.observation_stream_len = 64; + config.observation_stream_mode = NyxObservationStreamMode::PadTruncate; + config.observation_pad_byte = 0; + config.reward_bits = 8; + config.reward_policy = NyxRewardPolicy::FromGuest; + config.reward_shaping = Some(NyxRewardShaping::TraceEntropy { + scale: 1.0, + normalize: true, + }); + config.action_source = NyxActionSource::Literal(vec![ + NyxActionSpec::named("nop", vec![]), + NyxActionSpec::named("action1", b"A".to_vec()), + ]); + let mut action_filter = NyxActionFilter::new(); + action_filter.min_entropy = Some(0.5); + action_filter.reject_reward = Some(-1); + config.action_filter = Some(action_filter); + config.protocol = NyxProtocolConfig::default(); + config.stats_backend = vm_test_stats_backend(); + let mut trace = NyxTraceConfig::new(); + trace.shared_region_name = Some("trace".to_string()); + trace.max_bytes = 1024; + trace.reset_on_episode = true; + config.trace = Some(trace); + config.debug_mode = false; + config.crash_log = None; assert_eq!(config.episode_steps, 100); assert_eq!(config.step_cost, 1); diff --git a/tests/oracle_tests.rs b/crates/infotheory/tests/oracle_tests.rs similarity index 62% rename from tests/oracle_tests.rs rename to crates/infotheory/tests/oracle_tests.rs index 83309b07..c5585e19 100644 --- a/tests/oracle_tests.rs +++ b/crates/infotheory/tests/oracle_tests.rs @@ -1,16 +1,35 @@ -use infotheory::axioms; -use infotheory::datagen; +use infotheory::api::empirical_entropy_bytes; #[cfg(feature = "backend-zpaq")] -use infotheory::{CompressionBackend, NcdVariant, ncd_bytes_backend}; -use infotheory::{ - RateBackend, entropy_rate_backend, marginal_entropy_bytes, mutual_information_bytes, +use infotheory::api::{ + CompressionBackend, NcdVariant, try_ncd_bytes_backend as try_ncd_bytes_backend_compiled, }; +#[cfg(any(feature = "backend-rosa", feature = "backend-ctw"))] +use infotheory::api::{RateBackend, try_entropy_rate_backend as try_entropy_rate_backend_compiled}; +use infotheory::axioms; +use infotheory::datagen; const TOLERANCE_ENTROPY: f64 = 0.1; const TOLERANCE_MI: f64 = 0.2; #[cfg(feature = "backend-zpaq")] const TOLERANCE_NCD: f64 = 0.1; +#[cfg(any(feature = "backend-rosa", feature = "backend-ctw"))] +fn try_entropy_rate_backend(data: &[u8], backend: &RateBackend) -> Result { + let compiled = backend.compile().map_err(|err| err.to_string())?; + try_entropy_rate_backend_compiled(data, &compiled).map_err(|err| err.to_string()) +} + +#[cfg(feature = "backend-zpaq")] +fn try_ncd_bytes_backend( + x: &[u8], + y: &[u8], + backend: &CompressionBackend, + variant: NcdVariant, +) -> Result { + let compiled = backend.compile().map_err(|err| err.to_string())?; + try_ncd_bytes_backend_compiled(x, y, &compiled, variant).map_err(|err| err.to_string()) +} + // ============================================================================ // Entropy Tests // ============================================================================ @@ -22,8 +41,8 @@ fn entropy_vs_theoretical_bernoulli() { let n = 20_000; let data = datagen::bernoulli(n, p, 42); - // Use order-0 entropy (marginal) since it's IID - let estimated = marginal_entropy_bytes(&data); + // Use order-0 entropy (empirical/IID Shannon plug-in) + let estimated = empirical_entropy_bytes(&data); let theoretical = datagen::bernoulli_entropy(p); println!( @@ -38,7 +57,10 @@ fn entropy_vs_theoretical_bernoulli() { ); // Also verify bounds axiom - assert!(axioms::verify_entropy_bounds(marginal_entropy_bytes, &data)); + assert!(axioms::verify_entropy_bounds( + empirical_entropy_bytes, + &data + )); } } @@ -54,8 +76,8 @@ fn mi_independent_is_zero() { let x: Vec = x.iter().map(|b| b & 0x0F).collect(); let y: Vec = y.iter().map(|b| b & 0x0F).collect(); - // Order-0 MI for IID data - let mi = mutual_information_bytes(&x, &y, 0); + // Use empirical (order-0 IID) mutual information for IID data + let mi = infotheory::api::empirical_mutual_information_bytes(&x, &y); println!("Independent MI (16-sym): {:.4}", mi); @@ -67,7 +89,7 @@ fn mi_independent_is_zero() { // Check non-negativity axiom assert!(axioms::verify_mi_nonnegative( - |a, b| mutual_information_bytes(a, b, 0), + infotheory::api::empirical_mutual_information_bytes, &x, &y )); @@ -75,6 +97,7 @@ fn mi_independent_is_zero() { // ... (other tests unchanged) ... +#[cfg(feature = "backend-rosa")] #[test] fn rosa_matches_theoretical_markov_entropy() { let (p00, p11) = (0.8, 0.8); @@ -83,8 +106,8 @@ fn rosa_matches_theoretical_markov_entropy() { let theoretical = datagen::markov_1_binary_entropy_rate(p00, p11); // ROSA - let backend = RateBackend::RosaPlus; - let estimated = entropy_rate_backend(&data, 20, &backend); + let backend = RateBackend::RosaPlus { max_order: 20 }; + let estimated = try_entropy_rate_backend(&data, &backend).expect("entropy rate"); println!( "ROSA Markov: Est={:.4}, Theory={:.4}", @@ -107,8 +130,8 @@ fn mi_deterministic_equals_entropy() { // Let's use Y = X + 1 (wrapping) let (x, y) = datagen::deterministic_func(n, 42, |b| b.wrapping_add(1)); - let mi = mutual_information_bytes(&x, &y, 0); - let h_y = marginal_entropy_bytes(&y); + let mi = infotheory::api::empirical_mutual_information_bytes(&x, &y); + let h_y = empirical_entropy_bytes(&y); println!("Deterministic: MI={:.4}, H(Y)={:.4}", mi, h_y); @@ -123,8 +146,8 @@ fn mi_identical_equals_entropy() { let n = 10_000; let (x, y) = datagen::identical_pair(n, 42); - let mi = mutual_information_bytes(&x, &y, 0); - let h_x = marginal_entropy_bytes(&x); + let mi = infotheory::api::empirical_mutual_information_bytes(&x, &y); + let h_x = empirical_entropy_bytes(&x); println!("Identical: MI={:.4}, H(X)={:.4}", mi, h_x); @@ -145,10 +168,8 @@ fn ncd_identity_is_zero() { let (x, y) = datagen::identical_pair(n, 42); // Using default ZPAQ method 1 - let backend = CompressionBackend::Zpaq { - method: "1".to_string(), - }; - let ncd = ncd_bytes_backend(&x, &y, &backend, NcdVariant::Vitanyi); + let backend = CompressionBackend::zpaq("1"); + let ncd = try_ncd_bytes_backend(&x, &y, &backend, NcdVariant::Vitanyi).expect("ncd"); println!("NCD(X,X) = {:.4}", ncd); @@ -164,10 +185,8 @@ fn ncd_independent_is_near_one() { let n = 2_000; let (x, y) = datagen::independent_pair(n, 12345, 67890); - let backend = CompressionBackend::Zpaq { - method: "1".to_string(), - }; - let ncd = ncd_bytes_backend(&x, &y, &backend, NcdVariant::Vitanyi); + let backend = CompressionBackend::zpaq("1"); + let ncd = try_ncd_bytes_backend(&x, &y, &backend, NcdVariant::Vitanyi).expect("ncd"); println!("NCD(X,Y) independent = {:.4}", ncd); @@ -188,10 +207,10 @@ fn ncd_triangle_inequality() { let y = datagen::uniform_random(n, 222); let z = datagen::uniform_random(n, 333); - let backend = CompressionBackend::Zpaq { - method: "1".to_string(), + let backend = CompressionBackend::zpaq("1"); + let metric = |a: &[u8], b: &[u8]| { + try_ncd_bytes_backend(a, b, &backend, NcdVariant::Vitanyi).expect("ncd") }; - let metric = |a: &[u8], b: &[u8]| ncd_bytes_backend(a, b, &backend, NcdVariant::Vitanyi); assert!( axioms::verify_triangle_inequality(metric, &x, &y, &z, 0.1), @@ -203,6 +222,7 @@ fn ncd_triangle_inequality() { // Backend Specific (CTW / ROSA) // ============================================================================ +#[cfg(feature = "backend-ctw")] #[test] fn ctw_matches_theoretical_markov_entropy() { // Generate Markov chain with known entropy rate @@ -210,18 +230,27 @@ fn ctw_matches_theoretical_markov_entropy() { let n = 10_000; let data = datagen::markov_1_binary(n, p00, p11, 42); let theoretical = datagen::markov_1_binary_entropy_rate(p00, p11); + let iid_baseline = empirical_entropy_bytes(&data); // Use CTW with sufficient depth to capture Markov-1 let backend = RateBackend::Ctw { depth: 8 }; - let estimated = entropy_rate_backend(&data, -1, &backend); // -1 max_order ignored for CTW + let estimated = try_entropy_rate_backend(&data, &backend).expect("entropy rate"); + let iid_gap = iid_baseline - theoretical; + let required_max = iid_baseline - 0.25 * iid_gap; println!( - "CTW Markov: Est={:.4}, Theory={:.4}", - estimated, theoretical + "CTW Markov: Est={estimated:.4}, Theory={theoretical:.4}, IID={iid_baseline:.4}, RequiredMax={required_max:.4}" ); + // Direct CTW models bytes as an MSB-first bit stream. For 0/1 byte-symbol + // Markov data, the finite-sample contract is that CTW stays near the source + // entropy rate while materially beating the IID byte-symbol baseline. + assert!( + estimated + TOLERANCE_ENTROPY >= theoretical, + "CTW entropy rate fell below source entropy beyond tolerance: est={estimated}, theory={theoretical}, tol={TOLERANCE_ENTROPY}" + ); assert!( - (estimated - theoretical).abs() < 0.2, - "CTW entropy rate: est={estimated}, theory={theoretical}" + estimated <= required_max, + "CTW entropy rate did not materially beat IID baseline: est={estimated}, required_max={required_max}, iid={iid_baseline}, theory={theoretical}" ); } diff --git a/tests/roundtrip_hashes.rs b/crates/infotheory/tests/roundtrip_hashes.rs similarity index 62% rename from tests/roundtrip_hashes.rs rename to crates/infotheory/tests/roundtrip_hashes.rs index 9505ea8a..d3222eb3 100644 --- a/tests/roundtrip_hashes.rs +++ b/crates/infotheory/tests/roundtrip_hashes.rs @@ -1,6 +1,8 @@ #![cfg(feature = "backend-zpaq")] -use infotheory::{CompressionBackend, compress_bytes_backend, decompress_bytes_backend}; +use infotheory::api::{ + CompressionBackend, try_compress_bytes_backend, try_decompress_bytes_backend, +}; use sha2::{Digest, Sha256}; #[cfg(feature = "cli")] use std::io::Write; @@ -16,6 +18,28 @@ fn sha256_hex(data: &[u8]) -> String { out.iter().map(|b| format!("{b:02x}")).collect() } +fn expected_zpaq_fixture_hashes() -> (&'static str, &'static str) { + // libzpaq's NOJIT Windows path does not promise the same compressed + // bytestream as the x86_64 JIT-enabled builds we exercise elsewhere in CI. + // Keep the roundtrip invariant universal, and pin the known stable output + // for each platform/codegen mode we ship in CI. + #[cfg(windows)] + { + ( + "cfa467b7e0d31d9762f8d469daa687b1e0a571896debc3cc42399bd574b43646", + "c816dea6bf09dc6bad4d7c1fc7bc52658ae89ca3fec1bbf238de71bf7a39e3f0", + ) + } + + #[cfg(not(windows))] + { + ( + "26ad22d35f5f014d7b99a403af46a0c2b172986352ffee21a03d1f7a39d67498", + "df691b88c9c1a9791b57f3e7d70fc05c6bb7a324f71e9b45900696472befb837", + ) + } +} + #[test] fn zpaq_roundtrip_fixture_a_and_hash_stability() { let input = std::fs::read(concat!( @@ -23,15 +47,15 @@ fn zpaq_roundtrip_fixture_a_and_hash_stability() { "/tests/fixtures/fixture_a.txt" )) .expect("failed to read fixture_a"); - let backend = CompressionBackend::Zpaq { - method: "5".to_string(), - }; - let compressed = compress_bytes_backend(&input, &backend).expect("compress failed"); - let restored = decompress_bytes_backend(&compressed, &backend).expect("decompress failed"); + let backend = CompressionBackend::zpaq("5"); + let backend = backend.compile().expect("compile zpaq backend"); + let compressed = try_compress_bytes_backend(&input, &backend).expect("compress failed"); + let restored = try_decompress_bytes_backend(&compressed, &backend).expect("decompress failed"); assert_eq!(restored, input, "zpaq roundtrip mismatch"); + let (expected_fixture_a_hash, _) = expected_zpaq_fixture_hashes(); assert_eq!( sha256_hex(&compressed), - "26ad22d35f5f014d7b99a403af46a0c2b172986352ffee21a03d1f7a39d67498", + expected_fixture_a_hash, "compressed bytes hash changed unexpectedly" ); } @@ -43,15 +67,15 @@ fn zpaq_roundtrip_fixture_b_and_hash_stability() { "/tests/fixtures/fixture_b.txt" )) .expect("failed to read fixture_b"); - let backend = CompressionBackend::Zpaq { - method: "5".to_string(), - }; - let compressed = compress_bytes_backend(&input, &backend).expect("compress failed"); - let restored = decompress_bytes_backend(&compressed, &backend).expect("decompress failed"); + let backend = CompressionBackend::zpaq("5"); + let backend = backend.compile().expect("compile zpaq backend"); + let compressed = try_compress_bytes_backend(&input, &backend).expect("compress failed"); + let restored = try_decompress_bytes_backend(&compressed, &backend).expect("decompress failed"); assert_eq!(restored, input, "zpaq roundtrip mismatch"); + let (_, expected_fixture_b_hash) = expected_zpaq_fixture_hashes(); assert_eq!( sha256_hex(&compressed), - "df691b88c9c1a9791b57f3e7d70fc05c6bb7a324f71e9b45900696472befb837", + expected_fixture_b_hash, "compressed bytes hash changed unexpectedly" ); } @@ -83,7 +107,6 @@ fn batch_metrics_output_hash_stability() { serde_json::json!({ "op": "metrics", "text": "abracadabra", - "max_order": 3, }) ) .expect("failed to write payload"); diff --git a/tests/rwkv_method_canonicalization.rs b/crates/infotheory/tests/rwkv_method_canonicalization.rs similarity index 100% rename from tests/rwkv_method_canonicalization.rs rename to crates/infotheory/tests/rwkv_method_canonicalization.rs diff --git a/crates/infotheory/tests/support/mod.rs b/crates/infotheory/tests/support/mod.rs new file mode 100644 index 00000000..b823af17 --- /dev/null +++ b/crates/infotheory/tests/support/mod.rs @@ -0,0 +1,2 @@ +#[path = "../../src/aixi/test_envs.rs"] +pub mod aixi_envs; diff --git a/crates/infotheory/tests/tuner_integration.rs b/crates/infotheory/tests/tuner_integration.rs new file mode 100644 index 00000000..4e7f5a3b --- /dev/null +++ b/crates/infotheory/tests/tuner_integration.rs @@ -0,0 +1,3627 @@ +#![cfg(all(feature = "tuner", feature = "backend-ctw"))] + +use crc32fast::Hasher; +use infotheory::aixi::warmstart_contract::TaskFingerprint; +use infotheory::spec::{SpecDocument, SpecEnvironment}; +use infotheory::tuner::{ + TimingCertificationTier, parse_tune_command_args, run_tune, run_tuner_eval_worker_from_env, +}; +use serde_json::{Value, json}; +use std::fs; +#[cfg(unix)] +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; +#[cfg(feature = "cli")] +use std::process::Command; +use std::time::{SystemTime, UNIX_EPOCH}; + +const PLACEHOLDER_TASK_FINGERPRINT_HEX: &str = + "1111111111111111111111111111111111111111111111111111111111111111"; +const PROBE_TASK_FINGERPRINT_HEX: &str = + "0000000000000000000000000000000000000000000000000000000000000000"; +const MISMATCH_TASK_FINGERPRINT_HEX: &str = + "deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef"; + +struct ControllerCase { + kind: &'static str, + status: &'static str, + runtime_path: &'static str, + agent_runtime: &'static str, + planner_run_controller_kind: &'static str, + reward_semantics: &'static str, + needs_teacher: bool, + controller: Value, +} + +fn temp_dir(label: &str) -> PathBuf { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("clock") + .as_nanos(); + let path = std::env::temp_dir().join(format!("infotheory_tuner_integration_{label}_{nanos}")); + fs::create_dir_all(&path).expect("create temp dir"); + path +} + +fn path_string(path: &Path) -> String { + path.to_string_lossy().to_string() +} + +fn crc32_hex(bytes: &[u8]) -> String { + let mut hasher = Hasher::new(); + hasher.update(bytes); + format!("{:08x}", hasher.finalize()) +} + +fn write_json(path: &Path, value: &Value) { + fs::write(path, serde_json::to_vec(value).expect("serialize json")).expect("write json"); +} + +fn write_tune_itsd(path: &Path, value: &Value, base_dir: &Path) { + let document = SpecDocument::parse_json_value(value, base_dir).expect("parse tune json"); + fs::write(path, document.to_binary()).expect("write tune itsd"); +} + +fn read_json(path: &Path) -> Value { + serde_json::from_slice(&fs::read(path).expect("read json")).expect("parse json") +} + +fn str_at<'a>(value: &'a Value, pointer: &str) -> &'a str { + value + .pointer(pointer) + .and_then(Value::as_str) + .unwrap_or_else(|| panic!("{pointer} must be a string in {value}")) +} + +fn u64_at(value: &Value, pointer: &str) -> u64 { + value + .pointer(pointer) + .and_then(Value::as_u64) + .unwrap_or_else(|| panic!("{pointer} must be an unsigned integer in {value}")) +} + +fn f64_at(value: &Value, pointer: &str) -> f64 { + value + .pointer(pointer) + .and_then(Value::as_f64) + .unwrap_or_else(|| panic!("{pointer} must be a finite number in {value}")) +} + +fn bool_at(value: &Value, pointer: &str) -> bool { + value + .pointer(pointer) + .and_then(Value::as_bool) + .unwrap_or_else(|| panic!("{pointer} must be a boolean in {value}")) +} + +#[cfg(unix)] +#[test] +#[ignore = "libtest entrypoint for spawned tuner evaluator workers"] +fn __infotheory_tuner_eval_worker() { + if std::env::var_os("INFOTHEORY_TUNER_EVAL_REQUEST_PATH").is_none() + || std::env::var_os("INFOTHEORY_TUNER_EVAL_RESPONSE_PATH").is_none() + { + return; + } + run_tuner_eval_worker_from_env().expect("run tuner evaluator worker from env"); +} + +fn interface() -> Value { + json!({ + "observation_bits": 8, + "observation_stream_len": 1, + "observation_key_mode": "full_stream", + "reward_bits": 16, + "agent_actions": 2, + }) +} + +fn baseline_candidate() -> Value { + json!({ + "kind": "rate-ac", + "rate_backend": { + "kind": "ctw", + "depth": 8, + }, + "framing": "framed", + }) +} + +fn bounds() -> Value { + json!({ + "allowed_backends": ["ctw"], + "forbidden_backends": [], + "parameter_ranges": [{ + "parameter": "rate_backend.depth", + "min": 1.0, + "max": 16.0, + }], + "max_experts": 2, + "max_mixture_nesting_depth": 1, + "min_experts": 1, + "allow_duplicate_experts": false, + "required_experts": [], + "forbidden_expert_pairs": [], + }) +} + +fn controller_cases() -> Vec { + vec![ + ControllerCase { + kind: "mc_aixi_fac_ctw", + status: "completed_mc_aixi_fac_ctw", + runtime_path: "finite_mutation_agent_bridge_mcaixi_fac_ctw", + agent_runtime: "aixi::agent::Agent", + planner_run_controller_kind: "mc_aixi", + reward_semantics: "exact_objective_difference", + needs_teacher: false, + controller: json!({ + "kind": "mc_aixi_fac_ctw", + "interface": interface(), + "planner_simulations_per_step": 2, + }), + }, + ControllerCase { + kind: "aiqi_discounted", + status: "completed_aiqi_discounted", + runtime_path: "finite_mutation_agent_bridge_aiqi_discounted", + agent_runtime: "aixi::aiqi::AiqiAgent", + planner_run_controller_kind: "aiqi_discounted", + reward_semantics: "normalized_clipped_improvement", + needs_teacher: false, + controller: json!({ + "kind": "aiqi_discounted", + "interface": interface(), + "planner_simulations_per_step": 2, + "return_horizon": 1, + "return_bins": 2, + "discount_factor": 0.5, + "min_improvement": 0.0, + "max_improvement": 1.0, + }), + }, + ControllerCase { + kind: "aiqi_warmstart_exact_jh", + status: "completed_aiqi_warmstart_exact_jh", + runtime_path: "finite_mutation_agent_bridge_aiqi_warmstart_exact_jh", + agent_runtime: "aixi::warmstart::WarmStartExactJhAgent", + planner_run_controller_kind: "aiqi_warmstart_exact_jh", + reward_semantics: "exact_objective_difference", + needs_teacher: true, + controller: json!({ + "kind": "aiqi_warmstart_exact_jh", + "interface": interface(), + "planner_simulations_per_step": 1, + "return_horizon": 1, + "warmstart_teacher_dataset_asset": "teacher", + "label_phase_period": 1, + }), + }, + ] +} + +fn tune_spec( + dataset_path: &Path, + output_path: &Path, + report_path: &Path, + controller: Value, + teacher_path: Option<&Path>, +) -> Value { + let mut assets = vec![json!({ + "id": "dataset", + "path": path_string(dataset_path), + })]; + if let Some(path) = teacher_path { + assets.push(json!({ + "id": "teacher", + "path": path_string(path), + })); + } + json!({ + "schema_version": 1, + "kind": "tune", + "assets": assets, + "input_asset": "dataset", + "baseline_candidate": baseline_candidate(), + "controller": controller, + "bounds": bounds(), + "eval_time_limit_seconds": 1.0, + "time_budget_seconds": 5.0, + "min_throughput_bytes_per_second": 1.0, + "max_memory_bytes": 1099511627776u64, + "output_config_path": path_string(output_path), + "seed": 7, + "report_path": path_string(report_path), + }) +} + +fn write_passive_dataset(path: &Path) { + fs::write(path, b"planner family passive integration dataset").expect("write passive dataset"); +} + +fn canonical_causal_dataset(codec_hash: &str, payload_key: &str, payload: Value) -> Value { + let mut object = serde_json::Map::new(); + object.insert("schema_version".to_string(), json!(1)); + object.insert("environment_id".to_string(), json!("test-env")); + object.insert("environment_config_crc32".to_string(), json!("00000000")); + object.insert("codec_hash".to_string(), json!(codec_hash)); + object.insert( + "reset_convention".to_string(), + json!("reset-before-episode"), + ); + object.insert("action_alphabet".to_string(), json!({"size": 2})); + object.insert( + "percept_schema".to_string(), + json!({ + "encoding": "bytes", + "channels": [{"channel": "percept", "domain": "bytes"}] + }), + ); + object.insert( + "reward_encoding".to_string(), + json!({"encoding": "bytes", "channel": "reward", "domain": "binary"}), + ); + object.insert( + "terminal_encoding".to_string(), + json!({"encoding": "bytes", "channel": "terminal", "domain": "binary"}), + ); + object.insert("collection_policy".to_string(), json!("test-policy")); + object.insert( + "target_domains".to_string(), + json!({ + "bytes": {"kind": "byte_alphabet"}, + "binary": {"kind": "enumerated_payloads", "payloads": [[0], [1]]} + }), + ); + object.insert( + "event_grammar".to_string(), + json!({ + "context_channels": ["action"], + "observe_target_no_score": [ + {"channel": "percept", "domain": "bytes"}, + {"channel": "percept", "domain": "binary"}, + {"channel": "reward", "domain": "binary"}, + {"channel": "terminal", "domain": "binary"} + ], + "target": [ + {"channel": "percept", "domain": "bytes"}, + {"channel": "percept", "domain": "binary"}, + {"channel": "reward", "domain": "binary"}, + {"channel": "terminal", "domain": "binary"} + ] + }), + ); + object.insert(payload_key.to_string(), payload); + Value::Object(object) +} + +fn write_teacher(path: &Path, task_fingerprint: TaskFingerprint, reward_cert_crc32: &str) { + write_json( + path, + &json!({ + "schema_version": 1, + "contract": { + "task_fingerprint": task_fingerprint.to_string(), + "action_alphabet_size": 2, + "observation_bits": 8, + "observation_stream_len": 1, + "observation_key_mode": "full_stream", + "observation_adapter_spec_ref": "single-channel-conditional-byte-adapter-v1", + "observation_adapter_content_crc32": observation_adapter_crc32(), + "reward_bits": 16, + "return_horizon": 1, + "label_phase_period": 1, + "scalar_representation": "scalar://finite-f64", + "exact_reward_encoding_certificate": reward_cert_crc32 + }, + "traces": [{ + "transitions": [ + {"action": 0, "observations": [0], "reward": 0}, + {"action": 1, "observations": [1], "reward": 1} + ] + }] + }), + ); +} + +fn write_placeholder_teacher(path: &Path) { + write_teacher( + path, + TaskFingerprint::parse_hex(PLACEHOLDER_TASK_FINGERPRINT_HEX) + .expect("placeholder task fingerprint"), + "11111111", + ); +} + +fn probe_task_fingerprint() -> TaskFingerprint { + TaskFingerprint::parse_hex(PROBE_TASK_FINGERPRINT_HEX).expect("probe task fingerprint") +} + +fn timing_label(timing: TimingCertificationTier) -> &'static str { + match timing { + TimingCertificationTier::BestEffort => "best_effort", + TimingCertificationTier::Isolated => "isolated", + TimingCertificationTier::RealTime => "real_time", + TimingCertificationTier::DeterministicTable => "deterministic_table", + _ => unreachable!("test covers known timing variants"), + } +} + +fn timing_certifies(timing: TimingCertificationTier) -> bool { + matches!( + timing, + TimingCertificationTier::RealTime | TimingCertificationTier::DeterministicTable + ) +} + +fn feature_set() -> Vec<&'static str> { + let mut features = Vec::new(); + if cfg!(feature = "default-backends") { + features.push("default-backends"); + } + if cfg!(feature = "capability-default") { + features.push("capability-default"); + } + if cfg!(feature = "capability-statistical") { + features.push("capability-statistical"); + } + if cfg!(feature = "capability-neural") { + features.push("capability-neural"); + } + if cfg!(feature = "capability-archive") { + features.push("capability-archive"); + } + if cfg!(feature = "capability-vm") { + features.push("capability-vm"); + } + if cfg!(feature = "aixi") { + features.push("aixi"); + } + if cfg!(feature = "tuner") { + features.push("tuner"); + } + if cfg!(feature = "aixi-gameengine") { + features.push("aixi-gameengine"); + } + if cfg!(feature = "aixi-gameengine-physics") { + features.push("aixi-gameengine-physics"); + } + if cfg!(feature = "aixi-vm") { + features.push("aixi-vm"); + } + if cfg!(feature = "all-backends") { + features.push("all-backends"); + } + if cfg!(feature = "backend-rosa") { + features.push("backend-rosa"); + } + if cfg!(feature = "backend-ctw") { + features.push("backend-ctw"); + } + if cfg!(feature = "backend-match") { + features.push("backend-match"); + } + if cfg!(feature = "backend-ppmd") { + features.push("backend-ppmd"); + } + if cfg!(feature = "backend-sequitur") { + features.push("backend-sequitur"); + } + if cfg!(feature = "backend-mixture") { + features.push("backend-mixture"); + } + if cfg!(feature = "backend-particle") { + features.push("backend-particle"); + } + if cfg!(feature = "backend-calibrated") { + features.push("backend-calibrated"); + } + if cfg!(feature = "backend-mamba") { + features.push("backend-mamba"); + } + if cfg!(feature = "backend-rwkv") { + features.push("backend-rwkv"); + } + if cfg!(feature = "backend-zpaq") { + features.push("backend-zpaq"); + } + if cfg!(feature = "cli") { + features.push("cli"); + } + if cfg!(feature = "vm") { + features.push("vm"); + } + features +} + +fn bounds_crc32() -> String { + let value = json!({ + "allowed_backends": ["ctw"], + "forbidden_backends": [], + "parameter_ranges": [{ + "parameter": "rate_backend.depth", + "min_bits": 1.0f64.to_bits(), + "max_bits": 16.0f64.to_bits(), + }], + "max_experts": 2, + "max_mixture_nesting_depth": 1, + "min_experts": 1, + "allow_duplicate_experts": false, + "required_experts": [], + "forbidden_expert_pairs": [], + }); + crc32_hex(&serde_json::to_vec(&value).expect("bounds hash json")) +} + +fn observation_adapter_crc32() -> String { + let value = json!({ + "kind": "single-channel-conditional-byte-adapter-v1", + "schema_version": 1, + "stream": "fixed_len_packed_u64_little_endian", + "fields": [ + "fail_flag", + "normalized_physical_size", + "normalized_target_loss", + "normalized_eval_time", + "physical_size_delta", + "eval_time_delta", + "candidate_signature_crc32", + "terminal" + ], + "missing_sentinel": 255, + "nonfinite_float_encoding": "forbidden_before_encoding", + "delta_time_epsilon": 1.0e-9f64, + }); + crc32_hex(&serde_json::to_vec(&value).expect("observation adapter hash json")) +} + +fn dataset_crc32(path: &Path) -> String { + crc32_hex(&fs::read(path).expect("read dataset for crc32")) +} + +fn evaluator_profile_crc32_with_runtime_profile_and_worker( + dataset_path: &Path, + timing: TimingCertificationTier, + deterministic_table_requested: bool, + worker_executable_override: Option<&Path>, +) -> String { + let ( + worker_executable_identity, + resolved_memory_accounting_kind, + resolved_memory_accounting_strict_theorem_facing, + resolved_evaluator_cgroup_parent, + backend_report_component_policy, + ) = if deterministic_table_requested { + ( + None::, + "deterministic_evaluator_table_row_peak_memory", + true, + None::, + "none_deterministic_table_row", + ) + } else { + let worker_executable = if let Some(path) = worker_executable_override { + if !path.is_file() { + panic!( + "evaluator_worker_executable '{}' does not resolve to a file", + path.display() + ); + } + path.to_path_buf() + } else if let Some(path) = std::env::var_os("INFOTHEORY_TUNER_EVAL_WORKER_EXE") { + let path = PathBuf::from(path); + if !path.is_file() { + panic!( + "INFOTHEORY_TUNER_EVAL_WORKER_EXE '{}' does not resolve to a file", + path.display() + ); + } + path + } else if let Some(path) = std::env::var_os("CARGO_BIN_EXE_infotheory") { + let path = PathBuf::from(path); + if path.is_file() { + path + } else { + std::env::current_exe().expect("resolve current executable for evaluator worker") + } + } else { + std::env::current_exe().expect("resolve current executable for evaluator worker") + }; + let worker_bytes = fs::read(&worker_executable).unwrap_or_else(|err| { + panic!( + "failed to read evaluator worker executable '{}' for profile hash: {err}", + worker_executable.display() + ) + }); + ( + Some(format!( + "crc32:{}:bytes:{}", + crc32_hex(&worker_bytes), + worker_bytes.len() + )), + "unix_process_rss_fallback_explicit", + false, + None, + "none", + ) + }; + let value = json!({ + "dataset_kind": "passive_bytes", + "objective_target": "passive_ac", + "dataset_lowering_version": "passive-bytes-v1", + "dataset_codec_hash": "passive-identity-bytes", + "event_grammar_hash": "passive-target-only-byte-stream", + "target_domain_support_hash": crc32_hex(b"passive-byte-alphabet"), + "causal_header_profile_hash": crc32_hex(b"passive-none"), + "target_size_function": "passive-bytes-len", + "evaluator_interface_version": "typed-causal-evaluator-v1", + "candidate_canonicalization_version": "bounds-v1", + "warmup_baseline_runs": 0, + "diagnostic_chunk_bytes": null, + "effective_eval_time_limit_seconds_bits": 1.0f64.to_bits(), + "evaluator_threads": 1, + "worker_isolation_mode": "spawn_exec_worker", + "worker_executable_identity": worker_executable_identity, + "resolved_memory_accounting_kind": resolved_memory_accounting_kind, + "resolved_memory_accounting_strict_theorem_facing": resolved_memory_accounting_strict_theorem_facing, + "resolved_evaluator_cgroup_parent": resolved_evaluator_cgroup_parent, + "backend_report_component_policy": backend_report_component_policy, + "evaluator_determinism": "deterministic_under_h", + "rss_mode": "process_rss_peak", + "timing_certification_tier": timing_label(timing), + "build_profile": option_env!("PROFILE").unwrap_or("unknown"), + "feature_set": feature_set(), + }); + let _ = dataset_path; + crc32_hex(&serde_json::to_vec(&value).expect("profile hash json")) +} + +fn compiled_tune_hashes(spec: &Value, dir: &Path) -> (String, String) { + let document = SpecDocument::parse_json_value(spec, dir).expect("parse tune spec for certs"); + let SpecDocument::Tune(tune) = document else { + panic!("expected tune spec"); + }; + let compiled = tune + .compile_in(&SpecEnvironment::new(dir)) + .expect("compile tune spec for certs"); + ( + crc32_hex(compiled.canonical_bytes().as_slice()), + crc32_hex(compiled.baseline_candidate().canonical_bytes().as_slice()), + ) +} + +fn common_certificate( + kind: &str, + dataset_path: &Path, + timing: TimingCertificationTier, + controller_kind: &str, +) -> Value { + common_certificate_with_runtime_profile(kind, dataset_path, timing, controller_kind, false) +} + +fn common_certificate_with_runtime_profile( + kind: &str, + dataset_path: &Path, + timing: TimingCertificationTier, + controller_kind: &str, + deterministic_table_requested: bool, +) -> Value { + common_certificate_with_runtime_profile_and_worker( + kind, + dataset_path, + timing, + controller_kind, + deterministic_table_requested, + None, + ) +} + +fn common_certificate_with_runtime_profile_and_worker( + kind: &str, + dataset_path: &Path, + timing: TimingCertificationTier, + controller_kind: &str, + deterministic_table_requested: bool, + worker_executable_override: Option<&Path>, +) -> Value { + json!({ + "schema_version": 1, + "kind": kind, + "dataset_crc32": dataset_crc32(dataset_path), + "bounds_crc32": bounds_crc32(), + "evaluator_profile_crc32": evaluator_profile_crc32_with_runtime_profile_and_worker( + dataset_path, + timing, + deterministic_table_requested, + worker_executable_override, + ), + "controller_kind": controller_kind, + "action_alphabet_size": 2, + }) +} + +fn write_common_certificate( + path: &Path, + kind: &str, + dataset_path: &Path, + timing: TimingCertificationTier, + controller_kind: &str, +) -> String { + write_common_certificate_with_runtime_profile( + path, + kind, + dataset_path, + timing, + controller_kind, + false, + ) +} + +fn write_common_certificate_with_runtime_profile( + path: &Path, + kind: &str, + dataset_path: &Path, + timing: TimingCertificationTier, + controller_kind: &str, + deterministic_table_requested: bool, +) -> String { + let value = common_certificate_with_runtime_profile( + kind, + dataset_path, + timing, + controller_kind, + deterministic_table_requested, + ); + write_json(path, &value); + crc32_hex(&fs::read(path).expect("read cert")) +} + +fn write_exact_reward_certificate( + path: &Path, + dataset_path: &Path, + timing: TimingCertificationTier, + controller_kind: &str, + max_reward: u64, +) -> String { + write_exact_reward_certificate_with_runtime_profile( + path, + dataset_path, + timing, + controller_kind, + max_reward, + false, + ) +} + +fn write_exact_reward_certificate_with_runtime_profile( + path: &Path, + dataset_path: &Path, + timing: TimingCertificationTier, + controller_kind: &str, + max_reward: u64, + deterministic_table_requested: bool, +) -> String { + write_exact_reward_certificate_with_runtime_profile_and_worker( + path, + dataset_path, + timing, + controller_kind, + max_reward, + deterministic_table_requested, + None, + ) +} + +fn write_exact_reward_certificate_with_runtime_profile_and_worker( + path: &Path, + dataset_path: &Path, + timing: TimingCertificationTier, + controller_kind: &str, + max_reward: u64, + deterministic_table_requested: bool, + worker_executable_override: Option<&Path>, +) -> String { + let mut value = common_certificate_with_runtime_profile_and_worker( + "exact_reward_encoding", + dataset_path, + timing, + controller_kind, + deterministic_table_requested, + worker_executable_override, + ); + let object = value.as_object_mut().expect("certificate object"); + object.insert( + "encoding".to_string(), + json!("integer_objective_difference"), + ); + object.insert( + "scalar_representation".to_string(), + json!("scalar://finite-f64"), + ); + object.insert("reward_bits".to_string(), json!(16)); + object.insert("max_reward".to_string(), json!(max_reward)); + write_json(path, &value); + crc32_hex(&fs::read(path).expect("read reward cert")) +} + +fn write_finite_reward_map_certificate( + path: &Path, + dataset_path: &Path, + timing: TimingCertificationTier, + controller_kind: &str, + max_reward: u64, + complete_nonnegative_interval_max: Option, + values: Value, +) -> String { + write_finite_reward_map_certificate_with_runtime_profile( + path, + dataset_path, + timing, + controller_kind, + max_reward, + complete_nonnegative_interval_max, + values, + false, + ) +} + +fn write_finite_reward_map_certificate_with_runtime_profile( + path: &Path, + dataset_path: &Path, + timing: TimingCertificationTier, + controller_kind: &str, + max_reward: u64, + complete_nonnegative_interval_max: Option, + values: Value, + deterministic_table_requested: bool, +) -> String { + let mut value = common_certificate_with_runtime_profile( + "exact_reward_encoding", + dataset_path, + timing, + controller_kind, + deterministic_table_requested, + ); + let object = value.as_object_mut().expect("certificate object"); + object.insert("encoding".to_string(), json!("finite_reward_map")); + object.insert( + "scalar_representation".to_string(), + json!("scalar://finite-f64"), + ); + object.insert("reward_bits".to_string(), json!(16)); + object.insert("max_reward".to_string(), json!(max_reward)); + if let Some(complete_max) = complete_nonnegative_interval_max { + object.insert( + "complete_nonnegative_interval_max".to_string(), + json!(complete_max), + ); + } + object.insert("values".to_string(), values); + write_json(path, &value); + crc32_hex(&fs::read(path).expect("read reward map cert")) +} + +fn write_observation_certificate( + path: &Path, + dataset_path: &Path, + timing: TimingCertificationTier, + controller_kind: &str, + finite_planner_state_certificate_crc32: &str, +) -> String { + write_observation_certificate_with_runtime_profile( + path, + dataset_path, + timing, + controller_kind, + finite_planner_state_certificate_crc32, + false, + ) +} + +fn write_observation_certificate_with_runtime_profile( + path: &Path, + dataset_path: &Path, + timing: TimingCertificationTier, + controller_kind: &str, + finite_planner_state_certificate_crc32: &str, + deterministic_table_requested: bool, +) -> String { + let mut value = common_certificate_with_runtime_profile( + "exact_state_observation", + dataset_path, + timing, + controller_kind, + deterministic_table_requested, + ); + let object = value.as_object_mut().expect("certificate object"); + object.insert("observation_key_mode".to_string(), json!("full_stream")); + object.insert( + "exact_state_encoder_spec_ref".to_string(), + json!("encoder://full-state"), + ); + object.insert( + "observation_adapter_spec_ref".to_string(), + json!("single-channel-conditional-byte-adapter-v1"), + ); + object.insert( + "observation_adapter_content_crc32".to_string(), + json!(observation_adapter_crc32()), + ); + object.insert( + "finite_planner_state_certificate_crc32".to_string(), + json!(finite_planner_state_certificate_crc32), + ); + object.insert( + "psi_h_outputs".to_string(), + json!([ + {"state_id": "baseline", "observations": [1]}, + {"state_id": "terminal", "observations": [2]} + ]), + ); + write_json(path, &value); + crc32_hex(&fs::read(path).expect("read observation cert")) +} + +fn write_deterministic_table( + path: &Path, + dataset_path: &Path, + controller_kind: &str, + baseline_candidate_crc32: &str, +) -> String { + write_deterministic_table_with_peak_memory( + path, + dataset_path, + controller_kind, + baseline_candidate_crc32, + 1, + ) +} + +fn write_deterministic_table_with_peak_memory( + path: &Path, + dataset_path: &Path, + controller_kind: &str, + baseline_candidate_crc32: &str, + peak_memory_bytes: u64, +) -> String { + let mut value = common_certificate_with_runtime_profile( + "deterministic_evaluator_table", + dataset_path, + TimingCertificationTier::DeterministicTable, + controller_kind, + true, + ); + let object = value.as_object_mut().expect("certificate object"); + object.insert( + "rows".to_string(), + json!([{ + "candidate_crc32": baseline_candidate_crc32, + "status": "success", + "compressed_bytes": 16, + "target_loss_bits": 128.0, + "elapsed_seconds": 0.01, + "peak_memory_bytes": peak_memory_bytes + }]), + ); + write_json(path, &value); + crc32_hex(&fs::read(path).expect("read table cert")) +} + +fn mismatch_marker_value(err: &str, marker: &str) -> Option { + let start = err.find(marker)? + marker.len(); + let rest = &err[start..]; + let end = rest.find('\'')?; + Some(rest[..end].to_string()) +} + +fn derive_runtime_warmstart_task_fingerprint( + args: &[String], + teacher_path: &Path, + reward_cert_crc32: &str, +) -> TaskFingerprint { + let request = parse_tune_command_args(args).expect("parse tune args for warmstart probe"); + write_teacher(teacher_path, probe_task_fingerprint(), reward_cert_crc32); + let err = run_tune(&request) + .expect_err("warmstart probe must fail before teacher task_fingerprint is corrected"); + if let Some(value) = mismatch_marker_value(&err, "current planner_run '") { + return TaskFingerprint::parse_hex(&value) + .expect("runtime planner_run fingerprint must be canonical hex"); + } + panic!("warmstart probe must expose current planner_run fingerprint marker, got: {err}"); +} + +fn tune_args( + spec_path: &Path, + timing: TimingCertificationTier, + max_evaluations: usize, +) -> Vec { + let args = vec![ + "infotheory".to_string(), + "tune".to_string(), + path_string(spec_path), + "--max-evaluations".to_string(), + max_evaluations.to_string(), + "--timing-tier".to_string(), + timing_label(timing).to_string(), + "--scalar-representation-ref".to_string(), + "scalar://finite-f64".to_string(), + "--exact-state-encoder-spec-ref".to_string(), + "encoder://full-state".to_string(), + "--claim-exact-finite-mdp".to_string(), + "--claim-exact-observed-markov".to_string(), + "--claim-planner-convergence".to_string(), + ]; + args +} + +fn run_tune_case( + dir: &Path, + label: &str, + dataset_path: &Path, + teacher_path: &Path, + case: &ControllerCase, + timing: TimingCertificationTier, + max_evaluations: usize, +) -> Value { + let suffix = format!("{label}_{}_{}", case.kind, timing_label(timing)); + let spec_path = dir.join(format!("{suffix}.json")); + let output_path = dir.join(format!("{suffix}_output.json")); + let report_path = dir.join(format!("{suffix}_report.json")); + let teacher = case.needs_teacher.then_some(teacher_path); + if case.needs_teacher { + write_placeholder_teacher(teacher_path); + } + let spec_value = tune_spec( + dataset_path, + &output_path, + &report_path, + case.controller.clone(), + teacher, + ); + write_json(&spec_path, &spec_value); + let (_spec_crc32, _baseline_crc32) = compiled_tune_hashes(&spec_value, dir); + let mut args = tune_args(&spec_path, timing, max_evaluations); + if matches!(case.kind, "mc_aixi_fac_ctw" | "aiqi_warmstart_exact_jh") { + let reward_cert_path = dir.join(format!("{suffix}_exact_reward.json")); + let reward_cert_crc32 = write_exact_reward_certificate( + &reward_cert_path, + dataset_path, + timing, + case.kind, + 65_535, + ); + args.push("--exact-reward-encoding-certificate".to_string()); + args.push(path_string(&reward_cert_path)); + if case.needs_teacher { + let task_fingerprint = + derive_runtime_warmstart_task_fingerprint(&args, teacher_path, &reward_cert_crc32); + write_teacher(teacher_path, task_fingerprint, &reward_cert_crc32); + } + } + let request = parse_tune_command_args(&args).expect("parse tune args"); + run_tune(&request).expect("run tune"); + assert!(output_path.exists()); + read_json(&report_path) +} + +fn assert_required_report_fields(report: &Value) { + assert_eq!(str_at(report, "/kind"), "tune_report"); + assert_eq!(u64_at(report, "/schema_version"), 1); + assert!(str_at(report, "/spec_crc32").len() == 8); + assert!(str_at(report, "/evaluator_profile_crc32").len() == 8); + assert!(str_at(report, "/provenance/bounds_crc32").len() == 8); + assert!(str_at(report, "/input_asset/content_crc32").len() == 8); + assert!(str_at(report, "/input_asset/lowered_skeleton_crc32").len() == 8); + assert!(str_at(report, "/baseline/candidate_crc32").len() == 8); + assert!(str_at(report, "/best/candidate_crc32").len() == 8); + assert_eq!( + str_at(report, "/cache/key_candidate_crc32"), + str_at(report, "/best/candidate_crc32") + ); + assert!(str_at(report, "/cache/key_digest_crc32").len() == 8); + assert!( + report["feature_set"] + .as_array() + .expect("feature_set array") + .iter() + .any(|item| item.as_str() == Some("tuner")) + ); + assert!(u64_at(report, "/baseline/model_bytes") > 0); + assert!(u64_at(report, "/best/model_bytes") > 0); + assert_eq!(str_at(report, "/baseline/status"), "success"); + assert_eq!(str_at(report, "/best/status"), "success"); + assert!(f64_at(report, "/baseline/target_loss_bits").is_finite()); + assert!(f64_at(report, "/baseline/objective_bits").is_finite()); + assert!(f64_at(report, "/baseline/throughput_runtime_cap_seconds").is_finite()); + assert!(f64_at(report, "/baseline/effective_eval_time_limit_seconds").is_finite()); + assert!(u64_at(report, "/cache/candidate_evaluations_executed") >= 1); + assert!(u64_at(report, "/search/fatal_evaluator_failures") <= 1); + let counted_results = u64_at(report, "/search/candidate_result_counts/success_deployable") + + u64_at( + report, + "/search/candidate_result_counts/success_non_deployable", + ) + + u64_at(report, "/search/candidate_result_counts/timeout") + + u64_at(report, "/search/candidate_result_counts/invalid") + + u64_at(report, "/search/candidate_result_counts/error_recoverable"); + assert_eq!( + counted_results, + u64_at(report, "/search/non_warmup_candidate_results_seen") + ); + let _external_asset_forbidden = u64_at( + report, + "/search/invalid_reason_counts/candidate_external_asset_forbidden", + ); + assert!(matches!( + str_at(report, "/provenance/canonical_code_certification/basis"), + "structural_self_delimiting_binary_encoding_plus_tests" + )); + assert!(bool_at( + report, + "/provenance/canonical_code_certification/top_level_length_prefix" + )); + assert!(bool_at( + report, + "/provenance/canonical_code_certification/trailing_bytes_rejected" + )); + assert!(bool_at( + report, + "/baseline/physical_compressed_bytes_diagnostic_only" + )); + assert!(bool_at( + report, + "/best/physical_compressed_bytes_diagnostic_only" + )); + assert!(bool_at(report, "/output/output_written")); + assert!(!str_at(report, "/output/output_config_path").is_empty()); + assert!(str_at(report, "/output/output_candidate_crc32").len() == 8); +} + +#[test] +fn canonical_tune_document_rejects_executor_side_fields() { + let dir = temp_dir("canonical_rejects_executor_fields"); + let dataset_path = dir.join("dataset.bin"); + let output_path = dir.join("output.json"); + let report_path = dir.join("report.json"); + write_passive_dataset(&dataset_path); + let base = tune_spec( + &dataset_path, + &output_path, + &report_path, + json!({ + "kind": "annealed_hill_climbing", + "max_mutation_radius": 1, + }), + None, + ); + for field in ["execution_profile", "theorem", "max_evaluations"] { + let mut rejected = base.clone(); + rejected[field] = json!({}); + let err = match SpecDocument::parse_json_value(&rejected, &dir) { + Ok(_) => panic!("canonical tune document accepted executor-side field {field}"), + Err(err) => err, + }; + assert!( + err.to_string() + .contains(&format!("unknown tune field '{field}'")), + "{err}" + ); + } + let _ = fs::remove_dir_all(dir); +} + +#[test] +fn canonical_tune_document_rejects_unknown_nested_fields() { + let dir = temp_dir("canonical_rejects_nested_fields"); + let dataset_path = dir.join("dataset.bin"); + let output_path = dir.join("output.json"); + let report_path = dir.join("report.json"); + write_passive_dataset(&dataset_path); + let base = tune_spec( + &dataset_path, + &output_path, + &report_path, + json!({ + "kind": "annealed_hill_climbing", + "max_mutation_radius": 1, + }), + None, + ); + let cases = [ + ( + "/assets/0", + "unexpected_asset_field", + "unknown tune.assets[0] field 'unexpected_asset_field'", + ), + ( + "/controller", + "unexpected_controller_field", + "unknown controller.annealed_hill_climbing field 'unexpected_controller_field'", + ), + ( + "/bounds", + "unexpected_bounds_field", + "unknown bounds field 'unexpected_bounds_field'", + ), + ( + "/bounds/parameter_ranges/0", + "unexpected_range_field", + "unknown bounds.parameter_ranges[0] field 'unexpected_range_field'", + ), + ]; + for (pointer, field, expected) in cases { + let mut rejected = base.clone(); + rejected + .pointer_mut(pointer) + .unwrap_or_else(|| panic!("{pointer} exists"))[field] = json!(true); + let err = match SpecDocument::parse_json_value(&rejected, &dir) { + Ok(_) => panic!("canonical tune document accepted nested field {field}"), + Err(err) => err, + }; + assert!(err.to_string().contains(expected), "{err}"); + } + let mut rejected = base.clone(); + rejected["controller"] = json!({ + "kind": "mc_aixi_fac_ctw", + "interface": { + "observation_bits": 8, + "observation_stream_len": 1, + "observation_key_mode": "full_stream", + "reward_bits": 8, + "agent_actions": 2, + "unexpected_interface_field": true, + }, + "planner_simulations_per_step": 2, + }); + let err = match SpecDocument::parse_json_value(&rejected, &dir) { + Ok(_) => panic!("canonical tune document accepted nested interface field"), + Err(err) => err, + }; + assert!( + err.to_string() + .contains("unknown controller.interface field 'unexpected_interface_field'"), + "{err}" + ); + + let mut rejected = base.clone(); + rejected["baseline_candidate"]["unexpected_candidate_field"] = json!(true); + let err = match SpecDocument::parse_json_value(&rejected, &dir) { + Ok(_) => panic!("canonical tune document accepted unknown baseline_candidate field"), + Err(err) => err, + }; + assert!( + err.to_string() + .contains("baseline_candidate must be canonical compression backend JSON"), + "{err}" + ); + + let mut rejected = base.clone(); + rejected["baseline_candidate"]["rate_backend"]["unexpected_rate_backend_field"] = json!(true); + let err = match SpecDocument::parse_json_value(&rejected, &dir) { + Ok(_) => panic!("canonical tune document accepted unknown rate_backend field"), + Err(err) => err, + }; + assert!( + err.to_string() + .contains("baseline_candidate must be canonical compression backend JSON"), + "{err}" + ); + + let _ = fs::remove_dir_all(dir); +} + +#[test] +fn canonical_tune_document_rejects_candidate_local_external_assets() { + const EXTERNAL_ASSET_FORBIDDEN: &str = "candidate_external_asset_forbidden"; + let dir = temp_dir("canonical_rejects_candidate_external_assets"); + let dataset_path = dir.join("dataset.bin"); + let output_path = dir.join("output.json"); + let report_path = dir.join("report.json"); + write_passive_dataset(&dataset_path); + for (field, value) in [ + ("model_path", json!("weights.safetensors")), + ("spec_path", json!("nested/spec.json")), + ("base_path", json!("nested/base.json")), + ] { + let mut candidate = baseline_candidate(); + candidate[field] = value; + let spec = tune_spec( + &dataset_path, + &output_path, + &report_path, + json!({ + "kind": "annealed_hill_climbing", + "max_mutation_radius": 1, + }), + None, + ); + let mut rejected = spec.clone(); + rejected["baseline_candidate"] = candidate; + let err = match SpecDocument::parse_json_value(&rejected, &dir) { + Ok(_) => panic!("candidate-local external asset field must fail"), + Err(err) => err, + }; + assert!(err.to_string().contains(EXTERNAL_ASSET_FORBIDDEN), "{err}"); + assert!( + err.to_string().contains("candidate-local external asset"), + "{err}" + ); + } + let mut rejected = tune_spec( + &dataset_path, + &output_path, + &report_path, + json!({ + "kind": "annealed_hill_climbing", + "max_mutation_radius": 1, + }), + None, + ); + rejected["baseline_candidate"]["rate_backend"]["method"] = + json!("online;policy:load_from=weights.safetensors"); + let err = match SpecDocument::parse_json_value(&rejected, &dir) { + Ok(_) => panic!("policy load_from must fail"), + Err(err) => err, + }; + assert!(err.to_string().contains(EXTERNAL_ASSET_FORBIDDEN), "{err}"); + assert!(err.to_string().contains("policy load_from"), "{err}"); + + let mut rejected = tune_spec( + &dataset_path, + &output_path, + &report_path, + json!({ + "kind": "annealed_hill_climbing", + "max_mutation_radius": 1, + }), + None, + ); + rejected["baseline_candidate"]["rate_backend"]["method"] = json!({ + "kind": "file", + "path": "weights.safetensors", + }); + let err = match SpecDocument::parse_json_value(&rejected, &dir) { + Ok(_) => panic!("method.path candidate-local asset must fail"), + Err(err) => err, + }; + assert!(err.to_string().contains(EXTERNAL_ASSET_FORBIDDEN), "{err}"); + assert!( + err.to_string() + .contains("candidate-local external asset field"), + "{err}" + ); + + let _ = fs::remove_dir_all(dir); +} + +#[test] +fn canonical_tune_document_requires_assets_array() { + let dir = temp_dir("canonical_requires_assets"); + let dataset_path = dir.join("dataset.bin"); + let output_path = dir.join("output.json"); + let report_path = dir.join("report.json"); + write_passive_dataset(&dataset_path); + let base = tune_spec( + &dataset_path, + &output_path, + &report_path, + json!({ + "kind": "annealed_hill_climbing", + "max_mutation_radius": 1, + }), + None, + ); + let mut missing = base.clone(); + missing.as_object_mut().expect("object").remove("assets"); + let err = match SpecDocument::parse_json_value(&missing, &dir) { + Ok(_) => panic!("canonical tune document accepted missing assets"), + Err(err) => err, + }; + assert!(err.to_string().contains("tune.assets is required"), "{err}"); + let mut wrong_type = base; + wrong_type["assets"] = json!({}); + let err = match SpecDocument::parse_json_value(&wrong_type, &dir) { + Ok(_) => panic!("canonical tune document accepted non-array assets"), + Err(err) => err, + }; + assert!( + err.to_string().contains("tune.assets must be an array"), + "{err}" + ); + let _ = fs::remove_dir_all(dir); +} + +#[cfg(unix)] +#[test] +fn tune_cli_loads_binary_itsd_tune_document() { + let dir = temp_dir("tune_itsd_cli"); + let dataset_path = dir.join("dataset.bin"); + let spec_path = dir.join("spec.itsd"); + let output_path = dir.join("output.json"); + let report_path = dir.join("report.json"); + write_passive_dataset(&dataset_path); + let spec_value = tune_spec( + &dataset_path, + &output_path, + &report_path, + json!({ + "kind": "annealed_hill_climbing", + "max_mutation_radius": 1, + }), + None, + ); + write_tune_itsd(&spec_path, &spec_value, &dir); + let args = [ + "infotheory".to_string(), + "tune".to_string(), + path_string(&spec_path), + "--max-evaluations".to_string(), + "1".to_string(), + ]; + let request = parse_tune_command_args(&args).expect("parse tune args"); + run_tune(&request).expect("run tune from itsd"); + let report = read_json(&report_path); + assert_eq!(str_at(&report, "/kind"), "tune_report"); + assert!(bool_at(&report, "/output/output_written")); + let _ = fs::remove_dir_all(dir); +} + +#[cfg(not(unix))] +#[test] +fn run_tune_reports_unix_only_runtime_contract_on_non_unix() { + let dir = temp_dir("non_unix_runtime_contract"); + let dataset_path = dir.join("dataset.bin"); + let spec_path = dir.join("spec.json"); + let output_path = dir.join("output.json"); + let report_path = dir.join("report.json"); + write_passive_dataset(&dataset_path); + write_json( + &spec_path, + &tune_spec( + &dataset_path, + &output_path, + &report_path, + json!({ + "kind": "annealed_hill_climbing", + "max_mutation_radius": 1, + }), + None, + ), + ); + let args = [ + "infotheory".to_string(), + "tune".to_string(), + path_string(&spec_path), + "--max-evaluations".to_string(), + "1".to_string(), + ]; + let request = parse_tune_command_args(&args).expect("parse tune args"); + let err = run_tune(&request).expect_err("non-unix should reject process-isolated runtime"); + assert!( + err.contains("tuner requires a Unix target for process-isolated candidate evaluation"), + "{err}" + ); + let _ = fs::remove_dir_all(dir); +} + +#[test] +fn tune_executor_accepts_supported_controls() { + let supported = [ + vec!["--threads", "2"], + vec!["--cpu-affinity", "0"], + vec!["--evaluator-worker-executable", "/tmp/infotheory-worker"], + vec![ + "--evaluator-cgroup-parent", + "/sys/fs/cgroup/infotheory-tuner", + ], + vec!["--log-path", "tune.log"], + vec!["--diagnostic-chunk-bytes", "4096"], + vec!["--rss-mode", "process_rss_peak"], + vec!["--rss-mode", "backend_reported"], + vec!["--rss-mode", "hybrid_strict_max"], + vec!["--planner-deployable-model"], + vec!["--warmstart-trace-refresh"], + vec![ + "--annealer-kernel-profile", + "compiled_uniform_metropolis_hastings", + ], + ]; + for flags in supported { + let mut args = vec![ + "infotheory".to_string(), + "tune".to_string(), + "spec.json".to_string(), + ]; + args.extend(flags.iter().map(|flag| (*flag).to_string())); + assert!( + parse_tune_command_args(&args).is_ok(), + "supported executor flags were rejected: {flags:?}" + ); + } +} + +#[cfg(unix)] +#[test] +fn run_tune_rejects_unusable_worker_executable() { + let dir = temp_dir("invalid_worker_executable"); + let dataset_path = dir.join("dataset.bin"); + let spec_path = dir.join("spec.json"); + let output_path = dir.join("output.json"); + let report_path = dir.join("report.json"); + let missing_worker = dir.join("missing-worker-bin"); + write_passive_dataset(&dataset_path); + write_json( + &spec_path, + &tune_spec( + &dataset_path, + &output_path, + &report_path, + json!({ + "kind": "annealed_hill_climbing", + "max_mutation_radius": 1 + }), + None, + ), + ); + let args = vec![ + "infotheory".to_string(), + "tune".to_string(), + path_string(&spec_path), + "--max-evaluations".to_string(), + "1".to_string(), + "--evaluator-worker-executable".to_string(), + path_string(&missing_worker), + ]; + let request = parse_tune_command_args(&args).expect("parse tune args"); + let err = run_tune(&request).expect_err("nonexistent worker executable must fail"); + assert!( + err.contains("execution.evaluator_worker_executable") + && err.contains("does not resolve to a file"), + "{err}" + ); + let _ = fs::remove_dir_all(dir); +} + +#[cfg(unix)] +#[test] +fn run_tune_strict_memory_mode_requires_delegated_cgroup_parent() { + let dir = temp_dir("strict_memory_requires_cgroup"); + let dataset_path = dir.join("dataset.bin"); + let spec_path = dir.join("spec.json"); + let output_path = dir.join("output.json"); + let report_path = dir.join("report.json"); + write_passive_dataset(&dataset_path); + write_json( + &spec_path, + &tune_spec( + &dataset_path, + &output_path, + &report_path, + json!({ + "kind": "annealed_hill_climbing", + "max_mutation_radius": 1 + }), + None, + ), + ); + let args = vec![ + "infotheory".to_string(), + "tune".to_string(), + path_string(&spec_path), + "--max-evaluations".to_string(), + "1".to_string(), + "--rss-mode".to_string(), + "hybrid_strict_max".to_string(), + ]; + let request = parse_tune_command_args(&args).expect("parse tune args"); + let err = run_tune(&request) + .expect_err("strict memory-accounting mode without cgroup parent must fail"); + #[cfg(target_os = "linux")] + assert!( + err.contains("strict memory-accounting mode") && err.contains("delegated cgroup-v2 parent"), + "{err}" + ); + #[cfg(not(target_os = "linux"))] + assert!( + err.contains("strict memory-accounting mode") && err.contains("Linux"), + "{err}" + ); + let _ = fs::remove_dir_all(dir); +} + +#[test] +fn tune_executor_rejects_zero_diagnostic_chunk_bytes() { + let args = vec![ + "infotheory".to_string(), + "tune".to_string(), + "spec.json".to_string(), + "--diagnostic-chunk-bytes".to_string(), + "0".to_string(), + ]; + let err = parse_tune_command_args(&args).expect_err("zero chunk size should fail"); + assert!(err.contains("diagnostic_chunk_bytes must be >= 1"), "{err}"); +} + +#[test] +fn tune_exec_config_cli_overrides_are_order_independent() { + let dir = temp_dir("exec_config_precedence"); + let exec_path = dir.join("exec.json"); + write_json( + &exec_path, + &json!({ + "max_evaluations": 1, + "warmup_baseline_runs": 1, + "theorem": { + "timing_certification_tier": "isolated" + } + }), + ); + let exec_arg = path_string(&exec_path); + let before = [ + "infotheory", + "tune", + "spec.json", + "--max-evaluations", + "3", + "--timing-tier", + "real_time", + "--exec-config", + exec_arg.as_str(), + ] + .iter() + .map(|item| (*item).to_string()) + .collect::>(); + let after = [ + "infotheory", + "tune", + "spec.json", + "--exec-config", + exec_arg.as_str(), + "--max-evaluations", + "3", + "--timing-tier", + "real_time", + ] + .iter() + .map(|item| (*item).to_string()) + .collect::>(); + let before = parse_tune_command_args(&before).expect("parse before"); + let after = parse_tune_command_args(&after).expect("parse after"); + assert_eq!(before.execution.max_evaluations, Some(3)); + assert_eq!(after.execution.max_evaluations, Some(3)); + assert_eq!(before.execution.warmup_baseline_runs, 1); + assert_eq!(after.execution.warmup_baseline_runs, 1); + assert_eq!( + before.execution.theorem.timing_certification_tier, + TimingCertificationTier::RealTime + ); + assert_eq!( + after.execution.theorem.timing_certification_tier, + TimingCertificationTier::RealTime + ); + let _ = fs::remove_dir_all(dir); +} + +#[test] +fn tune_exec_config_rejects_unknown_fields_and_malformed_theorem() { + let dir = temp_dir("exec_config_strict"); + let unknown_path = dir.join("unknown.json"); + write_json( + &unknown_path, + &json!({ + "max_evaluations": 1, + "determinism_dealine_certificate": "typo" + }), + ); + let err = parse_tune_command_args(&[ + "infotheory".to_string(), + "tune".to_string(), + "spec.json".to_string(), + "--exec-config".to_string(), + path_string(&unknown_path), + ]) + .expect_err("unknown key must be rejected"); + assert!(err.contains("unknown execution config field"), "{err}"); + + let malformed_path = dir.join("malformed_theorem.json"); + write_json( + &malformed_path, + &json!({ + "max_evaluations": 1, + "theorem": "not an object" + }), + ); + let err = parse_tune_command_args(&[ + "infotheory".to_string(), + "tune".to_string(), + "spec.json".to_string(), + "--exec-config".to_string(), + path_string(&malformed_path), + ]) + .expect_err("malformed theorem value must be rejected"); + assert!(err.contains("field 'theorem' must be an object"), "{err}"); + + let empty_cert_path = dir.join("empty_certificate.json"); + write_json( + &empty_cert_path, + &json!({ + "max_evaluations": 1, + "theorem": { + "finite_planner_state_certificate": "" + } + }), + ); + let err = parse_tune_command_args(&[ + "infotheory".to_string(), + "tune".to_string(), + "spec.json".to_string(), + "--exec-config".to_string(), + path_string(&empty_cert_path), + ]) + .expect_err("empty theorem certificate reference must be rejected"); + assert!( + err.contains("finite_planner_state_certificate must be a non-empty string"), + "{err}" + ); + let _ = fs::remove_dir_all(dir); +} + +#[cfg(unix)] +#[test] +fn tune_executor_rejects_unsupported_certificate_uri_scheme() { + let dir = temp_dir("unsupported_certificate_uri"); + let dataset_path = dir.join("dataset.bin"); + let spec_path = dir.join("spec.json"); + let output_path = dir.join("output.json"); + let report_path = dir.join("report.json"); + write_passive_dataset(&dataset_path); + write_json( + &spec_path, + &tune_spec( + &dataset_path, + &output_path, + &report_path, + json!({ + "kind": "annealed_hill_climbing", + "max_mutation_radius": 1, + }), + None, + ), + ); + let args = [ + "infotheory".to_string(), + "tune".to_string(), + path_string(&spec_path), + "--determinism-deadline-certificate".to_string(), + "https://example.invalid/deadline.json".to_string(), + ]; + let request = parse_tune_command_args(&args).expect("parse tune args"); + let err = run_tune(&request).expect_err("unsupported certificate URI must fail"); + assert!( + err.contains("unsupported theorem certificate reference scheme"), + "{err}" + ); + let _ = fs::remove_dir_all(dir); +} + +#[cfg(unix)] +#[test] +fn run_tune_records_log_controls() { + let dir = temp_dir("executor_logging"); + let dataset_path = dir.join("dataset.bin"); + let spec_path = dir.join("spec.json"); + let output_path = dir.join("output.json"); + let report_path = dir.join("report.json"); + let log_path = dir.join("tune.log"); + write_passive_dataset(&dataset_path); + write_json( + &spec_path, + &tune_spec( + &dataset_path, + &output_path, + &report_path, + json!({ + "kind": "annealed_hill_climbing", + "max_mutation_radius": 1, + }), + None, + ), + ); + let args = [ + "infotheory", + "tune", + path_string(&spec_path).as_str(), + "--max-evaluations", + "1", + "--log-path", + path_string(&log_path).as_str(), + "--diagnostic-chunk-bytes", + "2", + ] + .iter() + .map(|item| (*item).to_string()) + .collect::>(); + let request = parse_tune_command_args(&args).expect("parse tune args"); + run_tune(&request).expect("run tune"); + let report = read_json(&report_path); + assert_eq!( + str_at(&report, "/provenance/executor_controls/log_path"), + path_string(&log_path) + ); + assert_eq!( + u64_at( + &report, + "/provenance/executor_controls/diagnostic_chunk_bytes" + ), + 2 + ); + assert!(bool_at(&report, "/provenance/diagnostic_chunking/enabled")); + assert_eq!( + u64_at(&report, "/provenance/diagnostic_chunking/chunk_count"), + 21 + ); + assert_eq!( + u64_at(&report, "/input_asset/diagnostic_chunking/chunk_bytes"), + 2 + ); + assert!(!bool_at( + &report, + "/input_asset/diagnostic_chunking/affects_objective" + )); + let log = fs::read_to_string(&log_path).expect("read executor log"); + assert!(log.contains("\"event\":\"start\""), "{log}"); + assert!(log.contains("\"event\":\"finish\""), "{log}"); + let _ = fs::remove_dir_all(dir); +} + +#[cfg(unix)] +#[test] +fn exact_family_controller_rejects_missing_exact_reward_certificate() { + let dir = temp_dir("missing_exact_reward_certificate"); + let dataset_path = dir.join("dataset.bin"); + let spec_path = dir.join("spec.json"); + let output_path = dir.join("output.json"); + let report_path = dir.join("report.json"); + write_passive_dataset(&dataset_path); + let mc_case = controller_cases() + .into_iter() + .find(|case| case.kind == "mc_aixi_fac_ctw") + .expect("mc-aixi case"); + write_json( + &spec_path, + &tune_spec( + &dataset_path, + &output_path, + &report_path, + mc_case.controller, + None, + ), + ); + let args = [ + "infotheory".to_string(), + "tune".to_string(), + path_string(&spec_path), + "--max-evaluations".to_string(), + "1".to_string(), + ]; + let request = parse_tune_command_args(&args).expect("parse tune args"); + let err = run_tune(&request).expect_err("exact controller must reject missing certificate"); + assert!(err.contains("reward_encoding_unsafe"), "{err}"); + let _ = fs::remove_dir_all(dir); +} + +#[cfg(unix)] +#[test] +fn exact_reward_certificate_rejects_unrepresentable_reachable_reward() { + let dir = temp_dir("bad_exact_reward_certificate"); + let dataset_path = dir.join("dataset.bin"); + let spec_path = dir.join("spec.json"); + let output_path = dir.join("output.json"); + let report_path = dir.join("report.json"); + let reward_cert_path = dir.join("reward.json"); + write_passive_dataset(&dataset_path); + let mc_case = controller_cases() + .into_iter() + .find(|case| case.kind == "mc_aixi_fac_ctw") + .expect("mc-aixi case"); + write_json( + &spec_path, + &tune_spec( + &dataset_path, + &output_path, + &report_path, + mc_case.controller, + None, + ), + ); + write_exact_reward_certificate( + &reward_cert_path, + &dataset_path, + TimingCertificationTier::BestEffort, + "mc_aixi_fac_ctw", + 1, + ); + let args = [ + "infotheory".to_string(), + "tune".to_string(), + path_string(&spec_path), + "--max-evaluations".to_string(), + "1".to_string(), + "--scalar-representation-ref".to_string(), + "scalar://finite-f64".to_string(), + "--exact-reward-encoding-certificate".to_string(), + path_string(&reward_cert_path), + ]; + let request = parse_tune_command_args(&args).expect("parse tune args"); + let err = run_tune(&request).expect_err("too-small reward certificate must fail"); + assert!( + err.contains("exceeds verified exact reward maximum"), + "{err}" + ); + let _ = fs::remove_dir_all(dir); +} + +#[cfg(unix)] +#[test] +fn exact_controller_rejects_finite_reward_map_without_complete_interval() { + let dir = temp_dir("incomplete_finite_reward_map"); + let dataset_path = dir.join("dataset.bin"); + let spec_path = dir.join("spec.json"); + let output_path = dir.join("output.json"); + let report_path = dir.join("report.json"); + let reward_cert_path = dir.join("reward_map.json"); + write_passive_dataset(&dataset_path); + let mc_case = controller_cases() + .into_iter() + .find(|case| case.kind == "mc_aixi_fac_ctw") + .expect("mc-aixi case"); + let spec_value = tune_spec( + &dataset_path, + &output_path, + &report_path, + mc_case.controller, + None, + ); + write_json(&spec_path, &spec_value); + let reward_values = (0..=1_000u64) + .map(|objective_difference| { + json!({ + "objective_difference": objective_difference, + "symbol": objective_difference + }) + }) + .collect::>(); + write_finite_reward_map_certificate( + &reward_cert_path, + &dataset_path, + TimingCertificationTier::BestEffort, + "mc_aixi_fac_ctw", + 65_535, + None, + Value::Array(reward_values), + ); + let args = [ + "infotheory".to_string(), + "tune".to_string(), + path_string(&spec_path), + "--max-evaluations".to_string(), + "1".to_string(), + "--scalar-representation-ref".to_string(), + "scalar://finite-f64".to_string(), + "--exact-reward-encoding-certificate".to_string(), + path_string(&reward_cert_path), + ]; + let request = parse_tune_command_args(&args).expect("parse tune args"); + let err = run_tune(&request).expect_err("incomplete finite map must fail before runtime"); + assert!(err.contains("complete_nonnegative_interval_max"), "{err}"); + let _ = fs::remove_dir_all(dir); +} + +#[test] +fn deterministic_table_certificates_can_certify_theorem_claims_when_used() { + let dir = temp_dir("deterministic_table_certified"); + let dataset_path = dir.join("dataset.bin"); + let spec_path = dir.join("spec.json"); + let output_path = dir.join("output.json"); + let report_path = dir.join("report.json"); + write_passive_dataset(&dataset_path); + let mc_case = controller_cases() + .into_iter() + .find(|case| case.kind == "mc_aixi_fac_ctw") + .expect("mc-aixi case"); + let spec_value = tune_spec( + &dataset_path, + &output_path, + &report_path, + mc_case.controller, + None, + ); + write_json(&spec_path, &spec_value); + let (_, baseline_candidate_crc32) = compiled_tune_hashes(&spec_value, &dir); + + let finite_cert = dir.join("finite.json"); + let no_hidden_cert = dir.join("no_hidden.json"); + let reward_cert = dir.join("reward.json"); + let observation_cert = dir.join("observation.json"); + let table_cert = dir.join("table.json"); + let finite_cert_crc32 = write_common_certificate_with_runtime_profile( + &finite_cert, + "finite_planner_state", + &dataset_path, + TimingCertificationTier::DeterministicTable, + "mc_aixi_fac_ctw", + true, + ); + write_common_certificate_with_runtime_profile( + &no_hidden_cert, + "no_hidden_state", + &dataset_path, + TimingCertificationTier::DeterministicTable, + "mc_aixi_fac_ctw", + true, + ); + write_exact_reward_certificate_with_runtime_profile( + &reward_cert, + &dataset_path, + TimingCertificationTier::DeterministicTable, + "mc_aixi_fac_ctw", + 65_535, + true, + ); + write_observation_certificate_with_runtime_profile( + &observation_cert, + &dataset_path, + TimingCertificationTier::DeterministicTable, + "mc_aixi_fac_ctw", + &finite_cert_crc32, + true, + ); + write_deterministic_table( + &table_cert, + &dataset_path, + "mc_aixi_fac_ctw", + &baseline_candidate_crc32, + ); + + let args = [ + "infotheory".to_string(), + "tune".to_string(), + path_string(&spec_path), + "--max-evaluations".to_string(), + "1".to_string(), + "--timing-tier".to_string(), + "deterministic_table".to_string(), + "--scalar-representation-ref".to_string(), + "scalar://finite-f64".to_string(), + "--finite-planner-state-certificate".to_string(), + path_string(&finite_cert), + "--no-hidden-state-certificate".to_string(), + path_string(&no_hidden_cert), + "--exact-reward-encoding-certificate".to_string(), + path_string(&reward_cert), + "--exact-state-encoder-spec-ref".to_string(), + "encoder://full-state".to_string(), + "--exact-state-observation-certificate".to_string(), + path_string(&observation_cert), + "--deterministic-evaluator-table".to_string(), + path_string(&table_cert), + "--claim-exact-finite-mdp".to_string(), + "--claim-exact-observed-markov".to_string(), + "--claim-planner-convergence".to_string(), + ]; + let request = parse_tune_command_args(&args).expect("parse tune args"); + run_tune(&request).expect("run tune"); + let report = read_json(&report_path); + assert_eq!( + str_at(&report, "/theorem_claims/exact_finite_mdp/status"), + "certified" + ); + assert_eq!( + str_at(&report, "/theorem_claims/exact_observed_markov/status"), + "certified" + ); + assert_eq!( + str_at(&report, "/theorem_claims/planner_convergence/status"), + "certified" + ); + assert!( + report + .pointer("/theorem_claims/refs/exact_state_observation_certified") + .is_none(), + "theorem refs must not expose unchecked user observation-certification booleans" + ); + assert!(bool_at( + &report, + "/theorem_claims/refs/exact_state_observation_basis/verified_certificate" + )); + assert_eq!( + str_at(&report, "/evaluator_execution_model"), + "deterministic_table" + ); + assert_eq!( + str_at(&report, "/theorem_timing_basis"), + "verified_deterministic_evaluator_table" + ); + assert_eq!(u64_at(&report, "/baseline/compressed_bytes"), 16); + assert_eq!( + u64_at( + &report, + "/provenance/verified_theorem_inputs/deterministic_evaluator_table/rows" + ), + 1 + ); + let _ = fs::remove_dir_all(dir); +} + +#[cfg(unix)] +#[test] +fn real_time_timing_certificate_sets_verified_timing_basis() { + let dir = temp_dir("real_time_timing_certified"); + let dataset_path = dir.join("dataset.bin"); + let spec_path = dir.join("spec.json"); + let output_path = dir.join("output.json"); + let report_path = dir.join("report.json"); + write_passive_dataset(&dataset_path); + let mc_case = controller_cases() + .into_iter() + .find(|case| case.kind == "mc_aixi_fac_ctw") + .expect("mc-aixi case"); + let spec_value = tune_spec( + &dataset_path, + &output_path, + &report_path, + mc_case.controller, + None, + ); + write_json(&spec_path, &spec_value); + + let finite_cert = dir.join("finite.json"); + let no_hidden_cert = dir.join("no_hidden.json"); + let reward_cert = dir.join("reward.json"); + let observation_cert = dir.join("observation.json"); + let timing_cert = dir.join("timing.json"); + let finite_cert_crc32 = write_common_certificate( + &finite_cert, + "finite_planner_state", + &dataset_path, + TimingCertificationTier::RealTime, + "mc_aixi_fac_ctw", + ); + write_common_certificate( + &no_hidden_cert, + "no_hidden_state", + &dataset_path, + TimingCertificationTier::RealTime, + "mc_aixi_fac_ctw", + ); + write_exact_reward_certificate( + &reward_cert, + &dataset_path, + TimingCertificationTier::RealTime, + "mc_aixi_fac_ctw", + 65_535, + ); + write_observation_certificate( + &observation_cert, + &dataset_path, + TimingCertificationTier::RealTime, + "mc_aixi_fac_ctw", + &finite_cert_crc32, + ); + write_common_certificate( + &timing_cert, + "determinism_deadline", + &dataset_path, + TimingCertificationTier::RealTime, + "mc_aixi_fac_ctw", + ); + + let args = [ + "infotheory".to_string(), + "tune".to_string(), + path_string(&spec_path), + "--max-evaluations".to_string(), + "1".to_string(), + "--timing-tier".to_string(), + "real_time".to_string(), + "--scalar-representation-ref".to_string(), + "scalar://finite-f64".to_string(), + "--finite-planner-state-certificate".to_string(), + path_string(&finite_cert), + "--no-hidden-state-certificate".to_string(), + path_string(&no_hidden_cert), + "--exact-reward-encoding-certificate".to_string(), + path_string(&reward_cert), + "--exact-state-encoder-spec-ref".to_string(), + "encoder://full-state".to_string(), + "--exact-state-observation-certificate".to_string(), + path_string(&observation_cert), + "--determinism-deadline-certificate".to_string(), + path_string(&timing_cert), + "--claim-planner-convergence".to_string(), + ]; + let request = parse_tune_command_args(&args).expect("parse tune args"); + run_tune(&request).expect("run real-time certified tune"); + let report = read_json(&report_path); + assert_eq!( + str_at(&report, "/theorem_timing_basis"), + "verified_real_time_deadline_certificate" + ); + assert_eq!( + str_at(&report, "/theorem_claims/planner_convergence/status"), + "uncertified" + ); + assert!( + report["theorem_claims"]["planner_convergence"]["missing_prerequisites"] + .as_array() + .expect("missing prereqs") + .iter() + .any(|item| item.as_str() == Some("strict_theorem_facing_memory_accounting")) + ); + let _ = fs::remove_dir_all(dir); +} + +#[test] +fn deterministic_table_peak_memory_can_make_baseline_nondeployable() { + let dir = temp_dir("deterministic_table_memory_cap"); + let dataset_path = dir.join("dataset.bin"); + let spec_path = dir.join("spec.json"); + let output_path = dir.join("output.json"); + let report_path = dir.join("report.json"); + write_passive_dataset(&dataset_path); + let mc_case = controller_cases() + .into_iter() + .find(|case| case.kind == "mc_aixi_fac_ctw") + .expect("mc-aixi case"); + let mut spec_value = tune_spec( + &dataset_path, + &output_path, + &report_path, + mc_case.controller, + None, + ); + spec_value["max_memory_bytes"] = json!(1u64); + write_json(&spec_path, &spec_value); + let (_, baseline_candidate_crc32) = compiled_tune_hashes(&spec_value, &dir); + let reward_cert = dir.join("reward.json"); + let table_cert = dir.join("table.json"); + write_exact_reward_certificate_with_runtime_profile( + &reward_cert, + &dataset_path, + TimingCertificationTier::DeterministicTable, + "mc_aixi_fac_ctw", + 65_535, + true, + ); + write_deterministic_table_with_peak_memory( + &table_cert, + &dataset_path, + "mc_aixi_fac_ctw", + &baseline_candidate_crc32, + 2, + ); + let args = [ + "infotheory".to_string(), + "tune".to_string(), + path_string(&spec_path), + "--max-evaluations".to_string(), + "1".to_string(), + "--timing-tier".to_string(), + "deterministic_table".to_string(), + "--scalar-representation-ref".to_string(), + "scalar://finite-f64".to_string(), + "--exact-reward-encoding-certificate".to_string(), + path_string(&reward_cert), + "--deterministic-evaluator-table".to_string(), + path_string(&table_cert), + ]; + let request = parse_tune_command_args(&args).expect("parse tune args"); + let err = run_tune(&request).expect_err("memory cap should reject baseline"); + assert!(err.contains("peak memory 2 bytes"), "{err}"); + let report = read_json(&report_path); + assert!(!bool_at(&report, "/baseline/deployable")); + assert_eq!(u64_at(&report, "/baseline/peak_memory_bytes"), 2); + assert_eq!(u64_at(&report, "/baseline/max_memory_bytes"), 1); + let _ = fs::remove_dir_all(dir); +} + +#[cfg(unix)] +#[test] +fn exact_state_observation_certificate_requires_injectivity_basis() { + let dir = temp_dir("observation_injectivity_rejected"); + let dataset_path = dir.join("dataset.bin"); + let spec_path = dir.join("spec.json"); + let output_path = dir.join("output.json"); + let report_path = dir.join("report.json"); + write_passive_dataset(&dataset_path); + let mc_case = controller_cases() + .into_iter() + .find(|case| case.kind == "mc_aixi_fac_ctw") + .expect("mc-aixi case"); + let spec_value = tune_spec( + &dataset_path, + &output_path, + &report_path, + mc_case.controller, + None, + ); + write_json(&spec_path, &spec_value); + + let finite_cert = dir.join("finite.json"); + let reward_cert = dir.join("reward.json"); + let observation_cert = dir.join("bad_observation.json"); + let finite_cert_crc32 = write_common_certificate( + &finite_cert, + "finite_planner_state", + &dataset_path, + TimingCertificationTier::BestEffort, + "mc_aixi_fac_ctw", + ); + write_exact_reward_certificate( + &reward_cert, + &dataset_path, + TimingCertificationTier::BestEffort, + "mc_aixi_fac_ctw", + 65_535, + ); + let mut bad_observation = common_certificate( + "exact_state_observation", + &dataset_path, + TimingCertificationTier::BestEffort, + "mc_aixi_fac_ctw", + ); + let object = bad_observation + .as_object_mut() + .expect("observation certificate object"); + object.insert("observation_key_mode".to_string(), json!("full_stream")); + object.insert( + "exact_state_encoder_spec_ref".to_string(), + json!("encoder://full-state"), + ); + object.insert( + "observation_adapter_spec_ref".to_string(), + json!("single-channel-conditional-byte-adapter-v1"), + ); + object.insert( + "observation_adapter_content_crc32".to_string(), + json!(observation_adapter_crc32()), + ); + object.insert( + "finite_planner_state_certificate_crc32".to_string(), + json!(finite_cert_crc32), + ); + object.insert( + "psi_h_outputs".to_string(), + json!([ + {"state_id": "state-a", "observations": [7]}, + {"state_id": "state-b", "observations": [7]} + ]), + ); + write_json(&observation_cert, &bad_observation); + + let args = [ + "infotheory".to_string(), + "tune".to_string(), + path_string(&spec_path), + "--max-evaluations".to_string(), + "1".to_string(), + "--scalar-representation-ref".to_string(), + "scalar://finite-f64".to_string(), + "--finite-planner-state-certificate".to_string(), + path_string(&finite_cert), + "--exact-reward-encoding-certificate".to_string(), + path_string(&reward_cert), + "--exact-state-encoder-spec-ref".to_string(), + "encoder://full-state".to_string(), + "--exact-state-observation-certificate".to_string(), + path_string(&observation_cert), + "--claim-exact-observed-markov".to_string(), + ]; + let request = parse_tune_command_args(&args).expect("parse tune args"); + let err = run_tune(&request).expect_err("non-injective observation proof must fail"); + assert!(err.contains("not injective"), "{err}"); + let _ = fs::remove_dir_all(dir); +} + +#[cfg(unix)] +#[test] +fn exact_state_observation_certificate_rejects_duplicate_state_ids() { + let dir = temp_dir("observation_duplicate_state_id"); + let dataset_path = dir.join("dataset.bin"); + let spec_path = dir.join("spec.json"); + let output_path = dir.join("output.json"); + let report_path = dir.join("report.json"); + write_passive_dataset(&dataset_path); + let mc_case = controller_cases() + .into_iter() + .find(|case| case.kind == "mc_aixi_fac_ctw") + .expect("mc-aixi case"); + let spec_value = tune_spec( + &dataset_path, + &output_path, + &report_path, + mc_case.controller, + None, + ); + write_json(&spec_path, &spec_value); + + let finite_cert = dir.join("finite.json"); + let reward_cert = dir.join("reward.json"); + let observation_cert = dir.join("duplicate_state_observation.json"); + let finite_cert_crc32 = write_common_certificate( + &finite_cert, + "finite_planner_state", + &dataset_path, + TimingCertificationTier::BestEffort, + "mc_aixi_fac_ctw", + ); + write_exact_reward_certificate( + &reward_cert, + &dataset_path, + TimingCertificationTier::BestEffort, + "mc_aixi_fac_ctw", + 65_535, + ); + let mut bad_observation = common_certificate( + "exact_state_observation", + &dataset_path, + TimingCertificationTier::BestEffort, + "mc_aixi_fac_ctw", + ); + let object = bad_observation + .as_object_mut() + .expect("observation certificate object"); + object.insert("observation_key_mode".to_string(), json!("full_stream")); + object.insert( + "exact_state_encoder_spec_ref".to_string(), + json!("encoder://full-state"), + ); + object.insert( + "observation_adapter_spec_ref".to_string(), + json!("single-channel-conditional-byte-adapter-v1"), + ); + object.insert( + "observation_adapter_content_crc32".to_string(), + json!(observation_adapter_crc32()), + ); + object.insert( + "finite_planner_state_certificate_crc32".to_string(), + json!(finite_cert_crc32), + ); + object.insert( + "psi_h_outputs".to_string(), + json!([ + {"state_id": "duplicate", "observations": [7]}, + {"state_id": "duplicate", "observations": [8]} + ]), + ); + write_json(&observation_cert, &bad_observation); + + let args = [ + "infotheory".to_string(), + "tune".to_string(), + path_string(&spec_path), + "--max-evaluations".to_string(), + "1".to_string(), + "--scalar-representation-ref".to_string(), + "scalar://finite-f64".to_string(), + "--finite-planner-state-certificate".to_string(), + path_string(&finite_cert), + "--exact-reward-encoding-certificate".to_string(), + path_string(&reward_cert), + "--exact-state-encoder-spec-ref".to_string(), + "encoder://full-state".to_string(), + "--exact-state-observation-certificate".to_string(), + path_string(&observation_cert), + "--claim-exact-observed-markov".to_string(), + ]; + let request = parse_tune_command_args(&args).expect("parse tune args"); + let err = run_tune(&request).expect_err("duplicate observation state ids must fail"); + assert!(err.contains("duplicate state_id"), "{err}"); + let _ = fs::remove_dir_all(dir); +} + +#[cfg(unix)] +#[test] +fn run_tune_reports_per_candidate_timeout() { + let dir = temp_dir("candidate_timeout"); + let dataset_path = dir.join("dataset.bin"); + let spec_path = dir.join("spec.json"); + let output_path = dir.join("output.json"); + let report_path = dir.join("report.json"); + write_passive_dataset(&dataset_path); + let mut spec = tune_spec( + &dataset_path, + &output_path, + &report_path, + json!({ + "kind": "annealed_hill_climbing", + "max_mutation_radius": 1, + }), + None, + ); + spec["eval_time_limit_seconds"] = json!(0.000000001f64); + write_json(&spec_path, &spec); + let args = [ + "infotheory", + "tune", + path_string(&spec_path).as_str(), + "--max-evaluations", + "1", + ] + .iter() + .map(|item| (*item).to_string()) + .collect::>(); + let request = parse_tune_command_args(&args).expect("parse tune args"); + let err = run_tune(&request).expect_err("timeout is not deployable"); + assert!(err.contains("timed out"), "{err}"); + let report = read_json(&report_path); + assert_eq!(str_at(&report, "/baseline/status"), "timeout"); + assert_eq!(str_at(&report, "/best/status"), "timeout"); + assert!(!bool_at(&report, "/baseline/deployable")); + assert_eq!( + u64_at(&report, "/search/candidate_result_counts/timeout"), + 1 + ); + assert_eq!( + u64_at( + &report, + "/search/candidate_result_counts/success_non_deployable" + ), + 0 + ); + assert!(!output_path.exists()); + let _ = fs::remove_dir_all(dir); +} + +#[cfg(unix)] +#[test] +fn run_tune_treats_worker_ok_false_as_unrecoverable_evaluator_failure() { + let dir = temp_dir("worker_ok_false_fatal"); + let dataset_path = dir.join("dataset.bin"); + let spec_path = dir.join("spec.json"); + let output_path = dir.join("output.json"); + let report_path = dir.join("report.json"); + let worker_path = dir.join("synthetic-worker.sh"); + write_passive_dataset(&dataset_path); + write_json( + &spec_path, + &tune_spec( + &dataset_path, + &output_path, + &report_path, + json!({ + "kind": "annealed_hill_climbing", + "max_mutation_radius": 1, + }), + None, + ), + ); + fs::write( + &worker_path, + br#"#!/bin/sh +if [ "${INFOTHEORY_TUNER_EVAL_WORKER_PING:-0}" = "1" ]; then + exit 0 +fi +printf '%s\n' '{"ok":false,"error":"synthetic worker setup failure"}' > "$INFOTHEORY_TUNER_EVAL_RESPONSE_PATH" +exit 0 +"#, + ) + .expect("write synthetic worker script"); + let mut permissions = fs::metadata(&worker_path) + .expect("read worker metadata") + .permissions(); + permissions.set_mode(0o755); + fs::set_permissions(&worker_path, permissions).expect("set worker script executable"); + + let args = [ + "infotheory".to_string(), + "tune".to_string(), + path_string(&spec_path), + "--max-evaluations".to_string(), + "1".to_string(), + "--evaluator-worker-executable".to_string(), + path_string(&worker_path), + ]; + let request = parse_tune_command_args(&args).expect("parse tune args"); + let err = run_tune(&request).expect_err("worker ok:false must be fatal"); + assert!( + err.contains("unrecoverable evaluator failure during baseline evaluation"), + "{err}" + ); + assert!(err.contains("synthetic worker setup failure"), "{err}"); + if report_path.exists() { + let report = read_json(&report_path); + assert_ne!( + str_at(&report, "/search/termination_reason"), + "baseline_not_deployable" + ); + } + let _ = fs::remove_dir_all(dir); +} + +#[cfg(unix)] +#[test] +fn baseline_candidate_local_error_reports_baseline_not_deployable() { + let dir = temp_dir("baseline_candidate_local_error"); + let dataset_path = dir.join("dataset.bin"); + let spec_path = dir.join("spec.json"); + let output_path = dir.join("output.json"); + let report_path = dir.join("report.json"); + let worker_path = dir.join("synthetic-worker.sh"); + write_passive_dataset(&dataset_path); + write_json( + &spec_path, + &tune_spec( + &dataset_path, + &output_path, + &report_path, + json!({ + "kind": "annealed_hill_climbing", + "max_mutation_radius": 1, + }), + None, + ), + ); + fs::write( + &worker_path, + br#"#!/bin/sh +if [ "${INFOTHEORY_TUNER_EVAL_WORKER_PING:-0}" = "1" ]; then + exit 0 +fi +printf '%s\n' '{"ok":true,"status":"error","compressed_bytes":0,"elapsed_seconds":0.0,"effective_eval_time_limit_seconds":1.0,"throughput_bytes_per_second":0.0,"peak_memory_bytes":0,"target_loss_bits":null,"objective_bits":null,"deployable":false}' > "$INFOTHEORY_TUNER_EVAL_RESPONSE_PATH" +exit 0 +"#, + ) + .expect("write synthetic worker script"); + let mut permissions = fs::metadata(&worker_path) + .expect("read worker metadata") + .permissions(); + permissions.set_mode(0o755); + fs::set_permissions(&worker_path, permissions).expect("set worker script executable"); + + let args = [ + "infotheory".to_string(), + "tune".to_string(), + path_string(&spec_path), + "--max-evaluations".to_string(), + "1".to_string(), + "--evaluator-worker-executable".to_string(), + path_string(&worker_path), + ]; + let request = parse_tune_command_args(&args).expect("parse tune args"); + let err = run_tune(&request).expect_err("baseline error status must be non-deployable"); + assert!( + err.contains("baseline candidate is not deployable"), + "unexpected error: {err}" + ); + let report = read_json(&report_path); + assert_eq!( + str_at(&report, "/search/termination_reason"), + "baseline_not_deployable" + ); + assert_eq!(str_at(&report, "/baseline/status"), "error"); + assert_eq!(u64_at(&report, "/search/fatal_evaluator_failures"), 0); + assert_eq!( + u64_at(&report, "/search/candidate_result_counts/error_recoverable"), + 1 + ); + let _ = fs::remove_dir_all(dir); +} + +#[cfg(unix)] +#[test] +fn baseline_fatal_inner_eval_error_reports_fatal_evaluator_failure() { + let dir = temp_dir("baseline_fatal_inner_eval_error"); + let dataset_path = dir.join("dataset.bin"); + let spec_path = dir.join("spec.json"); + let output_path = dir.join("output.json"); + let report_path = dir.join("report.json"); + let worker_path = dir.join("synthetic-worker.sh"); + write_passive_dataset(&dataset_path); + write_json( + &spec_path, + &tune_spec( + &dataset_path, + &output_path, + &report_path, + json!({ + "kind": "annealed_hill_climbing", + "max_mutation_radius": 1, + }), + None, + ), + ); + fs::write( + &worker_path, + br#"#!/bin/sh +if [ "${INFOTHEORY_TUNER_EVAL_WORKER_PING:-0}" = "1" ]; then + exit 0 +fi +printf '%s\n' '{"ok":false,"error":"synthetic inner fatal evaluator failure"}' > "$INFOTHEORY_TUNER_EVAL_RESPONSE_PATH" +exit 0 +"#, + ) + .expect("write synthetic worker script"); + let mut permissions = fs::metadata(&worker_path) + .expect("read worker metadata") + .permissions(); + permissions.set_mode(0o755); + fs::set_permissions(&worker_path, permissions).expect("set worker script executable"); + + let args = [ + "infotheory".to_string(), + "tune".to_string(), + path_string(&spec_path), + "--max-evaluations".to_string(), + "1".to_string(), + "--evaluator-worker-executable".to_string(), + path_string(&worker_path), + ]; + let request = parse_tune_command_args(&args).expect("parse tune args"); + let err = run_tune(&request).expect_err("worker fatal must abort baseline"); + assert!( + err.contains("unrecoverable evaluator failure during baseline evaluation"), + "unexpected error: {err}" + ); + assert!( + err.contains("synthetic inner fatal evaluator failure"), + "{err}" + ); + assert!( + !report_path.exists(), + "fatal baseline evaluator failures should abort before report synthesis" + ); + let _ = fs::remove_dir_all(dir); +} + +#[test] +fn run_tune_terminates_on_unrecoverable_evaluator_failure() { + let dir = temp_dir("fatal_evaluator_failure"); + let dataset_path = dir.join("dataset.bin"); + let spec_path = dir.join("spec.json"); + let output_path = dir.join("output.json"); + let report_path = dir.join("report.json"); + let table_cert_path = dir.join("deterministic_table.json"); + let reward_cert_path = dir.join("exact_reward.json"); + write_passive_dataset(&dataset_path); + let mc_case = controller_cases() + .into_iter() + .find(|case| case.kind == "mc_aixi_fac_ctw") + .expect("mc_aixi controller case"); + let spec = tune_spec( + &dataset_path, + &output_path, + &report_path, + mc_case.controller, + None, + ); + write_json(&spec_path, &spec); + let (_spec_crc32, baseline_candidate_crc32) = compiled_tune_hashes(&spec, &dir); + write_exact_reward_certificate_with_runtime_profile( + &reward_cert_path, + &dataset_path, + TimingCertificationTier::DeterministicTable, + "mc_aixi_fac_ctw", + 65_535, + true, + ); + write_deterministic_table( + &table_cert_path, + &dataset_path, + "mc_aixi_fac_ctw", + &baseline_candidate_crc32, + ); + + let args = [ + "infotheory", + "tune", + path_string(&spec_path).as_str(), + "--max-evaluations", + "2", + "--timing-tier", + "deterministic_table", + "--scalar-representation-ref", + "scalar://finite-f64", + "--exact-reward-encoding-certificate", + path_string(&reward_cert_path).as_str(), + "--deterministic-evaluator-table", + path_string(&table_cert_path).as_str(), + ] + .iter() + .map(|item| (*item).to_string()) + .collect::>(); + let request = parse_tune_command_args(&args).expect("parse tune args"); + let err = run_tune(&request).expect_err("missing deterministic-table row must be fatal"); + assert!( + err.contains("unrecoverable evaluator failure"), + "unexpected error: {err}" + ); + + let report = read_json(&report_path); + assert_eq!( + str_at(&report, "/status"), + "terminated_unrecoverable_evaluator_failure" + ); + assert_eq!( + str_at(&report, "/search/termination_reason"), + "terminated_unrecoverable_evaluator_failure" + ); + assert_eq!(u64_at(&report, "/search/fatal_evaluator_failures"), 1); + assert!( + str_at(&report, "/search/fatal_evaluator_failure") + .contains("deterministic evaluator table missing row"), + "fatal diagnostic should retain deterministic table row failure context" + ); + let _ = fs::remove_dir_all(dir); +} + +#[cfg(unix)] +#[test] +fn run_tune_reports_compiled_uniform_mh_kernel() { + let dir = temp_dir("compiled_uniform_mh"); + let dataset_path = dir.join("dataset.bin"); + let spec_path = dir.join("spec.json"); + let output_path = dir.join("output.json"); + let report_path = dir.join("report.json"); + write_passive_dataset(&dataset_path); + write_json( + &spec_path, + &tune_spec( + &dataset_path, + &output_path, + &report_path, + json!({ + "kind": "annealed_hill_climbing", + "max_mutation_radius": 1, + }), + None, + ), + ); + let args = [ + "infotheory", + "tune", + path_string(&spec_path).as_str(), + "--max-evaluations", + "3", + "--annealer-kernel-profile", + "compiled_uniform_metropolis_hastings", + ] + .iter() + .map(|item| (*item).to_string()) + .collect::>(); + let request = parse_tune_command_args(&args).expect("parse tune args"); + run_tune(&request).expect("run tune"); + let report = read_json(&report_path); + assert_eq!( + str_at(&report, "/search/controller/runtime_path"), + "compiled_uniform_metropolis_hastings" + ); + assert_eq!( + str_at(&report, "/search/controller/proposal_action_distribution"), + "uniform_finite_bounded_numeric_elementary_descriptors" + ); + assert!(bool_at( + &report, + "/search/controller/proposal_mass_accounting" + )); + let _ = fs::remove_dir_all(dir); +} + +#[cfg(unix)] +#[test] +fn run_tune_warmstart_self_improvement_reports_equal_split_deadlines() { + let dir = temp_dir("warmstart_self_improvement_deadlines"); + let dataset_path = dir.join("dataset.bin"); + let teacher_path = dir.join("teacher.json"); + let spec_path = dir.join("spec.json"); + let output_path = dir.join("output.json"); + let report_path = dir.join("report.json"); + write_passive_dataset(&dataset_path); + write_placeholder_teacher(&teacher_path); + let warmstart = controller_cases() + .into_iter() + .find(|case| case.kind == "aiqi_warmstart_exact_jh") + .expect("warmstart case"); + let spec_value = tune_spec( + &dataset_path, + &output_path, + &report_path, + warmstart.controller, + Some(&teacher_path), + ); + write_json(&spec_path, &spec_value); + let (_spec_crc32, _) = compiled_tune_hashes(&spec_value, &dir); + let reward_cert_path = dir.join("exact_reward.json"); + let reward_cert_crc32 = write_exact_reward_certificate( + &reward_cert_path, + &dataset_path, + TimingCertificationTier::BestEffort, + "aiqi_warmstart_exact_jh", + 65_535, + ); + let args = [ + "infotheory", + "tune", + path_string(&spec_path).as_str(), + "--max-evaluations", + "2", + "--self-improvement-rounds", + "3", + "--scalar-representation-ref", + "scalar://finite-f64", + "--exact-reward-encoding-certificate", + path_string(&reward_cert_path).as_str(), + ] + .iter() + .map(|item| (*item).to_string()) + .collect::>(); + let task_fingerprint = + derive_runtime_warmstart_task_fingerprint(&args, &teacher_path, &reward_cert_crc32); + write_teacher(&teacher_path, task_fingerprint, &reward_cert_crc32); + let request = parse_tune_command_args(&args).expect("parse tune args"); + run_tune(&request).expect("run tune"); + let report = read_json(&report_path); + assert!(!bool_at( + &report, + "/provenance/self_improvement_policy/same_task_trace_refresh_enabled" + )); + assert!(bool_at( + &report, + "/provenance/self_improvement_policy/online_delayed_label_update_enabled" + )); + assert_eq!( + u64_at(&report, "/provenance/self_improvement_policy/rounds"), + 3 + ); + let deadlines = + report["provenance"]["self_improvement_policy"]["deterministic_round_deadlines_seconds"] + .as_array() + .expect("deterministic deadline array"); + assert_eq!(deadlines.len(), 3); + let expected = [5.0f64 / 3.0f64, 10.0f64 / 3.0f64, 5.0f64]; + for (index, expected_value) in expected.iter().enumerate() { + let observed = deadlines[index] + .as_f64() + .unwrap_or_else(|| panic!("deadline[{index}] must be f64")); + assert!((observed - expected_value).abs() <= 1.0e-9); + } + let realized = + report["provenance"]["self_improvement_policy"]["realized_trace_counts_by_round"] + .as_array() + .expect("realized trace counts"); + assert_eq!(realized.len(), 3); + let merges = report["provenance"]["self_improvement_policy"]["trace_refresh_merges_by_round"] + .as_array() + .expect("trace refresh merges"); + assert_eq!(merges.len(), 3); + assert!(merges.iter().all(|value| value.as_u64() == Some(0))); + let _ = fs::remove_dir_all(dir); +} + +#[cfg(unix)] +#[test] +fn warmstart_trace_refresh_merges_same_task_live_trace() { + let dir = temp_dir("warmstart_trace_refresh"); + let dataset_path = dir.join("dataset.bin"); + let teacher_path = dir.join("teacher.json"); + let spec_path = dir.join("spec.json"); + let output_path = dir.join("output.json"); + let report_path = dir.join("report.json"); + write_passive_dataset(&dataset_path); + write_placeholder_teacher(&teacher_path); + let warmstart = controller_cases() + .into_iter() + .find(|case| case.kind == "aiqi_warmstart_exact_jh") + .expect("warmstart case"); + let spec_value = tune_spec( + &dataset_path, + &output_path, + &report_path, + warmstart.controller, + Some(&teacher_path), + ); + write_json(&spec_path, &spec_value); + let (_spec_crc32, _) = compiled_tune_hashes(&spec_value, &dir); + let reward_cert_path = dir.join("exact_reward.json"); + let reward_cert_crc32 = write_exact_reward_certificate( + &reward_cert_path, + &dataset_path, + TimingCertificationTier::BestEffort, + "aiqi_warmstart_exact_jh", + 65_535, + ); + write_teacher(&teacher_path, probe_task_fingerprint(), "00000000"); + let args = [ + "infotheory", + "tune", + path_string(&spec_path).as_str(), + "--max-evaluations", + "3", + "--self-improvement-rounds", + "3", + "--warmstart-trace-refresh", + "--scalar-representation-ref", + "scalar://finite-f64", + "--exact-reward-encoding-certificate", + path_string(&reward_cert_path).as_str(), + ] + .iter() + .map(|item| (*item).to_string()) + .collect::>(); + let task_fingerprint = + derive_runtime_warmstart_task_fingerprint(&args, &teacher_path, &reward_cert_crc32); + write_teacher(&teacher_path, task_fingerprint, &reward_cert_crc32); + let request = parse_tune_command_args(&args).expect("parse tune args"); + run_tune(&request).expect("run tune"); + let report = read_json(&report_path); + assert!(bool_at( + &report, + "/provenance/self_improvement_policy/same_task_trace_refresh_enabled" + )); + assert!(!bool_at( + &report, + "/provenance/self_improvement_policy/online_delayed_label_update_enabled" + )); + assert_eq!( + report["search"]["controller"]["warmstart_self_improvement_update"].as_str(), + Some("same_task_trace_refresh_rebuild") + ); + let realized = + report["provenance"]["self_improvement_policy"]["realized_trace_counts_by_round"] + .as_array() + .expect("realized trace counts"); + let merges = report["provenance"]["self_improvement_policy"]["trace_refresh_merges_by_round"] + .as_array() + .expect("trace refresh merges"); + assert_eq!(realized.len(), 3); + assert_eq!(merges.len(), 3); + let merges_by_round = merges + .iter() + .enumerate() + .map(|(index, value)| { + value + .as_u64() + .unwrap_or_else(|| panic!("trace_refresh_merges_by_round[{index}] must be u64")) + }) + .collect::>(); + let rounds_with_insertions = merges_by_round.iter().filter(|count| **count > 0).count() as u64; + assert_eq!( + u64_at(&report, "/search/controller/warmstart_trace_refresh_merges"), + rounds_with_insertions + ); + assert_eq!(rounds_with_insertions, 1); + let _ = fs::remove_dir_all(dir); +} + +#[cfg(unix)] +#[test] +fn warmstart_teacher_fingerprint_mismatch_is_rejected() { + let dir = temp_dir("warmstart_teacher_mismatch"); + let dataset_path = dir.join("dataset.bin"); + let teacher_path = dir.join("teacher.json"); + let spec_path = dir.join("spec.json"); + let output_path = dir.join("output.json"); + let report_path = dir.join("report.json"); + write_passive_dataset(&dataset_path); + write_placeholder_teacher(&teacher_path); + let warmstart = controller_cases() + .into_iter() + .find(|case| case.kind == "aiqi_warmstart_exact_jh") + .expect("warmstart case"); + let spec_value = tune_spec( + &dataset_path, + &output_path, + &report_path, + warmstart.controller, + Some(&teacher_path), + ); + write_json(&spec_path, &spec_value); + let reward_cert_path = dir.join("exact_reward.json"); + let reward_cert_crc32 = write_exact_reward_certificate( + &reward_cert_path, + &dataset_path, + TimingCertificationTier::BestEffort, + "aiqi_warmstart_exact_jh", + 65_535, + ); + write_teacher( + &teacher_path, + TaskFingerprint::parse_hex(MISMATCH_TASK_FINGERPRINT_HEX) + .expect("mismatched task fingerprint"), + &reward_cert_crc32, + ); + let args = [ + "infotheory".to_string(), + "tune".to_string(), + path_string(&spec_path), + "--max-evaluations".to_string(), + "1".to_string(), + "--scalar-representation-ref".to_string(), + "scalar://finite-f64".to_string(), + "--exact-reward-encoding-certificate".to_string(), + path_string(&reward_cert_path), + ]; + let request = parse_tune_command_args(&args).expect("parse tune args"); + let err = run_tune(&request).expect_err("mismatched teacher must be rejected"); + assert!(err.contains("task_fingerprint"), "{err}"); + let _ = fs::remove_dir_all(dir); +} + +#[cfg(unix)] +#[test] +fn warmstart_teacher_observation_adapter_mismatch_is_rejected() { + let dir = temp_dir("warmstart_teacher_observation_mismatch"); + let dataset_path = dir.join("dataset.bin"); + let teacher_path = dir.join("teacher.json"); + let spec_path = dir.join("spec.json"); + let output_path = dir.join("output.json"); + let report_path = dir.join("report.json"); + write_passive_dataset(&dataset_path); + write_placeholder_teacher(&teacher_path); + let warmstart = controller_cases() + .into_iter() + .find(|case| case.kind == "aiqi_warmstart_exact_jh") + .expect("warmstart case"); + let spec_value = tune_spec( + &dataset_path, + &output_path, + &report_path, + warmstart.controller, + Some(&teacher_path), + ); + write_json(&spec_path, &spec_value); + let (_spec_crc32, _) = compiled_tune_hashes(&spec_value, &dir); + let reward_cert_path = dir.join("exact_reward.json"); + let reward_cert_crc32 = write_exact_reward_certificate( + &reward_cert_path, + &dataset_path, + TimingCertificationTier::BestEffort, + "aiqi_warmstart_exact_jh", + 65_535, + ); + write_teacher(&teacher_path, probe_task_fingerprint(), "00000000"); + let args = [ + "infotheory".to_string(), + "tune".to_string(), + path_string(&spec_path), + "--max-evaluations".to_string(), + "1".to_string(), + "--scalar-representation-ref".to_string(), + "scalar://finite-f64".to_string(), + "--exact-reward-encoding-certificate".to_string(), + path_string(&reward_cert_path), + ]; + let task_fingerprint = + derive_runtime_warmstart_task_fingerprint(&args, &teacher_path, &reward_cert_crc32); + write_teacher(&teacher_path, task_fingerprint, &reward_cert_crc32); + let mut teacher = read_json(&teacher_path); + teacher["contract"]["observation_adapter_content_crc32"] = json!("00000000"); + write_json(&teacher_path, &teacher); + let request = parse_tune_command_args(&args).expect("parse tune args"); + let err = run_tune(&request).expect_err("observation adapter mismatch must fail"); + assert!(err.contains("observation adapter fingerprint"), "{err}"); + let _ = fs::remove_dir_all(dir); +} + +#[test] +fn warmstart_exact_jh_rejects_nonidentity_finite_reward_map() { + let dir = temp_dir("warmstart_nonidentity_reward_map"); + let dataset_path = dir.join("dataset.bin"); + let teacher_path = dir.join("teacher.json"); + let spec_path = dir.join("spec.json"); + let output_path = dir.join("output.json"); + let report_path = dir.join("report.json"); + write_passive_dataset(&dataset_path); + write_placeholder_teacher(&teacher_path); + let warmstart = controller_cases() + .into_iter() + .find(|case| case.kind == "aiqi_warmstart_exact_jh") + .expect("warmstart case"); + let spec_value = tune_spec( + &dataset_path, + &output_path, + &report_path, + warmstart.controller, + Some(&teacher_path), + ); + write_json(&spec_path, &spec_value); + let (_spec_crc32, baseline_candidate_crc32) = compiled_tune_hashes(&spec_value, &dir); + let document = SpecDocument::parse_json_value(&spec_value, &dir).expect("parse tune spec"); + let SpecDocument::Tune(tune) = document else { + panic!("expected tune spec"); + }; + let compiled = tune + .compile_in(&SpecEnvironment::new(&dir)) + .expect("compile tune spec"); + let baseline_objective = ((compiled.baseline_candidate_model_bytes() as u64) * 8) + 128; + let reward_values = (0..=baseline_objective) + .map(|objective_difference: u64| { + let symbol = match objective_difference { + 1 => 2, + 2 => 1, + other => other, + }; + json!({ + "objective_difference": objective_difference, + "symbol": symbol + }) + }) + .collect::>(); + let reward_cert_path = dir.join("reward_map.json"); + let reward_cert_crc32 = write_finite_reward_map_certificate_with_runtime_profile( + &reward_cert_path, + &dataset_path, + TimingCertificationTier::DeterministicTable, + "aiqi_warmstart_exact_jh", + 65_535, + Some(baseline_objective), + Value::Array(reward_values), + true, + ); + write_teacher( + &teacher_path, + TaskFingerprint::parse_hex(PLACEHOLDER_TASK_FINGERPRINT_HEX) + .expect("placeholder task fingerprint"), + &reward_cert_crc32, + ); + let table_path = dir.join("table.json"); + write_deterministic_table( + &table_path, + &dataset_path, + "aiqi_warmstart_exact_jh", + &baseline_candidate_crc32, + ); + let args = [ + "infotheory".to_string(), + "tune".to_string(), + path_string(&spec_path), + "--max-evaluations".to_string(), + "1".to_string(), + "--timing-tier".to_string(), + "deterministic_table".to_string(), + "--scalar-representation-ref".to_string(), + "scalar://finite-f64".to_string(), + "--exact-reward-encoding-certificate".to_string(), + path_string(&reward_cert_path), + "--deterministic-evaluator-table".to_string(), + path_string(&table_path), + ]; + let request = parse_tune_command_args(&args).expect("parse tune args"); + let err = run_tune(&request).expect_err("nonidentity finite map must be rejected"); + assert!(err.contains("non-identity finite_reward_map"), "{err}"); + let _ = fs::remove_dir_all(dir); +} + +#[cfg(unix)] +#[test] +fn run_tune_planner_controller_timing_matrix_reports_dispatch_and_claim_gating() { + let dir = temp_dir("controller_timing_matrix"); + let dataset_path = dir.join("dataset.bin"); + let teacher_path = dir.join("teacher.json"); + write_passive_dataset(&dataset_path); + write_placeholder_teacher(&teacher_path); + let timings = [ + TimingCertificationTier::BestEffort, + TimingCertificationTier::Isolated, + TimingCertificationTier::RealTime, + TimingCertificationTier::DeterministicTable, + ]; + for case in controller_cases() { + for timing in timings { + let report = run_tune_case( + &dir, + "passive", + &dataset_path, + &teacher_path, + &case, + timing, + 2, + ); + assert_required_report_fields(&report); + assert_eq!(str_at(&report, "/status"), case.status); + assert_eq!(str_at(&report, "/search/controller/kind"), case.kind); + assert_eq!( + str_at(&report, "/search/controller/runtime_path"), + case.runtime_path + ); + assert_eq!( + str_at(&report, "/search/controller/agent_runtime"), + case.agent_runtime + ); + assert_eq!( + str_at(&report, "/search/controller/planner_run_controller_kind"), + case.planner_run_controller_kind + ); + assert_eq!( + str_at(&report, "/search/controller/reward_semantics"), + case.reward_semantics + ); + assert_eq!( + u64_at(&report, "/search/controller/compiled_action_count"), + 2 + ); + assert_eq!( + u64_at(&report, "/search/controller/declared_agent_actions"), + 2 + ); + assert_eq!( + str_at(&report, "/input_asset/dataset_kind"), + "passive_bytes" + ); + assert_eq!( + str_at(&report, "/evaluator_profile/objective_target"), + "passive_ac" + ); + assert_eq!( + str_at(&report, "/theorem_claims/exact_finite_mdp/status"), + "uncertified" + ); + assert_eq!( + str_at(&report, "/theorem_claims/exact_observed_markov/status"), + "uncertified" + ); + assert_eq!( + str_at(&report, "/theorem_claims/planner_convergence/status"), + "uncertified" + ); + assert!( + report["theorem_claims"]["exact_finite_mdp"]["missing_prerequisites"] + .as_array() + .expect("missing prereqs") + .iter() + .any(|item| item.as_str() == Some("verified_finite_planner_state_certificate")) + ); + assert!( + report["theorem_claims"]["exact_finite_mdp"]["missing_prerequisites"] + .as_array() + .expect("missing prereqs") + .iter() + .any(|item| { + item.as_str() == Some("verified_no_hidden_state_or_inert_state_certificate") + }) + ); + if case.kind == "aiqi_discounted" { + assert!( + report["theorem_claims"]["exact_finite_mdp"]["missing_prerequisites"] + .as_array() + .expect("missing prereqs") + .iter() + .any(|item| item.as_str() + == Some("verified_exact_reward_encoding_certificate")) + ); + assert!( + report["theorem_claims"]["exact_finite_mdp"]["missing_prerequisites"] + .as_array() + .expect("missing prereqs") + .iter() + .any(|item| { + item.as_str() == Some("exact_objective_difference_controller") + }) + ); + } + if !timing_certifies(timing) { + assert!( + report["theorem_claims"]["exact_finite_mdp"]["missing_prerequisites"] + .as_array() + .expect("missing prereqs") + .iter() + .any(|item| { + item.as_str() == Some("theorem_certified_timing_or_deterministic_table") + }) + ); + } + if case.kind != "mc_aixi_fac_ctw" { + assert!( + report["theorem_claims"]["planner_convergence"]["missing_prerequisites"] + .as_array() + .expect("missing prereqs") + .iter() + .any(|item| item.as_str() == Some("mc_aixi_fac_ctw_controller")) + ); + } + if case.needs_teacher { + assert!( + report["search"]["controller"]["warmstart_teacher_dataset"]["content_crc32"] + .as_str() + .is_some() + ); + } else { + assert!(report["search"]["controller"]["warmstart_teacher_dataset"].is_null()); + } + } + } + let _ = fs::remove_dir_all(dir); +} + +#[cfg(unix)] +#[test] +fn run_tune_causal_dataset_modes_report_lowering_under_planner_execution() { + let dir = temp_dir("causal_dataset_modes"); + let teacher_path = dir.join("teacher.json"); + write_placeholder_teacher(&teacher_path); + let datasets = [ + ( + "interactive", + canonical_causal_dataset( + "test-interactive-codec", + "events", + json!([ + {"kind": "context", "channel": "action", "bytes": [1]}, + {"kind": "observe_target_no_score", "channel": "percept", "domain": "bytes", "bytes": [2]}, + {"kind": "target", "channel": "percept", "domain": "bytes", "bytes": [3, 4]} + ]), + ), + "interactive_trace", + "interactive-trace-events-v1", + "interactive-trace-target-bytes", + 2u64, + 2u64, + 2.0f64, + ), + ( + "prefix", + canonical_causal_dataset( + "test-prefix-codec", + "examples", + json!([{ + "history": [{"kind": "observe_target_no_score", "channel": "percept", "domain": "bytes", "bytes": [7]}], + "action": [1], + "channel": "percept", + "domain": "bytes", + "target": [8], + "weight": 2.0 + }]), + ), + "causal_prefix_dataset", + "causal-prefix-examples-v1", + "weighted-target-bytes-sum", + 1u64, + 1u64, + 2.0f64, + ), + ]; + let case = controller_cases() + .into_iter() + .find(|case| case.kind == "aiqi_discounted") + .expect("aiqi-discounted case"); + for ( + label, + dataset, + expected_kind, + expected_lowering, + expected_size_function, + expected_charged_bytes, + expected_target_events, + expected_dataset_units, + ) in datasets + { + let dataset_path = dir.join(format!("{label}.json")); + write_json(&dataset_path, &dataset); + let report = run_tune_case( + &dir, + label, + &dataset_path, + &teacher_path, + &case, + TimingCertificationTier::DeterministicTable, + 2, + ); + assert_required_report_fields(&report); + assert_eq!(str_at(&report, "/input_asset/dataset_kind"), expected_kind); + assert_eq!( + str_at(&report, "/input_asset/target_size_function"), + expected_size_function + ); + assert_eq!( + str_at(&report, "/evaluator_profile/dataset_lowering_version"), + expected_lowering + ); + assert_eq!( + str_at(&report, "/evaluator_profile/objective_target"), + "interactive_causal_ac" + ); + assert_eq!( + u64_at(&report, "/input_asset/charged_target_bytes"), + expected_charged_bytes + ); + assert_eq!( + u64_at(&report, "/input_asset/target_events"), + expected_target_events + ); + assert_eq!( + f64_at(&report, "/input_asset/dataset_units"), + expected_dataset_units + ); + } + let _ = fs::remove_dir_all(dir); +} + +#[cfg(unix)] +#[test] +fn planner_deployable_model_flag_reports_objective_target_and_diagnostics() { + let dir = temp_dir("planner_deployable_report"); + let dataset_path = dir.join("dataset.bin"); + let spec_path = dir.join("spec.json"); + let output_path = dir.join("output.json"); + let report_path = dir.join("report.json"); + write_passive_dataset(&dataset_path); + let case = controller_cases() + .into_iter() + .find(|case| case.kind == "aiqi_discounted") + .expect("aiqi discounted case"); + let spec_value = tune_spec( + &dataset_path, + &output_path, + &report_path, + case.controller, + None, + ); + write_json(&spec_path, &spec_value); + let args = [ + "infotheory".to_string(), + "tune".to_string(), + path_string(&spec_path), + "--max-evaluations".to_string(), + "1".to_string(), + "--planner-deployable-model".to_string(), + ]; + let request = parse_tune_command_args(&args).expect("parse tune args"); + run_tune(&request).expect("run tune"); + let report = read_json(&report_path); + assert_eq!( + str_at(&report, "/evaluator_profile/objective_target"), + "planner_deployable_model" + ); + assert!(bool_at(&report, "/search/planner_deployability/enabled")); + assert!(bool_at( + &report, + "/search/planner_deployability/deployable_under_executor_limits" + )); + assert!(u64_at(&report, "/search/planner_deployability/model_state_bytes") > 0); + let _ = fs::remove_dir_all(dir); +} + +#[cfg(all(feature = "cli", unix))] +#[test] +fn tune_cli_accepts_executor_flags_and_writes_report() { + let dir = temp_dir("cli_smoke"); + let dataset_path = dir.join("dataset.bin"); + let spec_path = dir.join("spec.json"); + let output_path = dir.join("output.json"); + let report_path = dir.join("report.json"); + write_passive_dataset(&dataset_path); + let case = controller_cases() + .into_iter() + .find(|case| case.kind == "mc_aixi_fac_ctw") + .expect("mc-aixi case"); + let spec_value = tune_spec( + &dataset_path, + &output_path, + &report_path, + case.controller, + None, + ); + write_json(&spec_path, &spec_value); + let reward_cert_path = dir.join("reward.json"); + write_exact_reward_certificate_with_runtime_profile_and_worker( + &reward_cert_path, + &dataset_path, + TimingCertificationTier::DeterministicTable, + "mc_aixi_fac_ctw", + 65_535, + false, + Some(Path::new(env!("CARGO_BIN_EXE_infotheory"))), + ); + let spec_arg = path_string(&spec_path); + let reward_cert_arg = path_string(&reward_cert_path); + let output = Command::new(env!("CARGO_BIN_EXE_infotheory")) + .args([ + "tune", + spec_arg.as_str(), + "--max-evaluations", + "1", + "--timing-tier", + "deterministic_table", + "--scalar-representation-ref", + "scalar://finite-f64", + "--exact-reward-encoding-certificate", + reward_cert_arg.as_str(), + "--claim-exact-finite-mdp", + ]) + .output() + .expect("run tune cli"); + assert!( + output.status.success(), + "stderr={}", + String::from_utf8_lossy(&output.stderr) + ); + let report = read_json(&report_path); + assert_eq!(u64_at(&report, "/execution_profile/max_evaluations"), 1); + assert_eq!( + str_at(&report, "/provenance/executor_controls/rss_mode/requested"), + "process_rss_peak" + ); + let effective_measurement = str_at( + &report, + "/provenance/executor_controls/rss_mode/effective_measurement", + ); + assert_eq!(effective_measurement, "unix_process_rss_fallback_explicit"); + assert_eq!( + str_at(&report, "/theorem_claims/exact_finite_mdp/status"), + "uncertified" + ); + assert_eq!( + str_at(&report, "/evaluator_execution_model"), + "spawn_exec_worker_process_isolated_operational" + ); + assert_eq!( + str_at(&report, "/theorem_timing_basis"), + "operational_only_uncertified" + ); + assert!(output_path.exists()); + let _ = fs::remove_dir_all(dir); +} + +#[cfg(all(feature = "cli", not(unix)))] +#[test] +fn tune_cli_reports_unix_only_runtime_contract_on_non_unix() { + let dir = temp_dir("cli_non_unix_runtime_contract"); + let dataset_path = dir.join("dataset.bin"); + let spec_path = dir.join("spec.json"); + let output_path = dir.join("output.json"); + let report_path = dir.join("report.json"); + write_passive_dataset(&dataset_path); + let case = controller_cases() + .into_iter() + .find(|case| case.kind == "mc_aixi_fac_ctw") + .expect("mc-aixi case"); + let spec_value = tune_spec( + &dataset_path, + &output_path, + &report_path, + case.controller, + None, + ); + write_json(&spec_path, &spec_value); + let reward_cert_path = dir.join("reward.json"); + write_exact_reward_certificate_with_runtime_profile_and_worker( + &reward_cert_path, + &dataset_path, + TimingCertificationTier::DeterministicTable, + "mc_aixi_fac_ctw", + 65_535, + false, + Some(Path::new(env!("CARGO_BIN_EXE_infotheory"))), + ); + let spec_arg = path_string(&spec_path); + let reward_cert_arg = path_string(&reward_cert_path); + let output = Command::new(env!("CARGO_BIN_EXE_infotheory")) + .args([ + "tune", + spec_arg.as_str(), + "--max-evaluations", + "1", + "--timing-tier", + "deterministic_table", + "--scalar-representation-ref", + "scalar://finite-f64", + "--exact-reward-encoding-certificate", + reward_cert_arg.as_str(), + "--claim-exact-finite-mdp", + ]) + .output() + .expect("run tune cli"); + assert!( + !output.status.success(), + "stdout={} stderr={}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("tuner requires a Unix target for process-isolated candidate evaluation"), + "stderr={stderr}" + ); + let _ = fs::remove_dir_all(dir); +} diff --git a/crates/infotheory/tests/zpaq_rate_backend.rs b/crates/infotheory/tests/zpaq_rate_backend.rs new file mode 100644 index 00000000..9a585283 --- /dev/null +++ b/crates/infotheory/tests/zpaq_rate_backend.rs @@ -0,0 +1,173 @@ +//! ZPAQ rate-backend integration tests. +//! +//! Full appendix repro for cross-FFI preceding activity requires at least +//! `backend-zpaq,backend-ctw,backend-ppmd,backend-match` (or `all-backends`). +//! Narrow `backend-zpaq` slices still run ZPAQ settlement and first-symbol parity; +//! non-ZPAQ preceding coverage accumulates with enabled backend features. + +#![cfg(feature = "backend-zpaq")] + +use infotheory::api::{OnlineBytePredictor, RateBackend}; +use infotheory::backends::zpaq_rate::ZpaqRateModel; +use infotheory::mixture::{DEFAULT_MIN_PROB, RateBackendPredictor}; + +#[test] +#[cfg(feature = "backend-zpaq")] +fn zpaq_rate_backend_compresses_copy_data() { + use infotheory::api::try_entropy_rate_backend; + + let mut data = Vec::new(); + let pattern = b"copy-like-pattern-"; + for _ in 0..512 { + data.extend_from_slice(pattern); + } + let backend = RateBackend::Zpaq { + method: infotheory::api::ZpaqMethodSpec::literal("2"), + }; + let backend = backend.compile().expect("compile zpaq rate backend"); + let rate = try_entropy_rate_backend(&data, &backend).expect("entropy rate"); + assert!( + rate < 0.5, + "expected low entropy rate for copy-like data, got {rate:.4}" + ); +} + +#[allow(dead_code)] +fn exercise_preceding_backend(label: &str, backend: RateBackend) { + let mut predictor = RateBackendPredictor::from_backend(backend, DEFAULT_MIN_PROB); + predictor + .begin_stream(Some(64)) + .unwrap_or_else(|err| panic!("{label} begin_stream failed: {err}")); + let mut row = [0.0f64; 256]; + predictor.fill_log_probs(&mut row); + for &byte in b"mixed preceding backend activity" { + let _ = predictor.log_prob(byte); + predictor.update(byte); + } + predictor + .reset_frozen(Some(16)) + .unwrap_or_else(|err| panic!("{label} reset_frozen failed: {err}")); + for &byte in b"conditioned tail" { + predictor.update_frozen(byte); + } + predictor.fill_log_probs(&mut row); + assert!( + row.iter().all(|lp| lp.is_finite()), + "{label} preceding activity produced non-finite log probabilities" + ); +} + +#[cfg(feature = "backend-ctw")] +fn exercise_preceding_ctw() { + exercise_preceding_backend("ctw", RateBackend::Ctw { depth: 8 }); +} + +#[cfg(feature = "backend-match")] +fn exercise_preceding_match() { + exercise_preceding_backend( + "match", + RateBackend::Match { + hash_bits: 12, + min_len: 3, + max_len: 16, + base_mix: 0.02, + confidence_scale: 1.0, + }, + ); + exercise_preceding_backend( + "sparse-match", + RateBackend::SparseMatch { + hash_bits: 12, + min_len: 3, + max_len: 16, + gap_min: 0, + gap_max: 2, + base_mix: 0.02, + confidence_scale: 1.0, + }, + ); +} + +#[cfg(feature = "backend-ppmd")] +fn exercise_preceding_ppmd() { + exercise_preceding_backend( + "ppmd", + RateBackend::Ppmd { + order: 6, + memory_mb: 8, + }, + ); +} + +#[cfg(all( + feature = "backend-mixture", + feature = "backend-ctw", + feature = "backend-ppmd" +))] +fn exercise_preceding_mixture() { + use infotheory::api::{MixtureExpertSpec, MixtureKind, MixtureSpec}; + use std::sync::Arc; + + exercise_preceding_backend( + "mixture", + RateBackend::Mixture { + spec: Arc::new( + MixtureSpec::new( + MixtureKind::Bayes, + vec![ + MixtureExpertSpec::new(RateBackend::Ctw { depth: 6 }).with_name("ctw"), + MixtureExpertSpec::new(RateBackend::Ppmd { + order: 4, + memory_mb: 8, + }) + .with_name("ppmd"), + ], + ) + .with_alpha(0.03), + ), + }, + ); +} + +#[test] +fn zpaq_restart_first_symbol_parity_after_preceding_activity() { + #[cfg(feature = "backend-ctw")] + exercise_preceding_ctw(); + #[cfg(feature = "backend-match")] + exercise_preceding_match(); + #[cfg(feature = "backend-ppmd")] + exercise_preceding_ppmd(); + #[cfg(all( + feature = "backend-mixture", + feature = "backend-ctw", + feature = "backend-ppmd" + ))] + exercise_preceding_mixture(); + + // ZPAQ preceding (exercises the settle path; always available under backend-zpaq): + let _preceding_zpaq = ZpaqRateModel::new("1", 1e-9); + + let mut session = ZpaqRateModel::new("1", 1e-9); + session.begin_stream(); + let mut warm = [0.0f64; 256]; + session.fill_log_probs(&mut warm); + for &byte in b"zpaq history before restart" { + session.update(byte); + } + + session.begin_stream(); + let mut restarted = [0.0f64; 256]; + session.fill_log_probs(&mut restarted); + + let mut fresh = ZpaqRateModel::new("1", 1e-9); + let mut expected = [0.0f64; 256]; + fresh.fill_log_probs(&mut expected); + + for (symbol, (&actual, &expected)) in restarted.iter().zip(expected.iter()).enumerate() { + assert!( + (actual - expected).abs() < 1e-9, + "first-symbol parity after preceding + restart failed for symbol {symbol}; diff={}", + actual - expected + ); + } +} diff --git a/crates/infotheory_py/Cargo.toml b/crates/infotheory_py/Cargo.toml new file mode 100644 index 00000000..173499cf --- /dev/null +++ b/crates/infotheory_py/Cargo.toml @@ -0,0 +1,75 @@ +[package] +name = "infotheory_py" +version = "1.2.0" +edition = "2024" +license = "ISC OR Apache-2.0" +description = "PyO3 bindings for infotheory." +homepage = "https://infotheory.tech" +repository = "https://github.com/turtle261/infotheory" +publish = false + +[lib] +name = "_core" +crate-type = ["cdylib"] + +[dependencies] +anyhow = "1.0.100" +infotheory = { path = "../infotheory", default-features = false } +pyo3 = { version = "0.29.0", features = ["abi3-py310"] } +rayon = "1.11.0" +serde_json = "1.0.149" + +[features] +default = ["default-backends", "aixi"] +# Grouped capability topology. +default-backends = ["capability-default"] +capability-default = [ + "capability-statistical", + "capability-neural", + "capability-archive", +] +capability-statistical = [ + "infotheory/capability-statistical", + "backend-rosa", + "backend-ctw", + "backend-match", + "backend-ppmd", + "backend-sequitur", + "backend-mixture", + "backend-particle", + "backend-calibrated", +] +capability-neural = [ + "infotheory/capability-neural", + "backend-mamba", + "backend-rwkv", +] +capability-archive = [ + "infotheory/capability-archive", + "backend-zpaq", +] +capability-vm = ["infotheory/capability-vm"] +aixi = ["infotheory/aixi"] +tuner = ["infotheory/tuner"] +aixi-gameengine = ["aixi", "infotheory/aixi-gameengine"] +aixi-gameengine-physics = ["aixi-gameengine", "infotheory/aixi-gameengine-physics"] +aixi-vm = ["vm"] + +# Legacy compatibility aliases. +all-backends = ["capability-default"] +python-extension = ["pyo3/extension-module", "aixi"] +backend-rosa = ["infotheory/backend-rosa"] +backend-ctw = ["infotheory/backend-ctw"] +backend-match = ["infotheory/backend-match"] +backend-ppmd = ["infotheory/backend-ppmd"] +backend-sequitur = ["infotheory/backend-sequitur"] +backend-mixture = ["infotheory/backend-mixture"] +backend-particle = ["infotheory/backend-particle"] +backend-calibrated = ["infotheory/backend-calibrated"] +backend-mamba = ["infotheory/backend-mamba"] +backend-rwkv = ["infotheory/backend-rwkv"] +backend-zpaq = ["infotheory/backend-zpaq"] +vm = ["aixi", "infotheory/aixi-vm"] + +[lints] +workspace = true diff --git a/infotheory_py/src/lib.rs b/crates/infotheory_py/src/lib.rs similarity index 64% rename from infotheory_py/src/lib.rs rename to crates/infotheory_py/src/lib.rs index c0c1da7b..f1604eae 100644 --- a/infotheory_py/src/lib.rs +++ b/crates/infotheory_py/src/lib.rs @@ -1,16 +1,17 @@ #![allow(clippy::needless_pass_by_value)] -use infotheory::{ - CalibratedSpec, CalibrationContextKind, CompressionBackend, GenerationConfig, - GenerationStrategy, GenerationUpdateMode, InfotheoryCtx, MAX_MIXTURE_NESTING, - MixtureExpertSpec, MixtureKind, MixtureScheduleMode, MixtureSpec, NcdVariant, ParticleSpec, - RateBackend, RateBackendSession, +use infotheory::api::{ + self, BinaryPrediction, BitOrder, BitStreamSemantics, BytePrefixMass, CalibratedSpec, + CalibrationContextKind, CompiledCompressionBackend, CompiledRateBackend, CompressionBackend, + GenerationConfig, GenerationStrategy, GenerationUpdateMode, InfotheoryCtx, MixtureExpertSpec, + MixtureKind, MixtureScheduleMode, MixtureSpec, NcdVariant, OnlineBitPredictor, ParticleSpec, + RateBackend, RateBackendBitSession, RateBackendBitSessionCheckpoint, RateBackendSession, }; +use infotheory::error::InfotheoryError; use pyo3::exceptions::{PyRuntimeError, PyValueError}; use pyo3::prelude::*; use pyo3::types::{PyBytes, PyDict}; use std::panic::{AssertUnwindSafe, catch_unwind}; -use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex}; use std::time::Instant; @@ -56,6 +57,35 @@ fn py_hasattr_or_fatal(obj: &Bound<'_, PyAny>, name: &str, where_: &'static str) } } +fn py_spec_value_error(err: infotheory::spec::SpecError) -> PyErr { + PyValueError::new_err(err.to_string()) +} + +fn py_value_error(err: impl std::fmt::Display) -> PyErr { + PyValueError::new_err(err.to_string()) +} + +fn compile_rate_backend(backend: RateBackend) -> PyResult { + backend.compile().map_err(py_spec_value_error) +} + +fn compile_compression_backend( + backend: CompressionBackend, +) -> PyResult { + backend.compile().map_err(py_spec_value_error) +} + +fn py_infotheory_error(err: InfotheoryError) -> PyErr { + match err { + InfotheoryError::InvalidBackendConfig(_) + | InfotheoryError::Unsupported(_) + | InfotheoryError::Spec(_) => PyValueError::new_err(err.to_string()), + InfotheoryError::Runtime(_) | InfotheoryError::Io(_) => { + PyRuntimeError::new_err(err.to_string()) + } + } +} + fn parse_ncd_variant(s: &str) -> PyResult { match s.to_ascii_lowercase().as_str() { "vitanyi" | "v" => Ok(NcdVariant::Vitanyi), @@ -69,7 +99,7 @@ fn parse_ncd_variant(s: &str) -> PyResult { fn parse_framing_mode(s: &str) -> PyResult { match s.to_ascii_lowercase().as_str() { "raw" => Ok(infotheory::compression::FramingMode::Raw), - "framed" | "frame" => Ok(infotheory::compression::FramingMode::Framed), + "framed" => Ok(infotheory::compression::FramingMode::Framed), _ => Err(PyValueError::new_err(format!( "unknown framing '{s}' (expected 'raw' or 'framed')" ))), @@ -87,15 +117,15 @@ fn parse_observation_key_mode( match s.to_ascii_lowercase().as_str() { "first" => return Ok(infotheory::aixi::common::ObservationKeyMode::First), "last" => return Ok(infotheory::aixi::common::ObservationKeyMode::Last), - "streamhash" | "stream_hash" | "stream-hash" | "hash" => { + "stream_hash" => { return Ok(infotheory::aixi::common::ObservationKeyMode::StreamHash); } - "full" | "stream" | "fullstream" | "full_stream" | "full-stream" => { + "full_stream" => { return Ok(infotheory::aixi::common::ObservationKeyMode::FullStream); } _ => { return Err(PyValueError::new_err(format!( - "unknown ObservationKeyMode '{s}' (expected one of: first, last, hash/stream_hash/stream-hash, full/full-stream/full_stream/fullstream/stream)" + "unknown ObservationKeyMode '{s}' (expected one of: first, last, stream_hash, full_stream)" ))); } } @@ -105,6 +135,51 @@ fn parse_observation_key_mode( )) } +fn default_mcts_strategy() -> infotheory::aixi::common::MctsStrategy { + infotheory::aixi::common::MctsStrategy::RhoUct +} + +fn resolve_mcts_strategy( + mcts_strategy: Option<&PyMctsStrategy>, +) -> infotheory::aixi::common::MctsStrategy { + mcts_strategy + .map(|strategy| strategy.inner) + .unwrap_or_else(default_mcts_strategy) +} + +fn py_action_alphabet_from_usize( + value: usize, + context: &'static str, +) -> PyResult { + infotheory::aixi::common::ActionAlphabet::try_from_usize(value).map_err(|_| { + PyValueError::new_err(format!( + "{context} must be >= 1 (action alphabet must be non-empty)" + )) + }) +} + +fn format_mcts_strategy(strategy: infotheory::aixi::common::MctsStrategy) -> String { + match strategy { + infotheory::aixi::common::MctsStrategy::RhoUct => "MctsStrategy.rho_uct()".to_string(), + infotheory::aixi::common::MctsStrategy::ParallelUct { + workers, + bu_uct_m_max, + } => { + let workers = workers.get(); + match bu_uct_m_max { + Some(m_max) => { + format!("MctsStrategy.parallel_uct(workers={workers}, bu_uct_m_max={m_max})") + } + None => format!("MctsStrategy.parallel_uct(workers={workers})"), + } + } + // `MctsStrategy` is `#[non_exhaustive]` so additional variants may be + // introduced by future tranches without breaking this binding; surface + // a stable fallback that exposes only the canonical kind string. + other => format!("MctsStrategy(kind={:?})", other.kind_str()), + } +} + fn parse_generation_strategy_value(py_obj: &Bound<'_, PyAny>) -> PyResult { if let Ok(strategy) = py_obj.extract::>() { return Ok(strategy.inner); @@ -112,7 +187,7 @@ fn parse_generation_strategy_value(py_obj: &Bound<'_, PyAny>) -> PyResult() { return match s.to_ascii_lowercase().as_str() { "greedy" => Ok(GenerationStrategy::Greedy), - "sample" | "sampled" => Ok(GenerationStrategy::Sample), + "sample" => Ok(GenerationStrategy::Sample), _ => Err(PyValueError::new_err(format!( "unknown GenerationStrategy '{s}' (expected 'greedy' or 'sample')" ))), @@ -153,8 +228,6 @@ fn generation_config_from_py(config: Option<&Bound<'_, PyAny>>) -> PyResult "GenerationStrategy.Greedy", GenerationStrategy::Sample => "GenerationStrategy.Sample", + _ => "GenerationStrategy.", } } } @@ -215,6 +289,7 @@ impl PyGenerationUpdateMode { match self.inner { GenerationUpdateMode::Adaptive => "GenerationUpdateMode.Adaptive", GenerationUpdateMode::Frozen => "GenerationUpdateMode.Frozen", + _ => "GenerationUpdateMode.", } } } @@ -308,187 +383,13 @@ impl PyGenerationConfig { } } -fn parse_particle_spec_json(v: &serde_json::Value) -> PyResult { - if v.get("experts").is_some() { - return Err(PyValueError::new_err( - "looks like a mixture spec (found 'experts'); expected ParticleSpec JSON", - )); - } - if let Some(kind) = v.get("kind").and_then(|k| k.as_str()) { - let k = kind.to_ascii_lowercase(); - if matches!( - k.as_str(), - "bayes" - | "fading" - | "fading-bayes" - | "switch" - | "switching" - | "mdl" - | "neural" - | "mixture" - ) { - return Err(PyValueError::new_err(format!( - "looks like a mixture spec (kind='{kind}'); expected ParticleSpec JSON" - ))); - } - } - let d = ParticleSpec::default(); - Ok(ParticleSpec { - num_particles: v - .get("num_particles") - .and_then(|x| x.as_u64()) - .map(|x| x as usize) - .unwrap_or(d.num_particles), - context_window: v - .get("context_window") - .and_then(|x| x.as_u64()) - .map(|x| x as usize) - .unwrap_or(d.context_window), - unroll_steps: v - .get("unroll_steps") - .and_then(|x| x.as_u64()) - .map(|x| x as usize) - .unwrap_or(d.unroll_steps), - num_cells: v - .get("num_cells") - .and_then(|x| x.as_u64()) - .map(|x| x as usize) - .unwrap_or(d.num_cells), - cell_dim: v - .get("cell_dim") - .and_then(|x| x.as_u64()) - .map(|x| x as usize) - .unwrap_or(d.cell_dim), - num_rules: v - .get("num_rules") - .and_then(|x| x.as_u64()) - .map(|x| x as usize) - .unwrap_or(d.num_rules), - selector_hidden: v - .get("selector_hidden") - .and_then(|x| x.as_u64()) - .map(|x| x as usize) - .unwrap_or(d.selector_hidden), - rule_hidden: v - .get("rule_hidden") - .and_then(|x| x.as_u64()) - .map(|x| x as usize) - .unwrap_or(d.rule_hidden), - noise_dim: v - .get("noise_dim") - .and_then(|x| x.as_u64()) - .map(|x| x as usize) - .unwrap_or(d.noise_dim), - deterministic: v - .get("deterministic") - .and_then(|x| x.as_bool()) - .unwrap_or(d.deterministic), - enable_noise: v - .get("enable_noise") - .and_then(|x| x.as_bool()) - .unwrap_or(d.enable_noise), - noise_scale: v - .get("noise_scale") - .and_then(|x| x.as_f64()) - .unwrap_or(d.noise_scale), - noise_anneal_steps: v - .get("noise_anneal_steps") - .and_then(|x| x.as_u64()) - .map(|x| x as usize) - .unwrap_or(d.noise_anneal_steps), - learning_rate_readout: v - .get("learning_rate_readout") - .and_then(|x| x.as_f64()) - .unwrap_or(d.learning_rate_readout), - learning_rate_selector: v - .get("learning_rate_selector") - .and_then(|x| x.as_f64()) - .unwrap_or(d.learning_rate_selector), - learning_rate_rule: v - .get("learning_rate_rule") - .and_then(|x| x.as_f64()) - .unwrap_or(d.learning_rate_rule), - bptt_depth: v - .get("bptt_depth") - .and_then(|x| x.as_u64()) - .map(|x| x as usize) - .unwrap_or(d.bptt_depth), - optimizer_momentum: v - .get("optimizer_momentum") - .and_then(|x| x.as_f64()) - .unwrap_or(d.optimizer_momentum), - grad_clip: v - .get("grad_clip") - .and_then(|x| x.as_f64()) - .unwrap_or(d.grad_clip), - state_clip: v - .get("state_clip") - .and_then(|x| x.as_f64()) - .unwrap_or(d.state_clip), - forget_lambda: v - .get("forget_lambda") - .and_then(|x| x.as_f64()) - .unwrap_or(d.forget_lambda), - resample_threshold: v - .get("resample_threshold") - .and_then(|x| x.as_f64()) - .unwrap_or(d.resample_threshold), - mutate_fraction: v - .get("mutate_fraction") - .and_then(|x| x.as_f64()) - .unwrap_or(d.mutate_fraction), - mutate_scale: v - .get("mutate_scale") - .and_then(|x| x.as_f64()) - .unwrap_or(d.mutate_scale), - mutate_model_params: v - .get("mutate_model_params") - .and_then(|x| x.as_bool()) - .unwrap_or(d.mutate_model_params), - diagnostics_interval: v - .get("diagnostics_interval") - .and_then(|x| x.as_u64()) - .map(|x| x as usize) - .unwrap_or(d.diagnostics_interval), - min_prob: v - .get("min_prob") - .and_then(|x| x.as_f64()) - .unwrap_or(d.min_prob), - seed: v.get("seed").and_then(|x| x.as_u64()).unwrap_or(d.seed), - }) -} - -fn resolve_spec_path(base_dir: &Path, path: &str) -> PathBuf { - let path = Path::new(path); - if path.is_absolute() { - path.to_path_buf() - } else { - base_dir.join(path) - } -} - -fn load_json_value_from_path( - base_dir: &Path, - path: &str, - label: &str, -) -> PyResult<(serde_json::Value, PathBuf)> { - let full = resolve_spec_path(base_dir, path); - let raw = std::fs::read_to_string(&full) - .map_err(|e| PyValueError::new_err(format!("failed to read {label} '{path}': {e}")))?; - let value: serde_json::Value = serde_json::from_str(&raw) - .map_err(|e| PyValueError::new_err(format!("invalid {label} JSON: {e}")))?; - Ok((value, full)) -} - fn parse_calibration_context_kind_alias(s: &str) -> Option { match s.trim().to_ascii_lowercase().as_str() { "global" => Some(CalibrationContextKind::Global), - "byteclass" | "byte-class" | "byte_class" | "class" => { - Some(CalibrationContextKind::ByteClass) - } + "byteclass" => Some(CalibrationContextKind::ByteClass), "text" => Some(CalibrationContextKind::Text), "repeat" => Some(CalibrationContextKind::Repeat), - "textrepeat" | "text-repeat" | "text_repeat" => Some(CalibrationContextKind::TextRepeat), + "textrepeat" => Some(CalibrationContextKind::TextRepeat), _ => None, } } @@ -510,486 +411,10 @@ fn parse_calibration_context_kind_value( )) } -fn parse_calibration_context_kind_str(value: Option<&str>) -> PyResult { - parse_calibration_context_kind_alias(value.unwrap_or("text")).ok_or_else(|| { - PyValueError::new_err(format!( - "unknown calibration context '{}'", - value.unwrap_or("text") - )) - }) -} - -fn parse_mixture_kind_json(kind: &str) -> PyResult { - infotheory::parse_mixture_kind_name(kind).map_err(PyValueError::new_err) -} - -fn parse_mixture_schedule_json(schedule: &str) -> PyResult { - infotheory::parse_mixture_schedule_name(schedule).map_err(PyValueError::new_err) -} - -fn parse_calibrated_spec_json( - v: &serde_json::Value, - base_dir: &Path, - depth: usize, -) -> PyResult { - if depth == 0 { - return Err(PyValueError::new_err("calibrated spec nesting too deep")); - } - - let base_backend = if let Some(base_v) = v.get("base") { - parse_rate_backend_json(base_v, base_dir, depth - 1)? - } else if let Some(path) = v["base_path"].as_str().or_else(|| v["path"].as_str()) { - let (value, full) = load_json_value_from_path(base_dir, path, "calibrated base backend")?; - parse_rate_backend_json(&value, full.parent().unwrap_or(base_dir), depth - 1)? - } else { - return Err(PyValueError::new_err( - "calibrated backend requires 'base' or 'base_path'", - )); - }; - - Ok(CalibratedSpec { - base: base_backend, - context: parse_calibration_context_kind_str(v["context"].as_str())?, - bins: v["bins"].as_u64().unwrap_or(33) as usize, - learning_rate: v["learning_rate"].as_f64().unwrap_or(0.02), - bias_clip: v["bias_clip"].as_f64().unwrap_or(4.0), - }) -} - -fn parse_mixture_expert_json( - v: &serde_json::Value, - base_dir: &Path, - depth: usize, -) -> PyResult { - if depth == 0 { - return Err(PyValueError::new_err("mixture spec nesting too deep")); - } - - let backend = parse_rate_backend_json(v, base_dir, depth - 1)?; - let max_order = if matches!(backend, RateBackend::RosaPlus) { - v["max_order"] - .as_i64() - .or_else(|| v["order"].as_i64()) - .unwrap_or(8) - } else { - -1 - }; - - Ok(MixtureExpertSpec { - name: v["name"].as_str().map(|s| s.to_string()), - log_prior: v["log_prior"] - .as_f64() - .or_else(|| v["prior"].as_f64()) - .unwrap_or(0.0), - max_order, - backend, - }) -} - -fn parse_mixture_spec_json( - v: &serde_json::Value, - base_dir: &Path, - depth: usize, -) -> PyResult { - if depth == 0 { - return Err(PyValueError::new_err("mixture spec nesting too deep")); - } - - let kind_str = v["kind"] - .as_str() - .or_else(|| v["mixture_kind"].as_str()) - .unwrap_or("bayes"); - let kind = parse_mixture_kind_json(kind_str)?; - let schedule = v["schedule"] - .as_str() - .or_else(|| v["schedule_mode"].as_str()) - .or_else(|| v["mixture_schedule"].as_str()) - .map(parse_mixture_schedule_json) - .transpose()? - .unwrap_or(MixtureScheduleMode::Default); - - let experts_v = v["experts"] - .as_array() - .ok_or_else(|| PyValueError::new_err("mixture spec missing 'experts' array"))?; - if experts_v.is_empty() { - return Err(PyValueError::new_err( - "mixture spec must include at least one expert", - )); - } - - let mut experts = Vec::with_capacity(experts_v.len()); - for expert in experts_v { - experts.push(parse_mixture_expert_json(expert, base_dir, depth - 1)?); - } - - let mut spec = MixtureSpec::new(kind, experts).with_schedule(schedule); - if let Some(alpha) = v["alpha"].as_f64() { - spec = spec.with_alpha(alpha); - } - if let Some(decay) = v["decay"].as_f64() { - spec = spec.with_decay(decay); - } - spec.validate().map_err(PyValueError::new_err)?; - Ok(spec) -} - -fn parse_rate_backend_json( - v: &serde_json::Value, - base_dir: &Path, - depth: usize, -) -> PyResult { - if depth == 0 { - return Err(PyValueError::new_err("backend spec nesting too deep")); - } - - let raw_kind = v["kind"] - .as_str() - .or_else(|| v["type"].as_str()) - .or_else(|| v["backend"].as_str()) - .ok_or_else(|| PyValueError::new_err("backend spec missing 'kind'"))?; - let kind = match infotheory::backends::resolve_rate_backend_name(raw_kind) { - Some(infotheory::backends::BackendAvailability::Enabled(name)) => name, - Some(infotheory::backends::BackendAvailability::Disabled { canonical, feature }) => { - return Err(PyValueError::new_err(format!( - "backend '{canonical}' requires feature '{feature}'" - ))); - } - None => { - return Err(PyValueError::new_err(format!( - "unknown backend kind '{raw_kind}'" - ))); - } - }; - - match kind { - "rosaplus" => Ok(RateBackend::RosaPlus), - "ctw" => Ok(RateBackend::Ctw { - depth: v["depth"] - .as_u64() - .or_else(|| v["ct_depth"].as_u64()) - .unwrap_or(16) as usize, - }), - "fac-ctw" => { - let base_depth = v["base_depth"] - .as_u64() - .or_else(|| v["ct_depth"].as_u64()) - .unwrap_or(16) as usize; - let encoding_bits = v["encoding_bits"].as_u64().unwrap_or(8) as usize; - let num_percept_bits = v["num_percept_bits"] - .as_u64() - .unwrap_or(encoding_bits as u64) as usize; - Ok(RateBackend::FacCtw { - base_depth, - num_percept_bits, - encoding_bits, - }) - } - "match" => Ok(RateBackend::Match { - hash_bits: v["hash_bits"].as_u64().unwrap_or(20) as usize, - min_len: v["min_len"].as_u64().unwrap_or(4) as usize, - max_len: v["max_len"].as_u64().unwrap_or(255) as usize, - base_mix: v["base_mix"].as_f64().unwrap_or(0.02), - confidence_scale: v["confidence_scale"].as_f64().unwrap_or(1.0), - }), - "sparse-match" => Ok(RateBackend::SparseMatch { - hash_bits: v["hash_bits"].as_u64().unwrap_or(19) as usize, - min_len: v["min_len"].as_u64().unwrap_or(3) as usize, - max_len: v["max_len"].as_u64().unwrap_or(64) as usize, - gap_min: v["gap_min"].as_u64().unwrap_or(1) as usize, - gap_max: v["gap_max"].as_u64().unwrap_or(2) as usize, - base_mix: v["base_mix"].as_f64().unwrap_or(0.05), - confidence_scale: v["confidence_scale"].as_f64().unwrap_or(1.0), - }), - "ppmd" => Ok(RateBackend::Ppmd { - order: v["order"].as_u64().unwrap_or(10) as usize, - memory_mb: v["memory_mb"].as_u64().unwrap_or(64) as usize, - }), - "zpaq" => { - let method = v["method"].as_str().unwrap_or("1").to_string(); - infotheory::validate_zpaq_rate_method(&method).map_err(PyValueError::new_err)?; - Ok(RateBackend::Zpaq { method }) - } - #[cfg(feature = "backend-mamba")] - "mamba" => { - if let Some(method) = v["method"].as_str().or_else(|| v["mamba_method"].as_str()) { - Ok(RateBackend::MambaMethod { - method: method.to_string(), - }) - } else { - let model_path = v["mamba_model_path"] - .as_str() - .or_else(|| v["model_path"].as_str()) - .ok_or_else(|| { - PyValueError::new_err("mamba backend requires 'method' or 'model_path'") - })?; - let full = resolve_spec_path(base_dir, model_path); - let model = - infotheory::load_mamba_model_from_path(full.to_str().unwrap_or(model_path)); - Ok(RateBackend::Mamba { model }) - } - } - #[cfg(not(feature = "backend-mamba"))] - "mamba" => Err(PyValueError::new_err( - "mamba backend disabled at compile time", - )), - #[cfg(feature = "backend-rwkv")] - "rwkv7" => { - if let Some(method) = v["method"].as_str().or_else(|| v["rwkv_method"].as_str()) { - Ok(RateBackend::Rwkv7Method { - method: method.to_string(), - }) - } else { - let model_path = v["rwkv_model_path"] - .as_str() - .or_else(|| v["model_path"].as_str()) - .ok_or_else(|| { - PyValueError::new_err("rwkv7 backend requires 'method' or 'model_path'") - })?; - let full = resolve_spec_path(base_dir, model_path); - let model = - infotheory::load_rwkv7_model_from_path(full.to_str().unwrap_or(model_path)); - Ok(RateBackend::Rwkv7 { model }) - } - } - #[cfg(not(feature = "backend-rwkv"))] - "rwkv7" => Err(PyValueError::new_err( - "rwkv backend disabled at compile time", - )), - "mixture" => { - let spec = if let Some(spec_v) = v.get("spec").filter(|value| value.is_object()) { - parse_mixture_spec_json(spec_v, base_dir, depth - 1)? - } else if let Some(path) = v["spec_path"].as_str().or_else(|| v["spec"].as_str()) { - let (value, full) = load_json_value_from_path(base_dir, path, "mixture spec")?; - parse_mixture_spec_json(&value, full.parent().unwrap_or(base_dir), depth - 1)? - } else { - return Err(PyValueError::new_err( - "mixture backend requires inline 'spec' or 'spec_path'", - )); - }; - Ok(RateBackend::Mixture { - spec: Arc::new(spec), - }) - } - "particle" => { - let spec = if let Some(spec_v) = v.get("spec").filter(|value| value.is_object()) { - parse_particle_spec_json(spec_v)? - } else if let Some(path) = v["spec_path"].as_str().or_else(|| v["spec"].as_str()) { - let (value, _) = load_json_value_from_path(base_dir, path, "particle spec")?; - parse_particle_spec_json(&value)? - } else { - return Err(PyValueError::new_err( - "particle backend requires inline 'spec' or 'spec_path'", - )); - }; - spec.validate() - .map_err(|e| PyValueError::new_err(format!("invalid particle spec: {e}")))?; - Ok(RateBackend::Particle { - spec: Arc::new(spec), - }) - } - "calibrated" => { - let spec = if let Some(spec_v) = v.get("spec").filter(|value| value.is_object()) { - parse_calibrated_spec_json(spec_v, base_dir, depth - 1)? - } else if let Some(path) = v["spec_path"].as_str().or_else(|| v["spec"].as_str()) { - let (value, full) = load_json_value_from_path(base_dir, path, "calibrated spec")?; - parse_calibrated_spec_json(&value, full.parent().unwrap_or(base_dir), depth - 1)? - } else { - parse_calibrated_spec_json(v, base_dir, depth - 1)? - }; - Ok(RateBackend::Calibrated { - spec: Arc::new(spec), - }) - } - other => Err(PyValueError::new_err(format!( - "unsupported backend kind '{other}'" - ))), - } -} - fn parse_rate_backend(name: &str, method: Option<&str>) -> PyResult { - let m = method.unwrap_or_default(); - match name.to_ascii_lowercase().as_str() { - "rosa" | "rosaplus" => Ok(RateBackend::RosaPlus), - "match" => Ok(RateBackend::Match { - hash_bits: 20, - min_len: 4, - max_len: 255, - base_mix: 0.02, - confidence_scale: 1.0, - }), - "sparse-match" | "sparse_match" | "sparsematch" => Ok(RateBackend::SparseMatch { - hash_bits: 19, - min_len: 3, - max_len: 64, - gap_min: 1, - gap_max: 2, - base_mix: 0.05, - confidence_scale: 1.0, - }), - "ppmd" | "ppm" => Ok(RateBackend::Ppmd { - order: if m.is_empty() { - 10 - } else { - m.parse().unwrap_or(10) - }, - memory_mb: 64, - }), - "ctw" => Ok(RateBackend::Ctw { - depth: if m.is_empty() { - 16 - } else { - m.parse().unwrap_or(16) - }, - }), - "fac-ctw" | "facctw" => { - let depth = if m.is_empty() { - 16 - } else { - m.parse().unwrap_or(16) - }; - Ok(RateBackend::FacCtw { - base_depth: depth, - num_percept_bits: 8, - encoding_bits: 8, - }) - } - "zpaq" => Ok(RateBackend::Zpaq { - method: if m.is_empty() { - "1".to_string() - } else { - m.to_string() - }, - }), - #[cfg(feature = "backend-mamba")] - "mamba" | "mamba1" => { - if m.is_empty() { - Err(PyValueError::new_err( - "mamba backend requires method string (cfg:...;policy:... or file:...)", - )) - } else { - Ok(RateBackend::MambaMethod { - method: m.to_string(), - }) - } - } - #[cfg(not(feature = "backend-mamba"))] - "mamba" | "mamba1" => Err(PyValueError::new_err( - "mamba backend disabled at compile time", - )), - #[cfg(feature = "backend-rwkv")] - "rwkv" | "rwkv7" => { - if m.is_empty() { - Err(PyValueError::new_err( - "rwkv backend requires method string (cfg:...;policy:... or file:...)", - )) - } else { - Ok(RateBackend::Rwkv7Method { - method: m.to_string(), - }) - } - } - #[cfg(not(feature = "backend-rwkv"))] - "rwkv" | "rwkv7" => Err(PyValueError::new_err( - "rwkv backend disabled at compile time", - )), - "mixture" | "mix" => { - if m.is_empty() { - return Err(PyValueError::new_err( - "mixture backend requires method path to a MixtureSpec JSON file", - )); - } - let (value, full) = load_json_value_from_path(Path::new("."), m, "mixture spec")?; - let spec = parse_mixture_spec_json( - &value, - full.parent().unwrap_or(Path::new(".")), - MAX_MIXTURE_NESTING, - )?; - Ok(RateBackend::Mixture { - spec: Arc::new(spec), - }) - } - "particle" | "particles" => { - let spec = if m.is_empty() { - ParticleSpec::default() - } else { - let (value, _) = load_json_value_from_path(Path::new("."), m, "particle spec")?; - parse_particle_spec_json(&value)? - }; - spec.validate() - .map_err(|e| PyValueError::new_err(format!("invalid particle spec: {e}")))?; - Ok(RateBackend::Particle { - spec: Arc::new(spec), - }) - } - "calibrated" | "cal" => { - if m.is_empty() { - return Err(PyValueError::new_err( - "calibrated backend requires method path to a CalibratedSpec JSON file", - )); - } - let (value, full) = load_json_value_from_path(Path::new("."), m, "calibrated spec")?; - let spec = parse_calibrated_spec_json( - &value, - full.parent().unwrap_or(Path::new(".")), - MAX_CALIBRATED_SPEC_NESTING, - )?; - Ok(RateBackend::Calibrated { - spec: Arc::new(spec), - }) - } - _ => Err(PyValueError::new_err(format!( - "unknown rate backend '{name}'" - ))), - } -} - -#[cfg(feature = "backend-rwkv")] -fn parse_rwkv7_compression_backend(method: Option<&str>) -> PyResult { - let coder = infotheory::coders::CoderType::AC; - match method { - Some(m) if infotheory::backends::parse_rwkv7_coder(m).is_some() => { - let path = std::env::var("RWKV7_MODEL_PATH") - .map_err(|_| PyValueError::new_err("RWKV7_MODEL_PATH not set"))?; - let model = infotheory::load_rwkv7_model_from_path(&path); - Ok(CompressionBackend::Rwkv7 { - model, - coder: infotheory::backends::parse_rwkv7_coder(m) - .expect("coder alias already validated"), - }) - } - Some(m) => match infotheory::rwkvzip::parse_method_spec(m) { - Ok(infotheory::rwkvzip::MethodSpec::File { path, policy: None }) => { - let model = infotheory::load_rwkv7_model_from_path(path.to_string_lossy().as_ref()); - Ok(CompressionBackend::Rwkv7 { model, coder }) - } - Ok(infotheory::rwkvzip::MethodSpec::File { - policy: Some(_), .. - }) - | Ok(infotheory::rwkvzip::MethodSpec::Online { .. }) => Ok(CompressionBackend::Rate { - rate_backend: RateBackend::Rwkv7Method { - method: m.to_string(), - }, - coder, - framing: infotheory::compression::FramingMode::Raw, - }), - Err(e) => Err(PyValueError::new_err(format!( - "invalid rwkv method string: {e}" - ))), - }, - None => { - let path = std::env::var("RWKV7_MODEL_PATH") - .map_err(|_| PyValueError::new_err("RWKV7_MODEL_PATH not set"))?; - let model = infotheory::load_rwkv7_model_from_path(&path); - Ok(CompressionBackend::Rwkv7 { model, coder }) - } - } -} - -#[cfg(not(feature = "backend-rwkv"))] -fn parse_rwkv7_compression_backend(_method: Option<&str>) -> PyResult { - Err(PyValueError::new_err( - "rwkv7 compression backend disabled at compile time", - )) + let opts = infotheory::spec::RateBackendShorthandOptions::default(); + infotheory::spec::parse_rate_backend_name_method(name, method, &opts) + .map_err(py_spec_value_error) } fn parse_compression_backend( @@ -997,30 +422,18 @@ fn parse_compression_backend( method: Option<&str>, rate_backend: Option, ) -> PyResult { - let m = method.unwrap_or_default(); - match name.to_ascii_lowercase().as_str() { - "zpaq" => Ok(CompressionBackend::Zpaq { - method: if m.is_empty() { - "5".to_string() - } else { - m.to_string() - }, - }), - "rate-ac" | "rate_ac" | "rateac" => Ok(CompressionBackend::Rate { - rate_backend: rate_backend.unwrap_or_default(), - coder: infotheory::coders::CoderType::AC, - framing: infotheory::compression::FramingMode::Framed, - }), - "rate-rans" | "rate_rans" | "raterans" => Ok(CompressionBackend::Rate { - rate_backend: rate_backend.unwrap_or_default(), - coder: infotheory::coders::CoderType::RANS, - framing: infotheory::compression::FramingMode::Framed, - }), - "rwkv" | "rwkv7" => parse_rwkv7_compression_backend(method), - _ => Err(PyValueError::new_err(format!( - "unknown compression backend '{name}'" - ))), + let mut opts = infotheory::spec::CompressionBackendShorthandOptions::default(); + opts.default_rate_backend = rate_backend; + opts.default_framing = infotheory::compression::FramingMode::Framed; + + #[cfg(feature = "backend-rwkv")] + if name.trim().eq_ignore_ascii_case("rwkv7") { + if let Ok(path) = std::env::var("RWKV7_MODEL_PATH") { + opts.default_rwkv_model_path = Some(path); + } } + infotheory::spec::parse_compression_backend_name_method(name, method, None, &opts) + .map_err(py_spec_value_error) } #[pyclass(name = "MixtureKind", from_py_object)] @@ -1109,16 +522,12 @@ struct PyMixtureExpertSpec { #[pymethods] impl PyMixtureExpertSpec { #[new] - #[pyo3(signature = (backend, max_order=-1, log_prior=0.0, name=None))] - fn new(backend: &PyRateBackend, max_order: i64, log_prior: f64, name: Option) -> Self { - Self { - inner: MixtureExpertSpec { - name, - log_prior, - max_order, - backend: backend.inner.clone(), - }, - } + #[pyo3(signature = (backend, log_prior=0.0, name=None))] + fn new(backend: &PyRateBackend, log_prior: f64, name: Option) -> Self { + let mut inner = MixtureExpertSpec::new(backend.inner.clone()); + inner.name = name; + inner.log_prior = log_prior; + Self { inner } } } @@ -1221,36 +630,35 @@ impl PyParticleSpec { min_prob: f64, seed: u64, ) -> PyResult { - let spec = ParticleSpec { - num_particles, - context_window, - unroll_steps, - num_cells, - cell_dim, - num_rules, - selector_hidden, - rule_hidden, - noise_dim, - deterministic, - enable_noise, - noise_scale, - noise_anneal_steps, - learning_rate_readout, - learning_rate_selector, - learning_rate_rule, - bptt_depth, - optimizer_momentum, - grad_clip, - state_clip, - forget_lambda, - resample_threshold, - mutate_fraction, - mutate_scale, - mutate_model_params, - diagnostics_interval, - min_prob, - seed, - }; + let mut spec = ParticleSpec::default(); + spec.num_particles = num_particles; + spec.context_window = context_window; + spec.unroll_steps = unroll_steps; + spec.num_cells = num_cells; + spec.cell_dim = cell_dim; + spec.num_rules = num_rules; + spec.selector_hidden = selector_hidden; + spec.rule_hidden = rule_hidden; + spec.noise_dim = noise_dim; + spec.deterministic = deterministic; + spec.enable_noise = enable_noise; + spec.noise_scale = noise_scale; + spec.noise_anneal_steps = noise_anneal_steps; + spec.learning_rate_readout = learning_rate_readout; + spec.learning_rate_selector = learning_rate_selector; + spec.learning_rate_rule = learning_rate_rule; + spec.bptt_depth = bptt_depth; + spec.optimizer_momentum = optimizer_momentum; + spec.grad_clip = grad_clip; + spec.state_clip = state_clip; + spec.forget_lambda = forget_lambda; + spec.resample_threshold = resample_threshold; + spec.mutate_fraction = mutate_fraction; + spec.mutate_scale = mutate_scale; + spec.mutate_model_params = mutate_model_params; + spec.diagnostics_interval = diagnostics_interval; + spec.min_prob = min_prob; + spec.seed = seed; spec.validate() .map_err(|e| PyValueError::new_err(format!("invalid ParticleSpec: {e}")))?; Ok(Self { inner: spec }) @@ -1319,6 +727,7 @@ impl PyCalibrationContextKind { CalibrationContextKind::Text => "CalibrationContextKind.Text", CalibrationContextKind::Repeat => "CalibrationContextKind.Repeat", CalibrationContextKind::TextRepeat => "CalibrationContextKind.TextRepeat", + _ => "CalibrationContextKind.", } } } @@ -1332,9 +741,10 @@ struct PyRateBackend { #[pymethods] impl PyRateBackend { #[staticmethod] - fn rosaplus() -> Self { + #[pyo3(signature = (max_order=-1))] + fn rosaplus(max_order: i64) -> Self { Self { - inner: RateBackend::RosaPlus, + inner: RateBackend::RosaPlus { max_order }, } } @@ -1347,13 +757,19 @@ impl PyRateBackend { } #[staticmethod] - #[pyo3(signature = (base_depth=16, num_percept_bits=8, encoding_bits=8))] - fn fac_ctw(base_depth: usize, num_percept_bits: usize, encoding_bits: usize) -> Self { + #[pyo3(signature = (base_depth=16, num_percept_bits=8, encoding_bits=8, msb_first=None))] + fn fac_ctw( + base_depth: usize, + num_percept_bits: usize, + encoding_bits: usize, + msb_first: Option, + ) -> Self { Self { inner: RateBackend::FacCtw { base_depth, num_percept_bits, encoding_bits, + msb_first, }, } } @@ -1421,28 +837,27 @@ impl PyRateBackend { #[staticmethod] #[pyo3(signature = (method=None))] - fn zpaq(method: Option) -> Self { - Self { + fn zpaq(method: Option) -> PyResult { + infotheory::spec::resolve_enabled_rate_backend_name("zpaq").map_err(py_spec_value_error)?; + Ok(Self { inner: RateBackend::Zpaq { - method: method.unwrap_or_else(|| "1".to_string()), + method: infotheory::api::ZpaqMethodSpec::literal( + method.unwrap_or_else(|| "2".to_string()), + ), }, - } + }) } #[staticmethod] #[cfg(feature = "backend-mamba")] - fn mamba(method: String) -> Self { - Self { - inner: RateBackend::MambaMethod { method }, - } + fn mamba(method: String) -> PyResult { + parse_rate_backend("mamba", Some(method.as_str())).map(|inner| Self { inner }) } #[staticmethod] #[cfg(feature = "backend-rwkv")] - fn rwkv7(method: String) -> Self { - Self { - inner: RateBackend::Rwkv7Method { method }, - } + fn rwkv7(method: String) -> PyResult { + parse_rate_backend("rwkv7", Some(method.as_str())).map(|inner| Self { inner }) } #[staticmethod] @@ -1475,18 +890,17 @@ impl PyRateBackend { learning_rate: f64, bias_clip: f64, ) -> PyResult { + let context_kind = context + .map(parse_calibration_context_kind_value) + .transpose()? + .unwrap_or(CalibrationContextKind::Text); + let mut cal_spec = CalibratedSpec::new(base_backend.inner.clone(), context_kind); + cal_spec.bins = bins; + cal_spec.learning_rate = learning_rate; + cal_spec.bias_clip = bias_clip; Ok(Self { inner: RateBackend::Calibrated { - spec: Arc::new(CalibratedSpec { - base: base_backend.inner.clone(), - context: context - .map(parse_calibration_context_kind_value) - .transpose()? - .unwrap_or(CalibrationContextKind::Text), - bins, - learning_rate, - bias_clip, - }), + spec: Arc::new(cal_spec), }, }) } @@ -1506,12 +920,12 @@ struct PyCompressionBackend { impl PyCompressionBackend { #[staticmethod] #[pyo3(signature = (method=None))] - fn zpaq(method: Option) -> Self { - Self { - inner: CompressionBackend::Zpaq { - method: method.unwrap_or_else(|| "5".to_string()), - }, - } + fn zpaq(method: Option) -> PyResult { + infotheory::spec::resolve_enabled_compression_backend_name("zpaq") + .map_err(py_spec_value_error)?; + Ok(Self { + inner: CompressionBackend::zpaq(method.unwrap_or_else(|| "5".to_string())), + }) } #[staticmethod] @@ -1546,38 +960,18 @@ impl PyCompressionBackend { fn rwkv7(method: Option, coder: &str) -> PyResult { let coder = infotheory::backends::parse_rwkv7_coder(coder) .ok_or_else(|| PyValueError::new_err("coder must be 'ac' or 'rans'"))?; - match method { - Some(m) => match infotheory::rwkvzip::parse_method_spec(&m) { - Ok(infotheory::rwkvzip::MethodSpec::File { path, policy: None }) => { - let model = - infotheory::load_rwkv7_model_from_path(path.to_string_lossy().as_ref()); - Ok(Self { - inner: CompressionBackend::Rwkv7 { model, coder }, - }) - } - Ok(infotheory::rwkvzip::MethodSpec::File { - policy: Some(_), .. - }) - | Ok(infotheory::rwkvzip::MethodSpec::Online { .. }) => Ok(Self { - inner: CompressionBackend::Rate { - rate_backend: RateBackend::Rwkv7Method { method: m }, - coder, - framing: infotheory::compression::FramingMode::Raw, - }, - }), - Err(e) => Err(PyValueError::new_err(format!( - "invalid rwkv method string: {e}" - ))), - }, - None => { - let path = std::env::var("RWKV7_MODEL_PATH") - .map_err(|_| PyValueError::new_err("RWKV7_MODEL_PATH not set"))?; - let model = infotheory::load_rwkv7_model_from_path(&path); - Ok(Self { - inner: CompressionBackend::Rwkv7 { model, coder }, - }) - } + let mut opts = infotheory::spec::CompressionBackendShorthandOptions::default(); + opts.default_framing = infotheory::compression::FramingMode::Framed; + if let Ok(path) = std::env::var("RWKV7_MODEL_PATH") { + opts.default_rwkv_model_path = Some(path); } + let inner = infotheory::spec::parse_rwkv7_compression_backend_method( + method.as_deref(), + coder, + &opts, + ) + .map_err(py_spec_value_error)?; + Ok(Self { inner }) } fn __repr__(&self) -> String { @@ -1591,12 +985,6 @@ struct PyInfotheoryCtx { inner: InfotheoryCtx, } -#[pyclass(name = "RateBackendSession", from_py_object)] -#[derive(Clone)] -struct PyRateBackendSession { - inner: Arc>, -} - #[pymethods] impl PyInfotheoryCtx { #[new] @@ -1604,36 +992,59 @@ impl PyInfotheoryCtx { fn new( rate_backend: Option<&PyRateBackend>, compression_backend: Option<&PyCompressionBackend>, - ) -> Self { - let rb = rate_backend.map(|b| b.inner.clone()).unwrap_or_default(); - let cb = compression_backend - .map(|b| b.inner.clone()) - .unwrap_or_default(); - Self { + ) -> PyResult { + let rb = compile_rate_backend(match rate_backend { + Some(backend) => backend.inner.clone(), + None => RateBackend::try_default().map_err(py_infotheory_error)?, + })?; + let cb = compile_compression_backend(match compression_backend { + Some(backend) => backend.inner.clone(), + None => CompressionBackend::try_default().map_err(py_infotheory_error)?, + })?; + Ok(Self { inner: InfotheoryCtx::new(rb, cb), - } + }) } - fn entropy_rate_bytes(&self, py: Python<'_>, data: &[u8], max_order: i64) -> PyResult { - py.detach(|| py_try(|| Ok(self.inner.entropy_rate_bytes(data, max_order)))) + fn entropy_rate_bytes(&self, py: Python<'_>, data: &[u8]) -> PyResult { + py.detach(|| { + py_try(|| { + self.inner + .try_entropy_rate_bytes(data) + .map_err(py_infotheory_error) + }) + }) } - fn biased_entropy_rate_bytes( - &self, - py: Python<'_>, - data: &[u8], - max_order: i64, - ) -> PyResult { - py.detach(|| py_try(|| Ok(self.inner.biased_entropy_rate_bytes(data, max_order)))) + fn biased_entropy_rate_bytes(&self, py: Python<'_>, data: &[u8]) -> PyResult { + py.detach(|| { + py_try(|| { + self.inner + .try_biased_entropy_rate_bytes(data) + .map_err(py_infotheory_error) + }) + }) } fn compress_size(&self, py: Python<'_>, data: &[u8]) -> PyResult { - py.detach(|| py_try(|| Ok(self.inner.compress_size(data)))) + py.detach(|| { + py_try(|| { + self.inner + .try_compress_size(data) + .map_err(py_infotheory_error) + }) + }) } fn compress_size_chain(&self, py: Python<'_>, parts: Vec>) -> PyResult { let refs: Vec<&[u8]> = parts.iter().map(Vec::as_slice).collect(); - py.detach(|| py_try(|| Ok(self.inner.compress_size_chain(&refs)))) + py.detach(|| { + py_try(|| { + self.inner + .try_compress_size_chain(&refs) + .map_err(py_infotheory_error) + }) + }) } fn cross_entropy_rate_bytes( @@ -1641,13 +1052,12 @@ impl PyInfotheoryCtx { py: Python<'_>, test_data: &[u8], train_data: &[u8], - max_order: i64, ) -> PyResult { py.detach(|| { py_try(|| { - Ok(self - .inner - .cross_entropy_rate_bytes(test_data, train_data, max_order)) + self.inner + .try_cross_entropy_rate_bytes(test_data, train_data) + .map_err(py_infotheory_error) }) }) } @@ -1657,35 +1067,34 @@ impl PyInfotheoryCtx { py: Python<'_>, test_data: &[u8], train_data: &[u8], - max_order: i64, ) -> PyResult { py.detach(|| { py_try(|| { - Ok(self - .inner - .cross_entropy_bytes(test_data, train_data, max_order)) + self.inner + .try_cross_entropy_bytes(test_data, train_data) + .map_err(py_infotheory_error) }) }) } - fn joint_entropy_rate_bytes( - &self, - py: Python<'_>, - x: &[u8], - y: &[u8], - max_order: i64, - ) -> PyResult { - py.detach(|| py_try(|| Ok(self.inner.joint_entropy_rate_bytes(x, y, max_order)))) + fn joint_entropy_rate_bytes(&self, py: Python<'_>, x: &[u8], y: &[u8]) -> PyResult { + py.detach(|| { + py_try(|| { + self.inner + .try_joint_entropy_rate_bytes(x, y) + .map_err(py_infotheory_error) + }) + }) } - fn conditional_entropy_rate_bytes( - &self, - py: Python<'_>, - x: &[u8], - y: &[u8], - max_order: i64, - ) -> PyResult { - py.detach(|| py_try(|| Ok(self.inner.conditional_entropy_rate_bytes(x, y, max_order)))) + fn conditional_entropy_rate_bytes(&self, py: Python<'_>, x: &[u8], y: &[u8]) -> PyResult { + py.detach(|| { + py_try(|| { + self.inner + .try_conditional_entropy_rate_bytes(x, y) + .map_err(py_infotheory_error) + }) + }) } fn cross_entropy_conditional_chain( @@ -1695,58 +1104,71 @@ impl PyInfotheoryCtx { data: &[u8], ) -> PyResult { let refs: Vec<&[u8]> = prefix_parts.iter().map(Vec::as_slice).collect(); - py.detach(|| py_try(|| Ok(self.inner.cross_entropy_conditional_chain(&refs, data)))) + py.detach(|| { + py_try(|| { + self.inner + .try_cross_entropy_conditional_chain(&refs, data) + .map_err(py_infotheory_error) + }) + }) } - fn mutual_information_rate_bytes( - &self, - py: Python<'_>, - x: &[u8], - y: &[u8], - max_order: i64, - ) -> PyResult { - py.detach(|| py_try(|| Ok(self.inner.mutual_information_rate_bytes(x, y, max_order)))) + fn mutual_information_rate_bytes(&self, py: Python<'_>, x: &[u8], y: &[u8]) -> PyResult { + py.detach(|| { + py_try(|| { + self.inner + .try_mutual_information_rate_bytes(x, y) + .map_err(py_infotheory_error) + }) + }) } - fn mutual_information_bytes( - &self, - py: Python<'_>, - x: &[u8], - y: &[u8], - max_order: i64, - ) -> PyResult { - py.detach(|| py_try(|| Ok(self.inner.mutual_information_bytes(x, y, max_order)))) + fn mutual_information_bytes(&self, py: Python<'_>, x: &[u8], y: &[u8]) -> PyResult { + py.detach(|| { + py_try(|| { + self.inner + .try_mutual_information_bytes(x, y) + .map_err(py_infotheory_error) + }) + }) } - fn conditional_entropy_bytes( - &self, - py: Python<'_>, - x: &[u8], - y: &[u8], - max_order: i64, - ) -> PyResult { - py.detach(|| py_try(|| Ok(self.inner.conditional_entropy_bytes(x, y, max_order)))) + fn conditional_entropy_bytes(&self, py: Python<'_>, x: &[u8], y: &[u8]) -> PyResult { + py.detach(|| { + py_try(|| { + self.inner + .try_conditional_entropy_bytes(x, y) + .map_err(py_infotheory_error) + }) + }) } - fn ned_bytes(&self, py: Python<'_>, x: &[u8], y: &[u8], max_order: i64) -> PyResult { - py.detach(|| py_try(|| Ok(self.inner.ned_bytes(x, y, max_order)))) + fn ned_bytes(&self, py: Python<'_>, x: &[u8], y: &[u8]) -> PyResult { + py.detach(|| py_try(|| self.inner.try_ned_bytes(x, y).map_err(py_infotheory_error))) } - fn ned_cons_bytes(&self, py: Python<'_>, x: &[u8], y: &[u8], max_order: i64) -> PyResult { - py.detach(|| py_try(|| Ok(self.inner.ned_cons_bytes(x, y, max_order)))) + fn ned_cons_bytes(&self, py: Python<'_>, x: &[u8], y: &[u8]) -> PyResult { + py.detach(|| { + py_try(|| { + self.inner + .try_ned_cons_bytes(x, y) + .map_err(py_infotheory_error) + }) + }) } - fn nte_bytes(&self, py: Python<'_>, x: &[u8], y: &[u8], max_order: i64) -> PyResult { - py.detach(|| py_try(|| Ok(self.inner.nte_bytes(x, y, max_order)))) + fn nte_bytes(&self, py: Python<'_>, x: &[u8], y: &[u8]) -> PyResult { + py.detach(|| py_try(|| self.inner.try_nte_bytes(x, y).map_err(py_infotheory_error))) } - fn intrinsic_dependence_bytes( - &self, - py: Python<'_>, - data: &[u8], - max_order: i64, - ) -> PyResult { - py.detach(|| py_try(|| Ok(self.inner.intrinsic_dependence_bytes(data, max_order)))) + fn intrinsic_dependence_bytes(&self, py: Python<'_>, data: &[u8]) -> PyResult { + py.detach(|| { + py_try(|| { + self.inner + .try_intrinsic_dependence_bytes(data) + .map_err(py_infotheory_error) + }) + }) } fn resistance_to_transformation_bytes( @@ -1754,73 +1176,86 @@ impl PyInfotheoryCtx { py: Python<'_>, x: &[u8], tx: &[u8], - max_order: i64, ) -> PyResult { py.detach(|| { py_try(|| { - Ok(self - .inner - .resistance_to_transformation_bytes(x, tx, max_order)) + self.inner + .try_resistance_to_transformation_bytes(x, tx) + .map_err(py_infotheory_error) }) }) } - #[pyo3(signature = (prompt, bytes, max_order=-1, config=None))] + #[pyo3(signature = (prompt, bytes, config=None))] fn generate_bytes<'py>( &self, py: Python<'py>, prompt: &[u8], bytes: usize, - max_order: i64, config: Option<&Bound<'_, PyAny>>, ) -> PyResult> { let cfg = generation_config_from_py(config)?; - let out = py.detach(|| { + let out: Vec = py.detach(|| { py_try(|| { - Ok(self - .inner - .generate_bytes_with_config(prompt, bytes, max_order, cfg)) + self.inner + .try_generate_bytes_with_config(prompt, bytes, cfg) + .map_err(py_infotheory_error) }) })?; Ok(PyBytes::new(py, &out)) } - #[pyo3(signature = (prefix_parts, bytes, max_order=-1, config=None))] + #[pyo3(signature = (prefix_parts, bytes, config=None))] fn generate_bytes_conditional_chain<'py>( &self, py: Python<'py>, prefix_parts: Vec>, bytes: usize, - max_order: i64, config: Option<&Bound<'_, PyAny>>, ) -> PyResult> { let cfg = generation_config_from_py(config)?; let refs: Vec<&[u8]> = prefix_parts.iter().map(Vec::as_slice).collect(); - let out = py.detach(|| { + let out: Vec = py.detach(|| { py_try(|| { - Ok(self - .inner - .generate_bytes_conditional_chain_with_config(&refs, bytes, max_order, cfg)) + self.inner + .try_generate_bytes_conditional_chain_with_config(&refs, bytes, cfg) + .map_err(py_infotheory_error) }) })?; Ok(PyBytes::new(py, &out)) } - #[pyo3(signature = (max_order=-1, total_symbols=None))] - fn rate_backend_session( - &self, - max_order: i64, - total_symbols: Option, - ) -> PyResult { + #[pyo3(signature = (total_symbols=None))] + fn rate_backend_session(&self, total_symbols: Option) -> PyResult { let inner = self .inner - .rate_backend_session(max_order, total_symbols) - .map_err(|e| PyRuntimeError::new_err(e.to_string()))?; + .rate_backend_session(total_symbols) + .map_err(py_infotheory_error)?; Ok(PyRateBackendSession { inner: Arc::new(Mutex::new(inner)), }) } + #[pyo3(signature = (total_bits=None, semantics=None))] + fn rate_backend_bit_session( + &self, + total_bits: Option, + semantics: Option<&Bound<'_, PyAny>>, + ) -> PyResult { + let sem = match semantics { + Some(s) => parse_bit_stream_semantics_value(s)?, + None => BitStreamSemantics::default(), + }; + let inner = self + .inner + .rate_backend_bit_session(total_bits, sem) + .map_err(py_infotheory_error)?; + Ok(PyRateBackendBitSession { + inner: Arc::new(Mutex::new(inner)), + semantics: sem, + }) + } + #[pyo3(signature = (x, y, variant=None))] fn ncd_bytes( &self, @@ -1830,7 +1265,13 @@ impl PyInfotheoryCtx { variant: Option, ) -> PyResult { let v = parse_ncd_variant(variant.as_deref().unwrap_or("vitanyi"))?; - py.detach(|| py_try(|| Ok(self.inner.ncd_bytes(x, y, v)))) + py.detach(|| { + py_try(|| { + self.inner + .try_ncd_bytes(x, y, v) + .map_err(py_infotheory_error) + }) + }) } #[pyo3(signature = (x, y, variant=None))] @@ -1844,26 +1285,30 @@ impl PyInfotheoryCtx { let v = parse_ncd_variant(variant.as_deref().unwrap_or("vitanyi"))?; py.detach(|| { py_try(|| { - Ok(infotheory::ncd_paths_backend( - x, - y, - &self.inner.compression_backend, - v, - )) + api::try_ncd_paths_backend(x, y, self.inner.compression_backend.canonical_spec(), v) + .map_err(py_infotheory_error) }) }) } } +#[pyclass(name = "RateBackendSession", from_py_object)] +#[derive(Clone)] +struct PyRateBackendSession { + inner: Arc>, +} + #[pymethods] impl PyRateBackendSession { #[new] - #[pyo3(signature = (backend, max_order=-1, total_symbols=None))] - fn new(backend: &PyRateBackend, max_order: i64, total_symbols: Option) -> PyResult { + #[pyo3(signature = (backend, total_symbols=None))] + fn new(backend: &PyRateBackend, total_symbols: Option) -> PyResult { py_try(|| { - let inner = - RateBackendSession::from_backend(backend.inner.clone(), max_order, total_symbols) - .map_err(|e| PyRuntimeError::new_err(e.to_string()))?; + let inner = RateBackendSession::from_backend( + compile_rate_backend(backend.inner.clone())?, + total_symbols, + ) + .map_err(py_infotheory_error)?; Ok(Self { inner: Arc::new(Mutex::new(inner)), }) @@ -1882,7 +1327,14 @@ impl PyRateBackendSession { fn reset_frozen(&self, total_symbols: Option) -> PyResult<()> { lock_recover(&self.inner) .reset_frozen(total_symbols) - .map_err(|e| PyRuntimeError::new_err(e.to_string())) + .map_err(py_infotheory_error) + } + + #[pyo3(signature = (total_symbols=None))] + fn begin_stream(&self, total_symbols: Option) -> PyResult<()> { + lock_recover(&self.inner) + .begin_stream(total_symbols) + .map_err(py_infotheory_error) } fn fill_log_probs(&self) -> Vec { @@ -1909,79 +1361,495 @@ impl PyRateBackendSession { fn finish(&self) -> PyResult<()> { lock_recover(&self.inner) .finish() - .map_err(|e| PyRuntimeError::new_err(e.to_string())) + .map_err(py_infotheory_error) } } -#[pyfunction] -fn get_default_ctx() -> PyInfotheoryCtx { - PyInfotheoryCtx { - inner: infotheory::get_default_ctx(), +/// Ordering configuration for byte-to-bit factorizations. +#[pyclass(name = "BitOrder", eq, from_py_object)] +#[derive(Clone, Copy, Eq, PartialEq, Debug)] +struct PyBitOrder { + inner: BitOrder, +} + +#[pymethods] +impl PyBitOrder { + #[classattr] + #[pyo3(name = "MsbFirst")] + fn msb_first() -> Self { + Self { + inner: BitOrder::MsbFirst, + } + } + + #[classattr] + #[pyo3(name = "LsbFirst")] + fn lsb_first() -> Self { + Self { + inner: BitOrder::LsbFirst, + } + } + + fn __repr__(&self) -> &'static str { + match self.inner { + BitOrder::MsbFirst => "BitOrder.MsbFirst", + BitOrder::LsbFirst => "BitOrder.LsbFirst", + _ => "BitOrder.", + } + } +} + +/// Semantic interpretation of a bit stream (e.g. byte-packed or native binary tokens). +#[pyclass(name = "BitStreamSemantics", eq, from_py_object)] +#[derive(Clone, Copy, Eq, PartialEq, Debug)] +struct PyBitStreamSemantics { + inner: BitStreamSemantics, +} + +#[pymethods] +impl PyBitStreamSemantics { + #[staticmethod] + #[pyo3(signature = (order=None))] + fn byte_packed(order: Option<&Bound<'_, PyAny>>) -> PyResult { + let order = parse_bit_order_arg(order)?; + Ok(Self { + inner: BitStreamSemantics::BytePacked { order }, + }) + } + + #[staticmethod] + fn binary_tokens() -> Self { + Self { + inner: BitStreamSemantics::BinaryTokens, + } + } + + #[getter] + fn kind(&self) -> &'static str { + match self.inner { + BitStreamSemantics::BytePacked { .. } => "byte_packed", + BitStreamSemantics::BinaryTokens => "binary_tokens", + _ => "unknown", + } + } + + #[getter] + fn order(&self) -> Option { + match self.inner { + BitStreamSemantics::BytePacked { order } => Some(PyBitOrder { inner: order }), + BitStreamSemantics::BinaryTokens => None, + _ => None, + } + } + + fn __repr__(&self) -> String { + match self.inner { + BitStreamSemantics::BytePacked { order } => { + let order_repr = PyBitOrder { inner: order }.__repr__(); + format!("BitStreamSemantics.byte_packed(order={})", order_repr) + } + BitStreamSemantics::BinaryTokens => "BitStreamSemantics.binary_tokens()".to_string(), + _ => "BitStreamSemantics.".to_string(), + } + } +} + +fn parse_bit_order_value(py_obj: &Bound<'_, PyAny>) -> PyResult { + if let Ok(order) = py_obj.extract::>() { + return Ok(order.inner); + } + if let Ok(s) = py_obj.extract::() { + let s = s.trim(); + if s.eq_ignore_ascii_case("msb") + || s.eq_ignore_ascii_case("msbfirst") + || s.eq_ignore_ascii_case("msb_first") + { + return Ok(BitOrder::MsbFirst); + } else if s.eq_ignore_ascii_case("lsb") + || s.eq_ignore_ascii_case("lsbfirst") + || s.eq_ignore_ascii_case("lsb_first") + { + return Ok(BitOrder::LsbFirst); + } + return Err(PyValueError::new_err(format!( + "unknown BitOrder '{s}' (expected 'msb_first' or 'lsb_first')" + ))); + } + Err(PyValueError::new_err( + "BitOrder must be a BitOrder enum value or string alias", + )) +} + +fn parse_bit_order_arg(order: Option<&Bound<'_, PyAny>>) -> PyResult { + match order { + Some(value) => parse_bit_order_value(value), + None => Ok(BitOrder::MsbFirst), + } +} + +fn parse_bit_stream_semantics_value(py_obj: &Bound<'_, PyAny>) -> PyResult { + if let Ok(semantics) = py_obj.extract::>() { + return Ok(semantics.inner); + } + if let Ok(s) = py_obj.extract::() { + let s = s.trim(); + if s.eq_ignore_ascii_case("bytepacked") + || s.eq_ignore_ascii_case("byte_packed") + || s.eq_ignore_ascii_case("byte") + { + return Ok(BitStreamSemantics::BytePacked { + order: BitOrder::MsbFirst, + }); + } else if s.eq_ignore_ascii_case("binarytokens") + || s.eq_ignore_ascii_case("binary_tokens") + || s.eq_ignore_ascii_case("binary") + || s.eq_ignore_ascii_case("bit") + { + return Ok(BitStreamSemantics::BinaryTokens); + } + return Err(PyValueError::new_err(format!( + "unknown BitStreamSemantics '{s}' (expected 'byte_packed' or 'binary_tokens')" + ))); + } + Err(PyValueError::new_err( + "BitStreamSemantics must be a BitStreamSemantics instance or string alias", + )) +} + +/// Represents an exact, normalized binary probability distribution. +/// Invariants: `p0` and `p1` are finite, >= 0.0, and sum exactly to 1.0. +#[pyclass(name = "BinaryPrediction", eq, from_py_object)] +#[derive(Clone, Copy, PartialEq)] +struct PyBinaryPrediction { + #[pyo3(get)] + p0: f64, + #[pyo3(get)] + p1: f64, +} + +const BINARY_PREDICTION_SUM_TOLERANCE: f64 = f64::EPSILON * 4.0; + +impl From for PyBinaryPrediction { + fn from(inner: BinaryPrediction) -> Self { + Self { + p0: inner.p0, + p1: inner.p1, + } + } +} + +fn invalid_binary_prediction_prob_one(p1: f64) -> PyErr { + PyValueError::new_err(format!( + "Invalid binary prediction probability: p1={p1} (must be finite)" + )) +} + +#[pymethods] +impl PyBinaryPrediction { + #[new] + fn new(p0: f64, p1: f64) -> PyResult { + if !p0.is_finite() || !p1.is_finite() || p0 < 0.0 || p1 < 0.0 { + return Err(PyValueError::new_err(format!( + "Invalid binary prediction probabilities: p0={}, p1={} (must be finite and >= 0)", + p0, p1 + ))); + } + let sum = p0 + p1; + if (sum - 1.0).abs() > BINARY_PREDICTION_SUM_TOLERANCE { + return Err(PyValueError::new_err(format!( + "Invalid binary prediction probabilities: p0={}, p1={} (must sum to 1)", + p0, p1 + ))); + } + // Canonicalize every accepted pair through the core type. Even when + // `p0 + p1` rounds to exactly `1.0`, floating-point addition does not + // guarantee that `p0 == 1.0 - p1`. + BinaryPrediction::checked_from_prob_one_exact(p1 / sum) + .map(Into::into) + .ok_or_else(|| invalid_binary_prediction_prob_one(p1)) + } + + #[staticmethod] + #[pyo3(signature = (p1, floor=None))] + fn from_prob_one(p1: f64, floor: Option) -> PyResult { + let floor = floor.unwrap_or(0.0); + BinaryPrediction::checked_from_prob_one(p1, floor) + .map(Into::into) + .ok_or_else(|| invalid_binary_prediction_prob_one(p1)) + } + + #[staticmethod] + fn from_prob_one_exact(p1: f64) -> PyResult { + BinaryPrediction::checked_from_prob_one_exact(p1) + .map(Into::into) + .ok_or_else(|| invalid_binary_prediction_prob_one(p1)) + } + + fn prob(&self, bit: bool) -> f64 { + if bit { self.p1 } else { self.p0 } + } + + fn __repr__(&self) -> String { + format!("BinaryPrediction(p0={}, p1={})", self.p0, self.p1) + } +} + +/// A live prefix-mass state used to factorize byte probabilities into bit probabilities. +#[pyclass(name = "BytePrefixMass", from_py_object)] +#[derive(Clone)] +struct PyBytePrefixMass { + inner: BytePrefixMass, +} + +const BYTE_PREFIX_MASS_WIDTH: usize = 256; + +impl From for PyBytePrefixMass { + fn from(inner: BytePrefixMass) -> Self { + Self { inner } + } +} + +fn validate_byte_prefix_mass_len(values_len: usize, constructor: &'static str) -> PyResult<()> { + if values_len != BYTE_PREFIX_MASS_WIDTH { + return Err(PyValueError::new_err(format!( + "BytePrefixMass.{constructor} expects exactly {BYTE_PREFIX_MASS_WIDTH} entries, got {values_len}" + ))); + } + Ok(()) +} + +#[pymethods] +impl PyBytePrefixMass { + #[staticmethod] + #[pyo3(signature = (pdf, order=None))] + fn from_pdf(pdf: Vec, order: Option<&Bound<'_, PyAny>>) -> PyResult { + validate_byte_prefix_mass_len(pdf.len(), "from_pdf")?; + Ok(BytePrefixMass::from_pdf(&pdf, parse_bit_order_arg(order)?).into()) + } + + #[staticmethod] + #[pyo3(signature = (log_probs, order=None))] + fn from_log_probs(log_probs: Vec, order: Option<&Bound<'_, PyAny>>) -> PyResult { + validate_byte_prefix_mass_len(log_probs.len(), "from_log_probs")?; + Ok(BytePrefixMass::from_log_probs(&log_probs, parse_bit_order_arg(order)?).into()) + } + + fn prediction(&self) -> PyBinaryPrediction { + self.inner.prediction().into() + } + + fn observe(&mut self, bit: bool) { + self.inner.observe(bit); + } + + fn is_complete(&self) -> bool { + self.inner.is_complete() + } + + fn has_partial_bits(&self) -> bool { + self.inner.has_partial_bits() + } + + fn symbol(&self) -> PyResult { + if !self.inner.is_complete() { + return Err(PyRuntimeError::new_err( + "BytePrefixMass.symbol() is only meaningful after a full byte has been observed", + )); + } + Ok(self.inner.symbol()) + } + + fn __repr__(&self) -> String { + if self.inner.is_complete() { + return format!( + "BytePrefixMass(complete=True, symbol={})", + self.inner.symbol() + ); + } + format!( + "BytePrefixMass(complete=False, has_partial_bits={})", + self.inner.has_partial_bits() + ) + } +} + +/// A predictive session for bit streams. +/// Supports both native binary tokens and byte-packed bit factorizations. +#[pyclass(name = "RateBackendBitSession", from_py_object)] +#[derive(Clone)] +struct PyRateBackendBitSession { + inner: Arc>, + semantics: BitStreamSemantics, +} + +/// Opaque checkpoint for exact `RateBackendBitSession` restoration. +#[pyclass(name = "RateBackendBitSessionCheckpoint", from_py_object)] +#[derive(Clone)] +struct PyRateBackendBitSessionCheckpoint { + inner: Arc>, +} + +#[pymethods] +impl PyRateBackendBitSession { + #[new] + #[pyo3(signature = (backend, total_bits=None, semantics=None))] + fn new( + backend: &PyRateBackend, + total_bits: Option, + semantics: Option<&Bound<'_, PyAny>>, + ) -> PyResult { + py_try(|| { + let sem = match semantics { + Some(s) => parse_bit_stream_semantics_value(s)?, + None => BitStreamSemantics::default(), + }; + let inner = RateBackendBitSession::from_backend( + compile_rate_backend(backend.inner.clone())?, + total_bits, + sem, + ) + .map_err(py_infotheory_error)?; + Ok(Self { + inner: Arc::new(Mutex::new(inner)), + semantics: sem, + }) + }) + } + + fn predict_bit(&self) -> PyBinaryPrediction { + lock_recover(&self.inner).predict_bit().into() + } + + fn predict_one(&self) -> f64 { + lock_recover(&self.inner).predict_one() + } + + fn checkpoint(&self) -> PyRateBackendBitSessionCheckpoint { + PyRateBackendBitSessionCheckpoint { + inner: Arc::new(Mutex::new(lock_recover(&self.inner).checkpoint())), + } + } + + fn restore_checkpoint(&self, checkpoint: &PyRateBackendBitSessionCheckpoint) -> PyResult<()> { + let checkpoint = lock_recover(&checkpoint.inner); + lock_recover(&self.inner) + .restore_checkpoint(&checkpoint) + .map_err(py_infotheory_error) + } + + fn clear_checkpoints_if_supported(&self) { + lock_recover(&self.inner).clear_checkpoints_if_supported(); + } + + fn step_bit(&self, bit: bool) -> PyResult { + lock_recover(&self.inner) + .try_step_bit(bit) + .map(Into::into) + .map_err(py_infotheory_error) + } + + fn observe_bit(&self, bit: bool) -> PyResult<()> { + lock_recover(&self.inner) + .try_observe_bit(bit) + .map_err(py_infotheory_error) + } + + fn condition_bit(&self, bit: bool) -> PyResult<()> { + lock_recover(&self.inner) + .try_condition_bit(bit) + .map_err(py_infotheory_error) + } + + #[pyo3(signature = (total_bits=None))] + fn reset_frozen(&self, total_bits: Option) -> PyResult<()> { + lock_recover(&self.inner) + .reset_frozen(total_bits) + .map_err(py_infotheory_error) + } + + #[pyo3(signature = (total_bits=None, semantics=None))] + fn begin_bit_stream( + &self, + total_bits: Option, + semantics: Option<&Bound<'_, PyAny>>, + ) -> PyResult<()> { + let sem = match semantics { + Some(s) => parse_bit_stream_semantics_value(s)?, + None => self.semantics, + }; + lock_recover(&self.inner) + .begin_bit_stream(total_bits, sem) + .map_err(|err| py_infotheory_error(InfotheoryError::runtime(err))) + } + + fn finish(&self) -> PyResult<()> { + lock_recover(&self.inner) + .finish() + .map_err(py_infotheory_error) + } +} + +#[pymethods] +impl PyRateBackendBitSessionCheckpoint { + fn __repr__(&self) -> String { + "RateBackendBitSessionCheckpoint(...)".to_owned() } } +#[pyfunction] +fn get_default_ctx() -> PyResult { + Ok(PyInfotheoryCtx { + inner: api::get_default_ctx().map_err(py_infotheory_error)?, + }) +} + #[pyfunction] fn set_default_ctx(ctx: &PyInfotheoryCtx) { - infotheory::set_default_ctx(ctx.inner.clone()); + api::set_default_ctx(ctx.inner.clone()); +} + +fn ncd_default_zpaq_bytes(x: &[u8], y: &[u8]) -> f64 { + let Ok(cb) = compile_compression_backend(CompressionBackend::zpaq("5")) else { + return f64::NAN; + }; + api::try_ncd_bytes_backend(x, y, &cb, NcdVariant::Vitanyi).unwrap_or(f64::NAN) } #[pyfunction] #[pyo3(signature = (x, y, tolerance=1e-9))] fn verify_identity(x: &[u8], y: &[u8], tolerance: f64) -> bool { - infotheory::axioms::verify_identity( - |a, b| infotheory::ncd_bytes(a, b, "5", NcdVariant::Vitanyi), - x, - tolerance, - ) && infotheory::axioms::verify_identity( - |a, b| infotheory::ncd_bytes(a, b, "5", NcdVariant::Vitanyi), - y, - tolerance, - ) + infotheory::axioms::verify_identity(ncd_default_zpaq_bytes, x, tolerance) + && infotheory::axioms::verify_identity(ncd_default_zpaq_bytes, y, tolerance) } #[pyfunction] #[pyo3(signature = (x, y, tolerance=1e-9))] fn verify_symmetry(x: &[u8], y: &[u8], tolerance: f64) -> bool { - infotheory::axioms::verify_symmetry( - |a, b| infotheory::ncd_bytes(a, b, "5", NcdVariant::Vitanyi), - x, - y, - tolerance, - ) + infotheory::axioms::verify_symmetry(ncd_default_zpaq_bytes, x, y, tolerance) } #[pyfunction] #[pyo3(signature = (x, y, z, tolerance=1e-9))] fn verify_triangle_inequality(x: &[u8], y: &[u8], z: &[u8], tolerance: f64) -> bool { - infotheory::axioms::verify_triangle_inequality( - |a, b| infotheory::ncd_bytes(a, b, "5", NcdVariant::Vitanyi), - x, - y, - z, - tolerance, - ) + infotheory::axioms::verify_triangle_inequality(ncd_default_zpaq_bytes, x, y, z, tolerance) } #[pyfunction] fn verify_non_negativity(x: &[u8], y: &[u8]) -> bool { - infotheory::axioms::verify_non_negativity( - |a, b| infotheory::ncd_bytes(a, b, "5", NcdVariant::Vitanyi), - x, - y, - ) + infotheory::axioms::verify_non_negativity(ncd_default_zpaq_bytes, x, y) } #[pyfunction] fn verify_mi_nonnegative(x: &[u8], y: &[u8]) -> bool { - infotheory::axioms::verify_mi_nonnegative(infotheory::mutual_information_marg_bytes, x, y) + infotheory::axioms::verify_mi_nonnegative(api::empirical_mutual_information_bytes, x, y) } #[pyfunction] #[pyo3(signature = (x, y, tolerance=1e-9))] fn verify_subadditivity(x: &[u8], y: &[u8], tolerance: f64) -> bool { infotheory::axioms::verify_subadditivity( - infotheory::joint_marginal_entropy_bytes, - infotheory::marginal_entropy_bytes, + api::empirical_joint_entropy_bytes, + api::empirical_entropy_bytes, x, y, tolerance, @@ -1992,8 +1860,8 @@ fn verify_subadditivity(x: &[u8], y: &[u8], tolerance: f64) -> bool { #[pyo3(signature = (x, y, tolerance=1e-9))] fn verify_conditioning_reduces_entropy(x: &[u8], y: &[u8], tolerance: f64) -> bool { infotheory::axioms::verify_conditioning_reduces_entropy( - |a, b| infotheory::conditional_entropy_bytes(a, b, 6), - infotheory::marginal_entropy_bytes, + |a, b| api::try_conditional_entropy_bytes(a, b).unwrap_or(f64::NAN), + api::empirical_entropy_bytes, x, y, tolerance, @@ -2004,9 +1872,9 @@ fn verify_conditioning_reduces_entropy(x: &[u8], y: &[u8], tolerance: f64) -> bo #[pyo3(signature = (x, y, tolerance=1e-9))] fn verify_chain_rule(x: &[u8], y: &[u8], tolerance: f64) -> bool { infotheory::axioms::verify_chain_rule( - |a, b| infotheory::joint_entropy_rate_bytes(a, b, 6), - |a| infotheory::entropy_rate_bytes(a, 6), - |a, b| infotheory::conditional_entropy_rate_bytes(a, b, 6), + |a, b| api::try_joint_entropy_rate_bytes(a, b).unwrap_or(f64::NAN), + |a| api::try_entropy_rate_bytes(a).unwrap_or(f64::NAN), + |a, b| api::try_conditional_entropy_rate_bytes(a, b).unwrap_or(f64::NAN), x, y, tolerance, @@ -2015,16 +1883,12 @@ fn verify_chain_rule(x: &[u8], y: &[u8], tolerance: f64) -> bool { #[pyfunction] fn verify_ncd_bounds(x: &[u8], y: &[u8]) -> bool { - infotheory::axioms::verify_ncd_bounds( - |a, b| infotheory::ncd_bytes(a, b, "5", NcdVariant::Vitanyi), - x, - y, - ) + infotheory::axioms::verify_ncd_bounds(ncd_default_zpaq_bytes, x, y) } #[pyfunction] fn verify_entropy_bounds(data: &[u8]) -> bool { - infotheory::axioms::verify_entropy_bounds(infotheory::marginal_entropy_bytes, data) + infotheory::axioms::verify_entropy_bounds(api::empirical_entropy_bytes, data) } fn compression_backend_from_py( @@ -2047,18 +1911,15 @@ fn compression_backend_from_py( } fn file_roundtrip_backend(backend: &CompressionBackend) -> CompressionBackend { - match backend { - CompressionBackend::Rate { - rate_backend, - coder, - .. - } => CompressionBackend::Rate { - rate_backend: rate_backend.clone(), - coder: *coder, - framing: infotheory::compression::FramingMode::Framed, - }, - _ => backend.clone(), - } + infotheory::backends::normalize_file_roundtrip_backend(backend) +} + +fn compiled_compression_backend_from_py( + backend: Option<&Bound<'_, PyAny>>, + method: Option<&str>, + rate_backend: Option, +) -> PyResult { + compile_compression_backend(compression_backend_from_py(backend, method, rate_backend)?) } fn rate_backend_from_py( @@ -2076,7 +1937,14 @@ fn rate_backend_from_py( "rate backend must be RateBackend or string", )); } - Ok(RateBackend::default()) + RateBackend::try_default().map_err(py_infotheory_error) +} + +fn compiled_rate_backend_from_py( + backend: Option<&Bound<'_, PyAny>>, + method: Option<&str>, +) -> PyResult { + compile_rate_backend(rate_backend_from_py(backend, method)?) } #[pyfunction] @@ -2091,7 +1959,7 @@ fn ncd_paths( ) -> PyResult { let v = parse_ncd_variant(variant)?; let cb = compression_backend_from_py(backend, Some(method), None)?; - py.detach(|| py_try(|| Ok(infotheory::ncd_paths_backend(x, y, &cb, v)))) + py.detach(|| py_try(|| api::try_ncd_paths_backend(x, y, &cb, v).map_err(py_infotheory_error))) } #[pyfunction] @@ -2105,8 +1973,8 @@ fn ncd_bytes( backend: Option<&Bound<'_, PyAny>>, ) -> PyResult { let v = parse_ncd_variant(variant)?; - let cb = compression_backend_from_py(backend, Some(method), None)?; - py.detach(|| py_try(|| Ok(infotheory::ncd_bytes_backend(x, y, &cb, v)))) + let cb = compiled_compression_backend_from_py(backend, Some(method), None)?; + py.detach(|| py_try(|| api::try_ncd_bytes_backend(x, y, &cb, v).map_err(py_infotheory_error))) } #[pyfunction] @@ -2121,8 +1989,7 @@ fn ncd_paths_with_backend( ) -> PyResult { let v = parse_ncd_variant(variant)?; let cb = compression_backend_from_py(backend, method, None)?; - - py.detach(|| py_try(|| Ok(infotheory::ncd_paths_backend(x, y, &cb, v)))) + py.detach(|| py_try(|| api::try_ncd_paths_backend(x, y, &cb, v).map_err(py_infotheory_error))) } #[pyfunction] @@ -2136,206 +2003,180 @@ fn ncd_bytes_with_backend( variant: &str, ) -> PyResult { let v = parse_ncd_variant(variant)?; - let cb = compression_backend_from_py(backend, method, None)?; - py.detach(|| py_try(|| Ok(infotheory::ncd_bytes_backend(x, y, &cb, v)))) + let cb = compiled_compression_backend_from_py(backend, method, None)?; + py.detach(|| py_try(|| api::try_ncd_bytes_backend(x, y, &cb, v).map_err(py_infotheory_error))) } #[pyfunction] #[pyo3(signature = (x, y, variant="vitanyi"))] fn ncd_bytes_default(py: Python<'_>, x: &[u8], y: &[u8], variant: &str) -> PyResult { let v = parse_ncd_variant(variant)?; - py.detach(|| py_try(|| Ok(infotheory::ncd_bytes_default(x, y, v)))) + py.detach(|| py_try(|| api::try_ncd_bytes_default(x, y, v).map_err(py_infotheory_error))) } #[pyfunction] -fn entropy_rate_bytes(py: Python<'_>, data: &[u8], max_order: i64) -> PyResult { - py.detach(|| py_try(|| Ok(infotheory::entropy_rate_bytes(data, max_order)))) +fn entropy_rate_bytes(py: Python<'_>, data: &[u8]) -> PyResult { + py.detach(|| py_try(|| api::try_entropy_rate_bytes(data).map_err(py_infotheory_error))) } #[pyfunction] -fn biased_entropy_rate_bytes(py: Python<'_>, data: &[u8], max_order: i64) -> PyResult { - py.detach(|| py_try(|| Ok(infotheory::biased_entropy_rate_bytes(data, max_order)))) +fn biased_entropy_rate_bytes(py: Python<'_>, data: &[u8]) -> PyResult { + py.detach(|| py_try(|| api::try_biased_entropy_rate_bytes(data).map_err(py_infotheory_error))) } #[pyfunction] -fn marginal_entropy_bytes(data: &[u8]) -> f64 { - infotheory::marginal_entropy_bytes(data) +fn empirical_entropy_bytes(data: &[u8]) -> f64 { + api::empirical_entropy_bytes(data) } #[pyfunction] -fn joint_marginal_entropy_bytes(x: &[u8], y: &[u8]) -> f64 { - infotheory::joint_marginal_entropy_bytes(x, y) +fn empirical_joint_entropy_bytes(x: &[u8], y: &[u8]) -> f64 { + api::empirical_joint_entropy_bytes(x, y) } -macro_rules! py_metric_bytes_3 { +macro_rules! py_metric_bytes_2 { ($fn_name:ident, $target:path) => { #[pyfunction] - fn $fn_name(py: Python<'_>, x: &[u8], y: &[u8], max_order: i64) -> PyResult { - py.detach(|| py_try(|| Ok($target(x, y, max_order)))) + fn $fn_name(py: Python<'_>, x: &[u8], y: &[u8]) -> PyResult { + py.detach(|| py_try(|| Ok($target(x, y)))) } }; } -macro_rules! py_metric_paths_3 { +macro_rules! py_metric_bytes_2_try { ($fn_name:ident, $target:path) => { #[pyfunction] - fn $fn_name(py: Python<'_>, x: &str, y: &str, max_order: i64) -> PyResult { - py.detach(|| py_try(|| Ok($target(x, y, max_order)))) + fn $fn_name(py: Python<'_>, x: &[u8], y: &[u8]) -> PyResult { + py.detach(|| py_try(|| $target(x, y).map_err(py_infotheory_error))) } }; } -py_metric_bytes_3!( - joint_entropy_rate_bytes, - infotheory::joint_entropy_rate_bytes -); -py_metric_bytes_3!( +macro_rules! py_metric_paths_2_try { + ($fn_name:ident, $target:path) => { + #[pyfunction] + fn $fn_name(py: Python<'_>, x: &str, y: &str) -> PyResult { + py.detach(|| { + py_try(|| $target(x, y).map_err(|e| PyRuntimeError::new_err(e.to_string()))) + }) + } + }; +} + +py_metric_bytes_2_try!(joint_entropy_rate_bytes, api::try_joint_entropy_rate_bytes); +py_metric_bytes_2_try!( conditional_entropy_rate_bytes, - infotheory::conditional_entropy_rate_bytes + api::try_conditional_entropy_rate_bytes ); -py_metric_bytes_3!( +py_metric_bytes_2_try!( conditional_entropy_bytes, - infotheory::conditional_entropy_bytes -); -py_metric_bytes_3!( - mutual_information_bytes, - infotheory::mutual_information_bytes + api::try_conditional_entropy_bytes ); -py_metric_bytes_3!( +py_metric_bytes_2_try!(mutual_information_bytes, api::try_mutual_information_bytes); +py_metric_bytes_2_try!( mutual_information_rate_bytes, - infotheory::mutual_information_rate_bytes + api::try_mutual_information_rate_bytes ); -py_metric_bytes_3!(ned_bytes, infotheory::ned_bytes); -py_metric_bytes_3!(nte_bytes, infotheory::nte_bytes); -py_metric_bytes_3!(tvd_bytes, infotheory::tvd_bytes); -py_metric_bytes_3!(nhd_bytes, infotheory::nhd_bytes); -py_metric_bytes_3!(cross_entropy_bytes, infotheory::cross_entropy_bytes); -py_metric_bytes_3!( - cross_entropy_rate_bytes, - infotheory::cross_entropy_rate_bytes -); - -py_metric_paths_3!(ned_paths, infotheory::ned_paths); -py_metric_paths_3!(nte_paths, infotheory::nte_paths); -py_metric_paths_3!(tvd_paths, infotheory::tvd_paths); -py_metric_paths_3!(nhd_paths, infotheory::nhd_paths); -py_metric_paths_3!( - mutual_information_paths, - infotheory::mutual_information_paths -); -py_metric_paths_3!( +py_metric_bytes_2_try!(ned_bytes, api::try_ned_bytes); +py_metric_bytes_2_try!(nte_bytes, api::try_nte_bytes); +py_metric_bytes_2!(tvd_bytes, api::tvd_bytes); +py_metric_bytes_2!(nhd_bytes, api::nhd_bytes); +py_metric_bytes_2_try!(cross_entropy_bytes, api::try_cross_entropy_bytes); +py_metric_bytes_2_try!(cross_entropy_rate_bytes, api::try_cross_entropy_rate_bytes); + +py_metric_paths_2_try!(ned_paths, api::try_ned_paths); +py_metric_paths_2_try!(nte_paths, api::try_nte_paths); +py_metric_paths_2_try!(tvd_paths, api::try_tvd_paths); +py_metric_paths_2_try!(nhd_paths, api::try_nhd_paths); +py_metric_paths_2_try!(mutual_information_paths, api::try_mutual_information_paths); +py_metric_paths_2_try!( conditional_entropy_paths, - infotheory::conditional_entropy_paths + api::try_conditional_entropy_paths ); -py_metric_paths_3!(cross_entropy_paths, infotheory::cross_entropy_paths); +py_metric_paths_2_try!(cross_entropy_paths, api::try_cross_entropy_paths); #[pyfunction] fn d_kl_bytes(x: &[u8], y: &[u8]) -> f64 { - infotheory::d_kl_bytes(x, y) + api::d_kl_bytes(x, y) } #[pyfunction] fn js_div_bytes(x: &[u8], y: &[u8]) -> f64 { - infotheory::js_div_bytes(x, y) + api::js_div_bytes(x, y) } #[pyfunction] fn kl_divergence_paths(py: Python<'_>, x: &str, y: &str) -> PyResult { - py.detach(|| py_try(|| Ok(infotheory::kl_divergence_paths(x, y)))) + py.detach(|| py_try(|| api::try_kl_divergence_paths(x, y).map_err(py_infotheory_error))) } #[pyfunction] fn js_divergence_paths(py: Python<'_>, x: &str, y: &str) -> PyResult { - py.detach(|| py_try(|| Ok(infotheory::js_divergence_paths(x, y)))) + py.detach(|| py_try(|| api::try_js_divergence_paths(x, y).map_err(py_infotheory_error))) } #[pyfunction] -fn intrinsic_dependence_bytes(py: Python<'_>, data: &[u8], max_order: i64) -> PyResult { - py.detach(|| py_try(|| Ok(infotheory::intrinsic_dependence_bytes(data, max_order)))) +fn intrinsic_dependence_bytes(py: Python<'_>, data: &[u8]) -> PyResult { + py.detach(|| py_try(|| api::try_intrinsic_dependence_bytes(data).map_err(py_infotheory_error))) } #[pyfunction] -fn resistance_to_transformation_bytes( - py: Python<'_>, - x: &[u8], - tx: &[u8], - max_order: i64, -) -> PyResult { +fn resistance_to_transformation_bytes(py: Python<'_>, x: &[u8], tx: &[u8]) -> PyResult { py.detach(|| { - py_try(|| { - Ok(infotheory::resistance_to_transformation_bytes( - x, tx, max_order, - )) - }) + py_try(|| api::try_resistance_to_transformation_bytes(x, tx).map_err(py_infotheory_error)) }) } #[pyfunction] -fn mutual_information_marg_bytes(x: &[u8], y: &[u8]) -> f64 { - infotheory::mutual_information_marg_bytes(x, y) +fn empirical_mutual_information_bytes(x: &[u8], y: &[u8]) -> f64 { + api::empirical_mutual_information_bytes(x, y) } #[pyfunction] -fn ned_marg_bytes(x: &[u8], y: &[u8]) -> f64 { - infotheory::ned_marg_bytes(x, y) +fn empirical_ned_bytes(x: &[u8], y: &[u8]) -> f64 { + api::empirical_ned_bytes(x, y) } #[pyfunction] -fn ned_rate_bytes(py: Python<'_>, x: &[u8], y: &[u8], max_order: i64) -> PyResult { - py.detach(|| py_try(|| Ok(infotheory::ned_rate_bytes(x, y, max_order)))) +fn ned_rate_bytes(py: Python<'_>, x: &[u8], y: &[u8]) -> PyResult { + py.detach(|| py_try(|| api::try_ned_rate_bytes(x, y).map_err(py_infotheory_error))) } #[pyfunction] -fn ned_cons_bytes(py: Python<'_>, x: &[u8], y: &[u8], max_order: i64) -> PyResult { - py.detach(|| py_try(|| Ok(infotheory::ned_cons_bytes(x, y, max_order)))) +fn ned_cons_bytes(py: Python<'_>, x: &[u8], y: &[u8]) -> PyResult { + py.detach(|| py_try(|| api::try_ned_cons_bytes(x, y).map_err(py_infotheory_error))) } #[pyfunction] -fn ned_cons_marg_bytes(x: &[u8], y: &[u8]) -> f64 { - infotheory::ned_cons_marg_bytes(x, y) +fn empirical_ned_cons_bytes(x: &[u8], y: &[u8]) -> f64 { + api::empirical_ned_cons_bytes(x, y) } #[pyfunction] -fn ned_cons_rate_bytes(py: Python<'_>, x: &[u8], y: &[u8], max_order: i64) -> PyResult { - py.detach(|| py_try(|| Ok(infotheory::ned_cons_rate_bytes(x, y, max_order)))) +fn ned_cons_rate_bytes(py: Python<'_>, x: &[u8], y: &[u8]) -> PyResult { + py.detach(|| py_try(|| api::try_ned_cons_rate_bytes(x, y).map_err(py_infotheory_error))) } #[pyfunction] -fn nte_marg_bytes(x: &[u8], y: &[u8]) -> f64 { - infotheory::nte_marg_bytes(x, y) +fn empirical_nte_bytes(x: &[u8], y: &[u8]) -> f64 { + api::empirical_nte_bytes(x, y) } #[pyfunction] -fn nte_rate_bytes(py: Python<'_>, x: &[u8], y: &[u8], max_order: i64) -> PyResult { - py.detach(|| py_try(|| Ok(infotheory::nte_rate_bytes(x, y, max_order)))) +fn nte_rate_bytes(py: Python<'_>, x: &[u8], y: &[u8]) -> PyResult { + py.detach(|| py_try(|| api::try_nte_rate_bytes(x, y).map_err(py_infotheory_error))) } #[pyfunction] fn validate_zpaq_rate_method(method: &str) -> PyResult<()> { - match infotheory::validate_zpaq_rate_method(method) { - Ok(()) => Ok(()), - Err(e) => Err(PyValueError::new_err(e)), - } + infotheory::validate_zpaq_rate_method(method).map_err(py_infotheory_error) } #[pyfunction] fn get_compressed_size(py: Python<'_>, path: &str, method: &str) -> PyResult { - py.detach(|| py_try(|| Ok(infotheory::get_compressed_size(path, method)))) -} - -#[pyfunction] -fn get_compressed_size_parallel( - py: Python<'_>, - path: &str, - method: &str, - threads: usize, -) -> PyResult { + let cb = compile_compression_backend(CompressionBackend::zpaq(method))?; py.detach(|| { - py_try(|| { - Ok(infotheory::get_compressed_size_parallel( - path, method, threads, - )) - }) + py_try(|| api::try_get_compressed_size_path_backend(path, &cb).map_err(py_infotheory_error)) }) } @@ -2347,107 +2188,64 @@ fn get_compressed_sizes_from_paths( method: &str, ) -> PyResult> { let refs: Vec<&str> = paths.iter().map(String::as_str).collect(); - py.detach(|| py_try(|| Ok(infotheory::get_compressed_sizes_from_paths(&refs, method)))) -} - -#[pyfunction] -#[pyo3(signature = (paths, method="5"))] -fn get_sequential_compressed_sizes_from_sequential_paths( - py: Python<'_>, - paths: Vec, - method: &str, -) -> PyResult> { - let refs: Vec<&str> = paths.iter().map(String::as_str).collect(); + let cb = compile_compression_backend(CompressionBackend::zpaq(method))?; py.detach(|| { py_try(|| { - Ok(infotheory::get_sequential_compressed_sizes_from_sequential_paths(&refs, method)) + api::try_get_compressed_sizes_from_paths_backend(&refs, &cb) + .map_err(py_infotheory_error) }) }) } #[pyfunction] -#[pyo3(signature = (paths, method="5", threads=1))] -fn get_parallel_compressed_sizes_from_sequential_paths( - py: Python<'_>, - paths: Vec, - method: &str, - threads: usize, -) -> PyResult> { +fn get_bytes_from_paths(py: Python<'_>, paths: Vec) -> PyResult>> { let refs: Vec<&str> = paths.iter().map(String::as_str).collect(); - py.detach(|| { - py_try(|| { - Ok( - infotheory::get_parallel_compressed_sizes_from_sequential_paths( - &refs, method, threads, - ), - ) - }) - }) + py.detach(|| py_try(|| api::try_get_bytes_from_paths(&refs).map_err(py_infotheory_error))) } #[pyfunction] -#[pyo3(signature = (paths, method="5"))] -fn get_sequential_compressed_sizes_from_parallel_paths( - py: Python<'_>, - paths: Vec, - method: &str, -) -> PyResult> { - let refs: Vec<&str> = paths.iter().map(String::as_str).collect(); +#[pyo3(signature = (x, y, method="5"))] +fn ncd_vitanyi(py: Python<'_>, x: &str, y: &str, method: &str) -> PyResult { + let cb = CompressionBackend::zpaq(method); py.detach(|| { py_try(|| { - Ok(infotheory::get_sequential_compressed_sizes_from_parallel_paths(&refs, method)) + api::try_ncd_paths_backend(x, y, &cb, NcdVariant::Vitanyi).map_err(py_infotheory_error) }) }) } #[pyfunction] -#[pyo3(signature = (paths, method="5", threads=1))] -fn get_parallel_compressed_sizes_from_parallel_paths( - py: Python<'_>, - paths: Vec, - method: &str, - threads: usize, -) -> PyResult> { - let refs: Vec<&str> = paths.iter().map(String::as_str).collect(); +#[pyo3(signature = (x, y, method="5"))] +fn ncd_sym_vitanyi(py: Python<'_>, x: &str, y: &str, method: &str) -> PyResult { + let cb = CompressionBackend::zpaq(method); py.detach(|| { py_try(|| { - Ok( - infotheory::get_parallel_compressed_sizes_from_parallel_paths( - &refs, method, threads, - ), - ) + api::try_ncd_paths_backend(x, y, &cb, NcdVariant::SymVitanyi) + .map_err(py_infotheory_error) }) }) } -#[pyfunction] -fn get_bytes_from_paths(py: Python<'_>, paths: Vec) -> PyResult>> { - let refs: Vec<&str> = paths.iter().map(String::as_str).collect(); - py.detach(|| py_try(|| Ok(infotheory::get_bytes_from_paths(&refs)))) -} - -#[pyfunction] -#[pyo3(signature = (x, y, method="5"))] -fn ncd_vitanyi(py: Python<'_>, x: &str, y: &str, method: &str) -> PyResult { - py.detach(|| py_try(|| Ok(infotheory::ncd_vitanyi(x, y, method)))) -} - -#[pyfunction] -#[pyo3(signature = (x, y, method="5"))] -fn ncd_sym_vitanyi(py: Python<'_>, x: &str, y: &str, method: &str) -> PyResult { - py.detach(|| py_try(|| Ok(infotheory::ncd_sym_vitanyi(x, y, method)))) -} - #[pyfunction] #[pyo3(signature = (x, y, method="5"))] fn ncd_cons(py: Python<'_>, x: &str, y: &str, method: &str) -> PyResult { - py.detach(|| py_try(|| Ok(infotheory::ncd_cons(x, y, method)))) + let cb = CompressionBackend::zpaq(method); + py.detach(|| { + py_try(|| { + api::try_ncd_paths_backend(x, y, &cb, NcdVariant::Cons).map_err(py_infotheory_error) + }) + }) } #[pyfunction] #[pyo3(signature = (x, y, method="5"))] fn ncd_sym_cons(py: Python<'_>, x: &str, y: &str, method: &str) -> PyResult { - py.detach(|| py_try(|| Ok(infotheory::ncd_sym_cons(x, y, method)))) + let cb = CompressionBackend::zpaq(method); + py.detach(|| { + py_try(|| { + api::try_ncd_paths_backend(x, y, &cb, NcdVariant::SymCons).map_err(py_infotheory_error) + }) + }) } #[pyfunction] @@ -2461,8 +2259,13 @@ fn compress_size_backend( rate_method: Option<&str>, ) -> PyResult { let rb = rate_backend_from_py(rate_backend, rate_method)?; - let cb = compression_backend_from_py(compression_backend, Some(method), Some(rb))?; - py.detach(|| py_try(|| Ok(infotheory::compress_size_backend(data, &cb)))) + let cb = compiled_compression_backend_from_py(compression_backend, Some(method), Some(rb))?; + py.detach(|| { + py_try(|| { + infotheory::api::try_compress_size_backend(data, &cb) + .map_err(|e| PyRuntimeError::new_err(e.to_string())) + }) + }) } #[pyfunction] @@ -2476,9 +2279,14 @@ fn compress_size_chain_backend( rate_method: Option<&str>, ) -> PyResult { let rb = rate_backend_from_py(rate_backend, rate_method)?; - let cb = compression_backend_from_py(compression_backend, Some(method), Some(rb))?; + let cb = compiled_compression_backend_from_py(compression_backend, Some(method), Some(rb))?; let refs: Vec<&[u8]> = parts.iter().map(Vec::as_slice).collect(); - py.detach(|| py_try(|| Ok(infotheory::compress_size_chain_backend(&refs, &cb)))) + py.detach(|| { + py_try(|| { + infotheory::api::try_compress_size_chain_backend(&refs, &cb) + .map_err(|e| PyRuntimeError::new_err(e.to_string())) + }) + }) } #[pyfunction] @@ -2492,11 +2300,10 @@ fn compress_bytes_backend<'py>( rate_method: Option<&str>, ) -> PyResult> { let rb = rate_backend_from_py(rate_backend, rate_method)?; - let cb = compression_backend_from_py(compression_backend, Some(method), Some(rb))?; + let cb = compiled_compression_backend_from_py(compression_backend, Some(method), Some(rb))?; let out = py.detach(|| { py_try(|| { - infotheory::compress_bytes_backend(data, &cb) - .map_err(|e| PyRuntimeError::new_err(e.to_string())) + infotheory::api::try_compress_bytes_backend(data, &cb).map_err(py_infotheory_error) }) })?; Ok(PyBytes::new(py, &out)) @@ -2513,11 +2320,10 @@ fn decompress_bytes_backend<'py>( rate_method: Option<&str>, ) -> PyResult> { let rb = rate_backend_from_py(rate_backend, rate_method)?; - let cb = compression_backend_from_py(compression_backend, Some(method), Some(rb))?; + let cb = compiled_compression_backend_from_py(compression_backend, Some(method), Some(rb))?; let out = py.detach(|| { py_try(|| { - infotheory::decompress_bytes_backend(input, &cb) - .map_err(|e| PyRuntimeError::new_err(e.to_string())) + infotheory::api::try_decompress_bytes_backend(input, &cb).map_err(py_infotheory_error) }) })?; Ok(PyBytes::new(py, &out)) @@ -2535,15 +2341,19 @@ fn compress_file( rate_method: Option<&str>, ) -> PyResult<()> { let rb = rate_backend_from_py(rate_backend, rate_method)?; - let cb = compression_backend_from_py(compression_backend, Some(method), Some(rb))?; - let cb = file_roundtrip_backend(&cb); + let cb = file_roundtrip_backend(&compression_backend_from_py( + compression_backend, + Some(method), + Some(rb), + )?); + let cb = compile_compression_backend(cb)?; py.detach(|| { py_try(|| { let input = std::fs::read(input_path).map_err(|e| { PyRuntimeError::new_err(format!("failed to read '{input_path}': {e}")) })?; - let out = infotheory::compress_bytes_backend(&input, &cb) - .map_err(|e| PyRuntimeError::new_err(e.to_string()))?; + let out = infotheory::api::try_compress_bytes_backend(&input, &cb) + .map_err(py_infotheory_error)?; std::fs::write(output_path, &out).map_err(|e| { PyRuntimeError::new_err(format!("failed to write '{output_path}': {e}")) })?; @@ -2564,15 +2374,19 @@ fn decompress_file( rate_method: Option<&str>, ) -> PyResult<()> { let rb = rate_backend_from_py(rate_backend, rate_method)?; - let cb = compression_backend_from_py(compression_backend, Some(method), Some(rb))?; - let cb = file_roundtrip_backend(&cb); + let cb = file_roundtrip_backend(&compression_backend_from_py( + compression_backend, + Some(method), + Some(rb), + )?); + let cb = compile_compression_backend(cb)?; py.detach(|| { py_try(|| { let input = std::fs::read(input_path).map_err(|e| { PyRuntimeError::new_err(format!("failed to read '{input_path}': {e}")) })?; - let out = infotheory::decompress_bytes_backend(&input, &cb) - .map_err(|e| PyRuntimeError::new_err(e.to_string()))?; + let out = infotheory::api::try_decompress_bytes_backend(&input, &cb) + .map_err(py_infotheory_error)?; std::fs::write(output_path, &out).map_err(|e| { PyRuntimeError::new_err(format!("failed to write '{output_path}': {e}")) })?; @@ -2582,31 +2396,32 @@ fn decompress_file( } #[pyfunction] -#[pyo3(signature = (prompt, bytes, max_order=-1, backend=None, method=None, config=None))] +#[pyo3(signature = (prompt, bytes, backend=None, method=None, config=None))] fn generate_bytes<'py>( py: Python<'py>, prompt: &[u8], bytes: usize, - max_order: i64, backend: Option<&Bound<'_, PyAny>>, method: Option<&str>, config: Option<&Bound<'_, PyAny>>, ) -> PyResult> { let cfg = generation_config_from_py(config)?; let out = if let Some(backend) = backend { - let rb = rate_backend_from_py(Some(backend), method)?; + let rb = compiled_rate_backend_from_py(Some(backend), method)?; + let cb = compile_compression_backend( + CompressionBackend::try_default().map_err(py_infotheory_error)?, + )?; py.detach(|| { py_try(|| { - let ctx = InfotheoryCtx::new(rb, CompressionBackend::default()); - Ok(ctx.generate_bytes_with_config(prompt, bytes, max_order, cfg)) + let ctx = InfotheoryCtx::new(rb, cb); + ctx.try_generate_bytes_with_config(prompt, bytes, cfg) + .map_err(py_infotheory_error) }) })? } else { py.detach(|| { py_try(|| { - Ok(infotheory::generate_bytes_with_config( - prompt, bytes, max_order, cfg, - )) + api::try_generate_bytes_with_config(prompt, bytes, cfg).map_err(py_infotheory_error) }) })? }; @@ -2614,12 +2429,11 @@ fn generate_bytes<'py>( } #[pyfunction] -#[pyo3(signature = (prefix_parts, bytes, max_order=-1, backend=None, method=None, config=None))] +#[pyo3(signature = (prefix_parts, bytes, backend=None, method=None, config=None))] fn generate_bytes_conditional_chain<'py>( py: Python<'py>, prefix_parts: Vec>, bytes: usize, - max_order: i64, backend: Option<&Bound<'_, PyAny>>, method: Option<&str>, config: Option<&Bound<'_, PyAny>>, @@ -2627,19 +2441,22 @@ fn generate_bytes_conditional_chain<'py>( let cfg = generation_config_from_py(config)?; let refs: Vec<&[u8]> = prefix_parts.iter().map(Vec::as_slice).collect(); let out = if let Some(backend) = backend { - let rb = rate_backend_from_py(Some(backend), method)?; + let rb = compiled_rate_backend_from_py(Some(backend), method)?; + let cb = compile_compression_backend( + CompressionBackend::try_default().map_err(py_infotheory_error)?, + )?; py.detach(|| { py_try(|| { - let ctx = InfotheoryCtx::new(rb, CompressionBackend::default()); - Ok(ctx.generate_bytes_conditional_chain_with_config(&refs, bytes, max_order, cfg)) + let ctx = InfotheoryCtx::new(rb, cb); + ctx.try_generate_bytes_conditional_chain_with_config(&refs, bytes, cfg) + .map_err(py_infotheory_error) }) })? } else { py.detach(|| { py_try(|| { - Ok(infotheory::generate_bytes_conditional_chain_with_config( - &refs, bytes, max_order, cfg, - )) + api::try_generate_bytes_conditional_chain_with_config(&refs, bytes, cfg) + .map_err(py_infotheory_error) }) })? }; @@ -2647,117 +2464,103 @@ fn generate_bytes_conditional_chain<'py>( } #[pyfunction] -#[pyo3(signature = (data, max_order, backend=None, method=None))] +#[pyo3(signature = (data, backend=None, method=None))] fn entropy_rate_backend( py: Python<'_>, data: &[u8], - max_order: i64, backend: Option<&Bound<'_, PyAny>>, method: Option<&str>, ) -> PyResult { - let rb = rate_backend_from_py(backend, method)?; - py.detach(|| py_try(|| Ok(infotheory::entropy_rate_backend(data, max_order, &rb)))) + let rb = compiled_rate_backend_from_py(backend, method)?; + py.detach(|| py_try(|| api::try_entropy_rate_backend(data, &rb).map_err(py_infotheory_error))) } #[pyfunction] -#[pyo3(signature = (data, max_order, backend=None, method=None))] +#[pyo3(signature = (data, backend=None, method=None))] fn biased_entropy_rate_backend( py: Python<'_>, data: &[u8], - max_order: i64, backend: Option<&Bound<'_, PyAny>>, method: Option<&str>, ) -> PyResult { - let rb = rate_backend_from_py(backend, method)?; + let rb = compiled_rate_backend_from_py(backend, method)?; py.detach(|| { - py_try(|| { - Ok(infotheory::biased_entropy_rate_backend( - data, max_order, &rb, - )) - }) + py_try(|| api::try_biased_entropy_rate_backend(data, &rb).map_err(py_infotheory_error)) }) } #[pyfunction] -#[pyo3(signature = (test_data, train_data, max_order, backend=None, method=None))] +#[pyo3(signature = (test_data, train_data, backend=None, method=None))] fn cross_entropy_rate_backend( py: Python<'_>, test_data: &[u8], train_data: &[u8], - max_order: i64, backend: Option<&Bound<'_, PyAny>>, method: Option<&str>, ) -> PyResult { - let rb = rate_backend_from_py(backend, method)?; + let rb = compiled_rate_backend_from_py(backend, method)?; py.detach(|| { py_try(|| { - Ok(infotheory::cross_entropy_rate_backend( - test_data, train_data, max_order, &rb, - )) + api::try_cross_entropy_rate_backend(test_data, train_data, &rb) + .map_err(py_infotheory_error) }) }) } #[pyfunction] -#[pyo3(signature = (x, y, max_order, backend=None, method=None))] +#[pyo3(signature = (x, y, backend=None, method=None))] fn joint_entropy_rate_backend( py: Python<'_>, x: &[u8], y: &[u8], - max_order: i64, backend: Option<&Bound<'_, PyAny>>, method: Option<&str>, ) -> PyResult { - let rb = rate_backend_from_py(backend, method)?; - py.detach(|| py_try(|| Ok(infotheory::joint_entropy_rate_backend(x, y, max_order, &rb)))) + let rb = compiled_rate_backend_from_py(backend, method)?; + py.detach(|| { + py_try(|| api::try_joint_entropy_rate_backend(x, y, &rb).map_err(py_infotheory_error)) + }) } #[pyfunction] -#[pyo3(signature = (x, y, max_order, backend=None, method=None))] +#[pyo3(signature = (x, y, backend=None, method=None))] fn mutual_information_rate_backend( py: Python<'_>, x: &[u8], y: &[u8], - max_order: i64, backend: Option<&Bound<'_, PyAny>>, method: Option<&str>, ) -> PyResult { - let rb = rate_backend_from_py(backend, method)?; + let rb = compiled_rate_backend_from_py(backend, method)?; py.detach(|| { - py_try(|| { - Ok(infotheory::mutual_information_rate_backend( - x, y, max_order, &rb, - )) - }) + py_try(|| api::try_mutual_information_rate_backend(x, y, &rb).map_err(py_infotheory_error)) }) } #[pyfunction] -#[pyo3(signature = (x, y, max_order, backend=None, method=None))] +#[pyo3(signature = (x, y, backend=None, method=None))] fn ned_rate_backend( py: Python<'_>, x: &[u8], y: &[u8], - max_order: i64, backend: Option<&Bound<'_, PyAny>>, method: Option<&str>, ) -> PyResult { - let rb = rate_backend_from_py(backend, method)?; - py.detach(|| py_try(|| Ok(infotheory::ned_rate_backend(x, y, max_order, &rb)))) + let rb = compiled_rate_backend_from_py(backend, method)?; + py.detach(|| py_try(|| api::try_ned_rate_backend(x, y, &rb).map_err(py_infotheory_error))) } #[pyfunction] -#[pyo3(signature = (x, y, max_order, backend=None, method=None))] +#[pyo3(signature = (x, y, backend=None, method=None))] fn nte_rate_backend( py: Python<'_>, x: &[u8], y: &[u8], - max_order: i64, backend: Option<&Bound<'_, PyAny>>, method: Option<&str>, ) -> PyResult { - let rb = rate_backend_from_py(backend, method)?; - py.detach(|| py_try(|| Ok(infotheory::nte_rate_backend(x, y, max_order, &rb)))) + let rb = compiled_rate_backend_from_py(backend, method)?; + py.detach(|| py_try(|| api::try_nte_rate_backend(x, y, &rb).map_err(py_infotheory_error))) } #[pyfunction] @@ -2770,7 +2573,10 @@ fn ncd_matrix_paths( ) -> PyResult> { let refs: Vec<&str> = paths.iter().map(String::as_str).collect(); let v = parse_ncd_variant(variant)?; - py.detach(|| py_try(|| Ok(infotheory::ncd_matrix_paths(&refs, method, v)))) + let cb = compile_compression_backend(CompressionBackend::zpaq(method))?; + py.detach(|| { + py_try(|| api::try_ncd_matrix_paths_backend(&refs, &cb, v).map_err(py_infotheory_error)) + }) } #[pyfunction] @@ -2782,7 +2588,10 @@ fn ncd_matrix_bytes( variant: &str, ) -> PyResult> { let v = parse_ncd_variant(variant)?; - py.detach(|| py_try(|| Ok(infotheory::ncd_matrix_bytes(&datas, method, v)))) + let cb = compile_compression_backend(CompressionBackend::zpaq(method))?; + py.detach(|| { + py_try(|| api::try_ncd_matrix_bytes_backend(&datas, &cb, v).map_err(py_infotheory_error)) + }) } #[pyfunction] @@ -2795,14 +2604,15 @@ fn ncd_matrix_bytes_with_backend( variant: &str, ) -> PyResult> { let v = parse_ncd_variant(variant)?; - let cb = compression_backend_from_py(backend, method, None)?; + let cb = compiled_compression_backend_from_py(backend, method, None)?; py.detach(|| { py_try(|| { let n = datas.len(); let mut out = vec![0.0; n * n]; for i in 0..n { for j in 0..n { - out[i * n + j] = infotheory::ncd_bytes_backend(&datas[i], &datas[j], &cb, v); + out[i * n + j] = api::try_ncd_bytes_backend(&datas[i], &datas[j], &cb, v) + .map_err(py_infotheory_error)?; } } Ok(out) @@ -2954,7 +2764,14 @@ impl PyRandomGenerator { #[new] fn new() -> Self { Self { - inner: infotheory::aixi::common::RandomGenerator::new(), + inner: infotheory::aixi::common::RandomGenerator::new(), + } + } + + #[staticmethod] + fn from_entropy() -> Self { + Self { + inner: infotheory::aixi::common::RandomGenerator::from_entropy(), } } fn next_u64(&mut self) -> u64 { @@ -3353,13 +3170,25 @@ impl infotheory::aixi::environment::Environment for PyEnvironmentShim { struct PyAgentSimulatorShim { obj: Mutex>, + // Structural simulator invariant cached once at the Python boundary. + action_alphabet: infotheory::aixi::common::ActionAlphabet, } impl PyAgentSimulatorShim { - fn new(obj: Py) -> Self { - Self { + fn try_new(obj: Py) -> PyResult { + // Validate the action alphabet at construction so downstream planner + // code can rely on a non-empty action set without repeated callbacks. + let action_alphabet = Python::attach(|py| { + let n = obj + .bind(py) + .call_method0("get_num_actions")? + .extract::()?; + py_action_alphabet_from_usize(n, "AgentSimulator.get_num_actions()") + })?; + Ok(Self { obj: Mutex::new(obj), - } + action_alphabet, + }) } fn parse_key_mode(py_obj: &Bound<'_, PyAny>) -> infotheory::aixi::common::ObservationKeyMode { @@ -3403,18 +3232,8 @@ impl PyAgentSimulatorShim { } impl infotheory::aixi::mcts::AgentSimulator for PyAgentSimulatorShim { - fn get_num_actions(&self) -> usize { - Python::attach(|py| { - let guard = lock_recover(&self.obj); - py_result_or_fatal( - py, - "AgentSimulator.get_num_actions", - guard - .bind(py) - .call_method0("get_num_actions") - .and_then(|v| v.extract::()), - ) - }) + fn get_num_actions(&self) -> infotheory::aixi::common::ActionAlphabet { + self.action_alphabet } fn get_num_observation_bits(&self) -> usize { @@ -3691,7 +3510,10 @@ impl infotheory::aixi::mcts::AgentSimulator for PyAgentSimulatorShim { }), CloneDecision::Fallback(src) => Self::clone_py_obj(&src), }; - Box::new(Self::new(cloned)) + Box::new(Self { + obj: Mutex::new(cloned), + action_alphabet: self.action_alphabet, + }) } } @@ -3750,6 +3572,7 @@ fn validate_observation_stream_len(expected: usize, actual: usize) -> PyResult<( } struct AixiRunSummary { + resolved_random_seed: u64, learn_total_reward: i64, eval_total_reward: i64, eval_average_reward: f64, @@ -3798,24 +3621,20 @@ fn run_agent_with_environment<'py>( "explore_gamma must be in [0, 1] for run_agent_with_environment", )); } - if config.inner.agent_actions == 0 { - return Err(PyValueError::new_err( - "AgentConfig.agent_actions must be >= 1 for run_agent_with_environment", - )); - } - config.inner.validate().map_err(PyValueError::new_err)?; + config.inner.validate().map_err(py_value_error)?; let summary = py.detach(|| { py_try(|| { use infotheory::aixi::agent::Agent; - use infotheory::aixi::common::RandomGenerator; + use infotheory::aixi::common::{ + EXPLORE_RANDOM_SALT, RandomGenerator, resolve_random_seed, + }; use infotheory::aixi::environment::Environment; let mut env = PyEnvironmentShim::new(environment); - if let Some(seed) = config.inner.random_seed { - env.set_random_seed(seed); - } - let mut agent = Agent::try_new(config.inner.clone()).map_err(PyValueError::new_err)?; + let resolved_seed = resolve_random_seed(config.inner.random_seed); + env.set_random_seed(resolved_seed); + let mut agent = Agent::try_new(config.inner.clone()).map_err(py_value_error)?; let observation_stream_len = config.inner.observation_stream_len.max(1); let (learn_cycles, eval_cycles) = match (learn_cycles, eval_cycles) { @@ -3831,11 +3650,8 @@ fn run_agent_with_environment<'py>( let mut obs_stream = env.drain_observations(); validate_observation_stream_len(observation_stream_len, obs_stream.len())?; let mut reward = env.get_reward(); - let mut explore_rng = if let Some(seed) = config.inner.random_seed { - RandomGenerator::from_seed(seed).fork_with(0x4558504c4f52455f) - } else { - RandomGenerator::new() - }; + let mut explore_rng = + RandomGenerator::from_seed(resolved_seed).fork_with(EXPLORE_RANDOM_SALT); let learn_start = Instant::now(); let mut learn_cycles_completed = 0usize; @@ -3849,7 +3665,7 @@ fn run_agent_with_environment<'py>( }; let action = if explore_p > 0.0 && explore_rng.gen_bool(explore_p.min(1.0)) { - explore_rng.gen_range(config.inner.agent_actions) as u64 + explore_rng.gen_range(config.inner.agent_actions.get()) as u64 } else { agent.get_planned_action(&obs_stream, reward, prev_action) }; @@ -3914,6 +3730,7 @@ fn run_agent_with_environment<'py>( }; Ok(AixiRunSummary { + resolved_random_seed: agent.resolved_random_seed(), learn_total_reward, eval_total_reward, eval_average_reward, @@ -3931,6 +3748,7 @@ fn run_agent_with_environment<'py>( })?; let out = PyDict::new(py); + out.set_item("resolved_random_seed", summary.resolved_random_seed)?; out.set_item("learn_total_reward", summary.learn_total_reward)?; out.set_item("eval_total_reward", summary.eval_total_reward)?; out.set_item("eval_average_reward", summary.eval_average_reward)?; @@ -3982,13 +3800,13 @@ fn run_aiqi_with_environment<'py>( let summary = py.detach(|| { py_try(|| { use infotheory::aixi::aiqi::AiqiAgent; + use infotheory::aixi::common::resolve_random_seed; use infotheory::aixi::environment::Environment; let mut env = PyEnvironmentShim::new(environment); - if let Some(seed) = config.inner.random_seed { - env.set_random_seed(seed); - } - let mut agent = AiqiAgent::new(config.inner.clone()).map_err(PyValueError::new_err)?; + let resolved_seed = resolve_random_seed(config.inner.random_seed); + env.set_random_seed(resolved_seed); + let mut agent = AiqiAgent::new(config.inner.clone()).map_err(py_value_error)?; let observation_stream_len = config.inner.observation_stream_len.max(1); let (learn_cycles, eval_cycles) = match (learn_cycles, eval_cycles) { @@ -4022,7 +3840,7 @@ fn run_aiqi_with_environment<'py>( agent .observe_transition(action, &next_obs_stream, next_reward) - .map_err(PyValueError::new_err)?; + .map_err(py_value_error)?; obs_stream = next_obs_stream; reward = next_reward; @@ -4047,7 +3865,7 @@ fn run_aiqi_with_environment<'py>( agent .observe_transition(action, &next_obs_stream, next_reward) - .map_err(PyValueError::new_err)?; + .map_err(py_value_error)?; obs_stream = next_obs_stream; reward = next_reward; @@ -4080,6 +3898,7 @@ fn run_aiqi_with_environment<'py>( }; Ok(AixiRunSummary { + resolved_random_seed: agent.resolved_random_seed(), learn_total_reward, eval_total_reward, eval_average_reward, @@ -4097,6 +3916,7 @@ fn run_aiqi_with_environment<'py>( })?; let out = PyDict::new(py); + out.set_item("resolved_random_seed", summary.resolved_random_seed)?; out.set_item("learn_total_reward", summary.learn_total_reward)?; out.set_item("eval_total_reward", summary.eval_total_reward)?; out.set_item("eval_average_reward", summary.eval_average_reward)?; @@ -4112,8 +3932,54 @@ fn run_aiqi_with_environment<'py>( Ok(out) } +enum PySearchPlannerState { + RhoUct(infotheory::aixi::mcts::RhoUctPlanner), + ParallelUct(infotheory::aixi::mcts::ParallelUctPlanner), +} + +impl PySearchPlannerState { + fn new(strategy: infotheory::aixi::common::MctsStrategy) -> PyResult { + match strategy { + infotheory::aixi::common::MctsStrategy::RhoUct => { + Ok(Self::RhoUct(infotheory::aixi::mcts::RhoUctPlanner::new())) + } + infotheory::aixi::common::MctsStrategy::ParallelUct { + workers, + bu_uct_m_max, + } => infotheory::aixi::mcts::ParallelUctPlanner::new(workers, bu_uct_m_max) + .map(Self::ParallelUct) + .map_err(py_value_error), + // `MctsStrategy` is `#[non_exhaustive]`. Future variants must be + // explicitly mapped to a concrete planner; until then, surface a + // stable Python `ValueError` instead of silently picking a default. + other => Err(PyValueError::new_err(format!( + "unsupported MctsStrategy variant '{}': not implemented in this binding", + other.kind_str() + ))), + } + } + + fn search( + &mut self, + agent: &mut dyn infotheory::aixi::mcts::AgentSimulator, + prev_obs_stream: &[u64], + prev_rew: i64, + prev_act: u64, + num_simulations: usize, + ) -> PyResult { + match self { + Self::RhoUct(planner) => { + Ok(planner.search(agent, prev_obs_stream, prev_rew, prev_act, num_simulations)) + } + Self::ParallelUct(planner) => planner + .search(agent, prev_obs_stream, prev_rew, prev_act, num_simulations) + .map_err(py_value_error), + } + } +} + #[pyfunction] -#[pyo3(signature = (simulator, prev_obs_stream, prev_rew, prev_act, num_simulations))] +#[pyo3(signature = (simulator, prev_obs_stream, prev_rew, prev_act, num_simulations, mcts_strategy=None))] fn search_with_simulator( py: Python<'_>, simulator: Py, @@ -4121,18 +3987,20 @@ fn search_with_simulator( prev_rew: i64, prev_act: u64, num_simulations: usize, + mcts_strategy: Option<&PyMctsStrategy>, ) -> PyResult { + let strategy = resolve_mcts_strategy(mcts_strategy); py.detach(|| { py_try(|| { - let mut sim = PyAgentSimulatorShim::new(simulator); - let mut tree = infotheory::aixi::mcts::SearchTree::new(); - Ok(tree.search( + let mut sim = PyAgentSimulatorShim::try_new(simulator)?; + let mut planner = PySearchPlannerState::new(strategy)?; + planner.search( &mut sim, &prev_obs_stream, prev_rew, prev_act, num_simulations, - )) + ) }) }) } @@ -4175,6 +4043,79 @@ impl PyObservationKeyMode { } } +#[pyclass(name = "MctsStrategy", from_py_object)] +#[derive(Clone, Copy)] +struct PyMctsStrategy { + inner: infotheory::aixi::common::MctsStrategy, +} + +#[pymethods] +impl PyMctsStrategy { + #[staticmethod] + fn rho_uct() -> Self { + Self { + inner: infotheory::aixi::common::MctsStrategy::RhoUct, + } + } + + #[staticmethod] + #[pyo3(signature = (workers, bu_uct_m_max=None))] + fn parallel_uct(workers: usize, bu_uct_m_max: Option) -> PyResult { + // `workers >= 1` is type-enforced inside the Rust strategy enum via + // `NonZeroUsize`, so we lift the Python integer through the smart + // constructor here and surface a Python-friendly `ValueError` if the + // caller passed zero. + let workers = std::num::NonZeroUsize::new(workers).ok_or_else(|| { + PyValueError::new_err("MctsStrategy.parallel_uct(workers=...) must be >= 1") + })?; + // Construct (and validate) a planner once to ensure `bu_uct_m_max` + // is in range; the strategy itself only stores the parameters. + infotheory::aixi::mcts::ParallelUctPlanner::new(workers, bu_uct_m_max) + .map_err(py_value_error)?; + Ok(Self { + inner: infotheory::aixi::common::MctsStrategy::ParallelUct { + workers, + bu_uct_m_max, + }, + }) + } + + #[getter] + fn kind(&self) -> &'static str { + self.inner.kind_str() + } + + #[getter] + fn workers(&self) -> Option { + match self.inner { + infotheory::aixi::common::MctsStrategy::RhoUct => None, + infotheory::aixi::common::MctsStrategy::ParallelUct { workers, .. } => { + Some(workers.get()) + } + // `MctsStrategy` is `#[non_exhaustive]`; unknown future variants + // expose `None` rather than guessing a worker count. + _ => None, + } + } + + #[getter] + fn bu_uct_m_max(&self) -> Option { + match self.inner { + infotheory::aixi::common::MctsStrategy::RhoUct => None, + infotheory::aixi::common::MctsStrategy::ParallelUct { bu_uct_m_max, .. } => { + bu_uct_m_max + } + // `MctsStrategy` is `#[non_exhaustive]`; unknown future variants + // expose `None` rather than fabricating a threshold. + _ => None, + } + } + + fn __repr__(&self) -> String { + format_mcts_strategy(self.inner) + } +} + #[pyclass(name = "AgentConfig", from_py_object)] #[derive(Clone)] struct PyAgentConfig { @@ -4192,8 +4133,7 @@ impl PyAgentConfig { #[new] #[allow(clippy::too_many_arguments)] #[pyo3(signature = ( - algorithm="fac-ctw".to_string(), - ct_depth=16, + rate_backend, agent_horizon=6, observation_bits=8, observation_stream_len=1, @@ -4201,21 +4141,17 @@ impl PyAgentConfig { reward_bits=8, agent_actions=2, num_simulations=256, + mcts_strategy=None, exploration_exploitation_ratio=1.41, discount_gamma=1.0, min_reward=-128, max_reward=127, reward_offset=128, random_seed=None, - rate_backend=None, - rate_backend_max_order=20, - rwkv_model_path=None, - rosa_max_order=None, - zpaq_method=None + bit_stream_semantics=None ))] fn new( - algorithm: String, - ct_depth: usize, + rate_backend: &PyRateBackend, agent_horizon: usize, observation_bits: usize, observation_stream_len: usize, @@ -4223,46 +4159,38 @@ impl PyAgentConfig { reward_bits: usize, agent_actions: usize, num_simulations: usize, + mcts_strategy: Option<&PyMctsStrategy>, exploration_exploitation_ratio: f64, discount_gamma: f64, min_reward: i64, max_reward: i64, reward_offset: i64, random_seed: Option, - rate_backend: Option<&PyRateBackend>, - rate_backend_max_order: i64, - rwkv_model_path: Option, - rosa_max_order: Option, - zpaq_method: Option, + bit_stream_semantics: Option<&Bound<'_, PyAny>>, ) -> PyResult { - let inner = infotheory::aixi::agent::AgentConfig { - algorithm, - ct_depth, - agent_horizon, - observation_bits, - observation_stream_len, - observation_key_mode: observation_key_mode - .map(|m| m.inner) - .unwrap_or(infotheory::aixi::common::ObservationKeyMode::FullStream), - reward_bits, - agent_actions, - num_simulations, - exploration_exploitation_ratio, - discount_gamma, - min_reward, - max_reward, - reward_offset, - random_seed, - rate_backend: rate_backend.map(|rb| rb.inner.clone()), - rate_backend_max_order, - rwkv_model_path, - rwkv_method: None, - mamba_model_path: None, - mamba_method: None, - rosa_max_order, - zpaq_method, - }; - inner.validate().map_err(PyValueError::new_err)?; + let mut inner = infotheory::aixi::agent::AgentConfig::default(); + inner.rate_backend = rate_backend.inner.clone(); + if let Some(semantics) = bit_stream_semantics { + inner.bit_stream_semantics = parse_bit_stream_semantics_value(semantics)?; + } + inner.agent_horizon = agent_horizon; + inner.observation_bits = observation_bits; + inner.observation_stream_len = observation_stream_len; + inner.observation_key_mode = observation_key_mode + .map(|m| m.inner) + .unwrap_or(infotheory::aixi::common::ObservationKeyMode::FullStream); + inner.reward_bits = reward_bits; + inner.agent_actions = + py_action_alphabet_from_usize(agent_actions, "AgentConfig.agent_actions")?; + inner.num_simulations = num_simulations; + inner.mcts_strategy = resolve_mcts_strategy(mcts_strategy); + inner.exploration_exploitation_ratio = exploration_exploitation_ratio; + inner.discount_gamma = discount_gamma; + inner.min_reward = min_reward; + inner.max_reward = max_reward; + inner.reward_offset = reward_offset; + inner.random_seed = random_seed; + inner.validate().map_err(py_value_error)?; Ok(Self { inner }) } } @@ -4272,8 +4200,7 @@ impl PyAiqiConfig { #[new] #[allow(clippy::too_many_arguments)] #[pyo3(signature = ( - algorithm="ac-ctw".to_string(), - ct_depth=16, + rate_backend, observation_bits=8, observation_stream_len=1, reward_bits=8, @@ -4288,15 +4215,10 @@ impl PyAiqiConfig { history_prune_keep_steps=None, baseline_exploration=0.01, random_seed=None, - rate_backend=None, - rate_backend_max_order=20, - rwkv_model_path=None, - rosa_max_order=None, - zpaq_method=None + bit_stream_semantics=None ))] fn new( - algorithm: String, - ct_depth: usize, + rate_backend: &PyRateBackend, observation_bits: usize, observation_stream_len: usize, reward_bits: usize, @@ -4311,36 +4233,29 @@ impl PyAiqiConfig { history_prune_keep_steps: Option, baseline_exploration: f64, random_seed: Option, - rate_backend: Option<&PyRateBackend>, - rate_backend_max_order: i64, - rwkv_model_path: Option, - rosa_max_order: Option, - zpaq_method: Option, + bit_stream_semantics: Option<&Bound<'_, PyAny>>, ) -> PyResult { - let inner = infotheory::aixi::aiqi::AiqiConfig { - algorithm, - ct_depth, - observation_bits, - observation_stream_len, - reward_bits, - agent_actions, - min_reward, - max_reward, - reward_offset, - discount_gamma, - return_horizon, - return_bins, - augmentation_period: augmentation_period.unwrap_or(return_horizon), - history_prune_keep_steps, - baseline_exploration, - random_seed, - rate_backend: rate_backend.map(|rb| rb.inner.clone()), - rate_backend_max_order, - rwkv_model_path, - rosa_max_order, - zpaq_method, - }; - inner.validate().map_err(PyValueError::new_err)?; + let mut inner = infotheory::aixi::aiqi::AiqiConfig::default(); + inner.rate_backend = rate_backend.inner.clone(); + if let Some(semantics) = bit_stream_semantics { + inner.bit_stream_semantics = parse_bit_stream_semantics_value(semantics)?; + } + inner.observation_bits = observation_bits; + inner.observation_stream_len = observation_stream_len; + inner.reward_bits = reward_bits; + inner.agent_actions = + py_action_alphabet_from_usize(agent_actions, "AiqiConfig.agent_actions")?; + inner.min_reward = min_reward; + inner.max_reward = max_reward; + inner.reward_offset = reward_offset; + inner.discount_gamma = discount_gamma; + inner.return_horizon = return_horizon; + inner.return_bins = return_bins; + inner.augmentation_period = augmentation_period.unwrap_or(return_horizon); + inner.history_prune_keep_steps = history_prune_keep_steps; + inner.baseline_exploration = baseline_exploration; + inner.random_seed = random_seed; + inner.validate().map_err(py_value_error)?; Ok(Self { inner }) } } @@ -4356,7 +4271,7 @@ impl PyAgent { fn new(config: &PyAgentConfig) -> PyResult { py_try(|| { let inner = infotheory::aixi::agent::Agent::try_new(config.inner.clone()) - .map_err(PyValueError::new_err)?; + .map_err(py_value_error)?; Ok(Self { inner }) }) } @@ -4391,6 +4306,10 @@ impl PyAgent { fn model_update_action_external(&mut self, action: u64) { self.inner.model_update_action_external(action) } + + fn resolved_random_seed(&self) -> u64 { + self.inner.resolved_random_seed() + } } #[pyclass(name = "AiqiAgent", unsendable)] @@ -4404,7 +4323,7 @@ impl PyAiqiAgent { fn new(config: &PyAiqiConfig) -> PyResult { py_try(|| { let inner = infotheory::aixi::aiqi::AiqiAgent::new(config.inner.clone()) - .map_err(PyValueError::new_err)?; + .map_err(py_value_error)?; Ok(Self { inner }) }) } @@ -4414,7 +4333,7 @@ impl PyAiqiAgent { } fn num_actions(&self) -> usize { - self.inner.num_actions() + self.inner.num_actions().get() } fn get_planned_action(&mut self) -> u64 { @@ -4435,15 +4354,21 @@ impl PyAiqiAgent { ) -> PyResult<()> { self.inner .observe_transition(action, &observations, reward) - .map_err(PyValueError::new_err) + .map_err(py_value_error) + } + + fn resolved_random_seed(&self) -> u64 { + self.inner.resolved_random_seed() } } +#[cfg(feature = "backend-ctw")] #[pyclass(name = "CtwPredictor")] struct PyCtwPredictor { inner: infotheory::aixi::model::CtwPredictor, } +#[cfg(feature = "backend-ctw")] #[pymethods] impl PyCtwPredictor { #[new] @@ -4482,11 +4407,13 @@ impl PyCtwPredictor { } } +#[cfg(feature = "backend-ctw")] #[pyclass(name = "FacCtwPredictor")] struct PyFacCtwPredictor { inner: infotheory::aixi::model::FacCtwPredictor, } +#[cfg(feature = "backend-ctw")] #[pymethods] impl PyFacCtwPredictor { #[new] @@ -4525,11 +4452,13 @@ impl PyFacCtwPredictor { } } +#[cfg(feature = "backend-rosa")] #[pyclass(name = "RosaPredictor")] struct PyRosaPredictor { inner: infotheory::aixi::model::RosaPredictor, } +#[cfg(feature = "backend-rosa")] #[pymethods] impl PyRosaPredictor { #[new] @@ -4569,11 +4498,13 @@ impl PyRosaPredictor { } } +#[cfg(feature = "backend-zpaq")] #[pyclass(name = "ZpaqPredictor", unsendable)] struct PyZpaqPredictor { inner: infotheory::aixi::model::ZpaqPredictor, } +#[cfg(feature = "backend-zpaq")] #[pymethods] impl PyZpaqPredictor { #[new] @@ -4616,510 +4547,308 @@ impl PyZpaqPredictor { } } -#[cfg(feature = "backend-rwkv")] -#[pyclass(name = "RwkvPredictor")] -struct PyRwkvPredictor { - inner: infotheory::aixi::model::RwkvPredictor, -} - -#[cfg(feature = "backend-rwkv")] -#[pymethods] -impl PyRwkvPredictor { - #[new] - fn new(model_path: String) -> Self { - let model = infotheory::load_rwkv7_model_from_path(&model_path); - Self { - inner: infotheory::aixi::model::RwkvPredictor::new(model), - } - } - fn update(&mut self, sym: bool) { - use infotheory::aixi::model::Predictor; - self.inner.update(sym); - } - fn update_history(&mut self, sym: bool) { - use infotheory::aixi::model::Predictor; - self.inner.update_history(sym); - } - fn revert(&mut self) { - use infotheory::aixi::model::Predictor; - self.inner.revert(); - } - fn pop_history(&mut self) { - use infotheory::aixi::model::Predictor; - self.inner.pop_history(); - } - fn predict_prob(&mut self, sym: bool) -> f64 { - use infotheory::aixi::model::Predictor; - self.inner.predict_prob(sym) - } - fn predict_one(&mut self) -> f64 { - use infotheory::aixi::model::Predictor; - self.inner.predict_one() - } - fn model_name(&self) -> String { - use infotheory::aixi::model::Predictor; - self.inner.model_name() - } -} - -#[pyclass(name = "SearchNode")] -struct PySearchNode { - inner: infotheory::aixi::mcts::SearchNode, -} - -#[pymethods] -impl PySearchNode { - #[new] - #[pyo3(signature = (is_chance_node=false))] - fn new(is_chance_node: bool) -> Self { - Self { - inner: infotheory::aixi::mcts::SearchNode::new(is_chance_node), - } - } - - fn best_action(&self, agent: &mut PyAgent) -> u64 { - self.inner.best_action(&mut agent.inner) - } -} - -#[pyclass(name = "SearchTree")] -struct PySearchTree { - inner: infotheory::aixi::mcts::SearchTree, -} - -#[pymethods] -impl PySearchTree { - #[new] - fn new() -> Self { - Self { - inner: infotheory::aixi::mcts::SearchTree::new(), - } - } - - fn search( - &mut self, - agent: &mut PyAgent, - prev_obs_stream: Vec, - prev_rew: i64, - prev_act: u64, - num_simulations: usize, - ) -> u64 { - self.inner.search( - &mut agent.inner, - &prev_obs_stream, - prev_rew, - prev_act, - num_simulations, - ) - } -} - -#[pyclass(name = "CoinFlipEnv")] -struct CoinFlipEnv { - inner: infotheory::aixi::environment::CoinFlip, -} - -#[pymethods] -impl CoinFlipEnv { - #[new] - #[pyo3(signature = (p=0.5, random_seed=None))] - fn new(p: f64, random_seed: Option) -> Self { - use infotheory::aixi::environment::Environment; - let mut inner = infotheory::aixi::environment::CoinFlip::new(p); - if let Some(seed) = random_seed { - inner.set_random_seed(seed); - } - Self { inner } - } - - fn set_random_seed(&mut self, seed: u64) { - use infotheory::aixi::environment::Environment; - self.inner.set_random_seed(seed); - } - - fn get_observation_bits(&self) -> usize { - use infotheory::aixi::environment::Environment; - self.inner.get_observation_bits() - } - - fn get_reward_bits(&self) -> usize { - use infotheory::aixi::environment::Environment; - self.inner.get_reward_bits() - } - - fn get_action_bits(&self) -> usize { - use infotheory::aixi::environment::Environment; - self.inner.get_action_bits() - } - - fn get_num_actions(&self) -> usize { - use infotheory::aixi::environment::Environment; - self.inner.get_num_actions() - } - - fn min_reward(&self) -> i64 { - use infotheory::aixi::environment::Environment; - self.inner.min_reward() - } - - fn max_reward(&self) -> i64 { - use infotheory::aixi::environment::Environment; - self.inner.max_reward() - } - - fn perform_action(&mut self, action: u64) { - use infotheory::aixi::environment::Environment; - self.inner.perform_action(action); - } - fn get_observation(&self) -> u64 { - use infotheory::aixi::environment::Environment; - self.inner.get_observation() - } - fn get_reward(&self) -> i64 { - use infotheory::aixi::environment::Environment; - self.inner.get_reward() - } - fn is_finished(&self) -> bool { - use infotheory::aixi::environment::Environment; - self.inner.is_finished() - } - fn drain_observations(&mut self) -> Vec { - use infotheory::aixi::environment::Environment; - self.inner.drain_observations() - } -} - -#[pyclass(name = "CtwTestEnv")] -struct CtwTestEnv { - inner: infotheory::aixi::environment::CtwTest, -} - -#[pymethods] -impl CtwTestEnv { - #[new] - fn new() -> Self { - Self { - inner: infotheory::aixi::environment::CtwTest::new(), - } - } - fn perform_action(&mut self, action: u64) { - use infotheory::aixi::environment::Environment; - self.inner.perform_action(action); - } - fn get_observation(&self) -> u64 { - use infotheory::aixi::environment::Environment; - self.inner.get_observation() - } - fn get_reward(&self) -> i64 { - use infotheory::aixi::environment::Environment; - self.inner.get_reward() - } - fn is_finished(&self) -> bool { - use infotheory::aixi::environment::Environment; - self.inner.is_finished() - } - fn drain_observations(&mut self) -> Vec { - use infotheory::aixi::environment::Environment; - self.inner.drain_observations() - } -} - -#[pyclass(name = "BiasedRockPaperScissorEnv")] -struct BiasedRockPaperScissorEnv { - inner: infotheory::aixi::environment::BiasedRockPaperScissor, +#[cfg(feature = "backend-rwkv")] +#[pyclass(name = "RwkvPredictor")] +struct PyRwkvPredictor { + inner: infotheory::aixi::model::RwkvPredictor, } +#[cfg(feature = "backend-rwkv")] #[pymethods] -impl BiasedRockPaperScissorEnv { +impl PyRwkvPredictor { #[new] - #[pyo3(signature = (random_seed=None))] - fn new(random_seed: Option) -> Self { - use infotheory::aixi::environment::Environment; - let mut inner = infotheory::aixi::environment::BiasedRockPaperScissor::new(); - if let Some(seed) = random_seed { - inner.set_random_seed(seed); - } - Self { inner } - } - - fn set_random_seed(&mut self, seed: u64) { - use infotheory::aixi::environment::Environment; - self.inner.set_random_seed(seed); - } - - fn get_observation_bits(&self) -> usize { - use infotheory::aixi::environment::Environment; - self.inner.get_observation_bits() - } - - fn get_reward_bits(&self) -> usize { - use infotheory::aixi::environment::Environment; - self.inner.get_reward_bits() - } - - fn get_action_bits(&self) -> usize { - use infotheory::aixi::environment::Environment; - self.inner.get_action_bits() - } - - fn get_num_actions(&self) -> usize { - use infotheory::aixi::environment::Environment; - self.inner.get_num_actions() + fn new(model_path: String) -> PyResult { + let model = infotheory::rwkvzip::Compressor::load_model(&model_path).map_err(|err| { + PyRuntimeError::new_err(format!("failed to load RWKV7 model: {err:#}")) + })?; + Ok(Self { + inner: infotheory::aixi::model::RwkvPredictor::new(model), + }) } - - fn min_reward(&self) -> i64 { - use infotheory::aixi::environment::Environment; - self.inner.min_reward() + fn update(&mut self, sym: bool) { + use infotheory::aixi::model::Predictor; + self.inner.update(sym); } - - fn max_reward(&self) -> i64 { - use infotheory::aixi::environment::Environment; - self.inner.max_reward() + fn update_history(&mut self, sym: bool) { + use infotheory::aixi::model::Predictor; + self.inner.update_history(sym); } - fn perform_action(&mut self, action: u64) { - use infotheory::aixi::environment::Environment; - self.inner.perform_action(action); + fn revert(&mut self) { + use infotheory::aixi::model::Predictor; + self.inner.revert(); } - fn get_observation(&self) -> u64 { - use infotheory::aixi::environment::Environment; - self.inner.get_observation() + fn pop_history(&mut self) { + use infotheory::aixi::model::Predictor; + self.inner.pop_history(); } - fn get_reward(&self) -> i64 { - use infotheory::aixi::environment::Environment; - self.inner.get_reward() + fn predict_prob(&mut self, sym: bool) -> f64 { + use infotheory::aixi::model::Predictor; + self.inner.predict_prob(sym) } - fn is_finished(&self) -> bool { - use infotheory::aixi::environment::Environment; - self.inner.is_finished() + fn predict_one(&mut self) -> f64 { + use infotheory::aixi::model::Predictor; + self.inner.predict_one() } - fn drain_observations(&mut self) -> Vec { - use infotheory::aixi::environment::Environment; - self.inner.drain_observations() + fn model_name(&self) -> String { + use infotheory::aixi::model::Predictor; + self.inner.model_name() } } -#[pyclass(name = "ExtendedTigerEnv")] -struct ExtendedTigerEnv { - inner: infotheory::aixi::environment::ExtendedTiger, +#[pyclass(name = "SearchTree")] +struct PySearchTree { + strategy: infotheory::aixi::common::MctsStrategy, + inner: PySearchPlannerState, } #[pymethods] -impl ExtendedTigerEnv { +impl PySearchTree { #[new] - #[pyo3(signature = (random_seed=None))] - fn new(random_seed: Option) -> Self { - use infotheory::aixi::environment::Environment; - let mut inner = infotheory::aixi::environment::ExtendedTiger::new(); - if let Some(seed) = random_seed { - inner.set_random_seed(seed); - } - Self { inner } + #[pyo3(signature = (mcts_strategy=None))] + fn new(mcts_strategy: Option<&PyMctsStrategy>) -> PyResult { + let strategy = resolve_mcts_strategy(mcts_strategy); + Ok(Self { + strategy, + inner: PySearchPlannerState::new(strategy)?, + }) } - fn set_random_seed(&mut self, seed: u64) { - use infotheory::aixi::environment::Environment; - self.inner.set_random_seed(seed); + #[getter] + fn mcts_strategy(&self) -> PyMctsStrategy { + PyMctsStrategy { + inner: self.strategy, + } } - fn get_observation_bits(&self) -> usize { - use infotheory::aixi::environment::Environment; - self.inner.get_observation_bits() + fn __repr__(&self) -> String { + format!( + "SearchTree(mcts_strategy={})", + format_mcts_strategy(self.strategy) + ) } - fn get_reward_bits(&self) -> usize { - use infotheory::aixi::environment::Environment; - self.inner.get_reward_bits() + fn search( + &mut self, + agent: &mut PyAgent, + prev_obs_stream: Vec, + prev_rew: i64, + prev_act: u64, + num_simulations: usize, + ) -> PyResult { + self.inner.search( + &mut agent.inner, + &prev_obs_stream, + prev_rew, + prev_act, + num_simulations, + ) } +} - fn get_action_bits(&self) -> usize { - use infotheory::aixi::environment::Environment; - self.inner.get_action_bits() - } +#[cfg(feature = "aixi-gameengine")] +fn new_gameengine_builtin( + builtin: infotheory::spec::BuiltinEnvironmentSpec, + random_seed: Option, +) -> PyResult> { + let resolved_seed = infotheory::aixi::common::resolve_random_seed(random_seed); + let env = + infotheory::aixi::gameengine::build_builtin_environment_with_seed(builtin, resolved_seed) + .map_err(|err| PyRuntimeError::new_err(err.to_string()))?; + Ok(env) +} - fn get_num_actions(&self) -> usize { - use infotheory::aixi::environment::Environment; - self.inner.get_num_actions() +#[cfg(feature = "aixi-gameengine")] +fn coin_flip_probability_parts(p: f64) -> PyResult<(u64, u64)> { + const DENOMINATOR: u64 = 1_000_000; + if !p.is_finite() || !(0.0..=1.0).contains(&p) { + return Err(PyValueError::new_err( + "CoinFlipEnv p must be a finite probability in [0.0, 1.0]", + )); } + Ok(((p * DENOMINATOR as f64).round() as u64, DENOMINATOR)) +} - fn min_reward(&self) -> i64 { - use infotheory::aixi::environment::Environment; - self.inner.min_reward() - } +#[cfg(feature = "aixi-gameengine")] +macro_rules! define_gameengine_env_class { + ($(#[$cfg:meta])* $name:ident, $py_name:literal, $builtin:expr) => { + $(#[$cfg])* + #[pyclass(name = $py_name, unsendable)] + struct $name { + inner: Box, + } - fn max_reward(&self) -> i64 { - use infotheory::aixi::environment::Environment; - self.inner.max_reward() - } - fn perform_action(&mut self, action: u64) { - use infotheory::aixi::environment::Environment; - self.inner.perform_action(action); - } - fn get_observation(&self) -> u64 { - use infotheory::aixi::environment::Environment; - self.inner.get_observation() - } - fn get_reward(&self) -> i64 { - use infotheory::aixi::environment::Environment; - self.inner.get_reward() - } - fn is_finished(&self) -> bool { - use infotheory::aixi::environment::Environment; - self.inner.is_finished() - } - fn drain_observations(&mut self) -> Vec { - use infotheory::aixi::environment::Environment; - self.inner.drain_observations() - } -} + $(#[$cfg])* + #[pymethods] + impl $name { + #[new] + #[pyo3(signature = (random_seed=None))] + fn new(random_seed: Option) -> PyResult { + Ok(Self { + inner: new_gameengine_builtin($builtin, random_seed)?, + }) + } -#[pyclass(name = "TicTacToeEnv")] -struct TicTacToeEnv { - inner: infotheory::aixi::environment::TicTacToe, -} + fn set_random_seed(&mut self, seed: u64) { + self.inner.set_random_seed(seed); + } -#[pymethods] -impl TicTacToeEnv { - #[new] - #[pyo3(signature = (random_seed=None))] - fn new(random_seed: Option) -> Self { - use infotheory::aixi::environment::Environment; - let mut inner = infotheory::aixi::environment::TicTacToe::new(); - if let Some(seed) = random_seed { - inner.set_random_seed(seed); - } - Self { inner } - } + fn get_observation_bits(&self) -> usize { + self.inner.get_observation_bits() + } - fn set_random_seed(&mut self, seed: u64) { - use infotheory::aixi::environment::Environment; - self.inner.set_random_seed(seed); - } + fn get_reward_bits(&self) -> usize { + self.inner.get_reward_bits() + } - fn get_observation_bits(&self) -> usize { - use infotheory::aixi::environment::Environment; - self.inner.get_observation_bits() - } + fn get_action_bits(&self) -> usize { + self.inner.get_action_bits() + } - fn get_reward_bits(&self) -> usize { - use infotheory::aixi::environment::Environment; - self.inner.get_reward_bits() - } + fn get_num_actions(&self) -> usize { + self.inner.get_num_actions().get() + } - fn get_action_bits(&self) -> usize { - use infotheory::aixi::environment::Environment; - self.inner.get_action_bits() - } + fn min_reward(&self) -> i64 { + self.inner.min_reward() + } - fn get_num_actions(&self) -> usize { - use infotheory::aixi::environment::Environment; - self.inner.get_num_actions() - } + fn max_reward(&self) -> i64 { + self.inner.max_reward() + } - fn min_reward(&self) -> i64 { - use infotheory::aixi::environment::Environment; - self.inner.min_reward() - } + fn perform_action(&mut self, action: u64) { + self.inner.perform_action(action); + } - fn max_reward(&self) -> i64 { - use infotheory::aixi::environment::Environment; - self.inner.max_reward() - } - fn perform_action(&mut self, action: u64) { - use infotheory::aixi::environment::Environment; - self.inner.perform_action(action); - } - fn get_observation(&self) -> u64 { - use infotheory::aixi::environment::Environment; - self.inner.get_observation() - } - fn get_reward(&self) -> i64 { - use infotheory::aixi::environment::Environment; - self.inner.get_reward() - } - fn is_finished(&self) -> bool { - use infotheory::aixi::environment::Environment; - self.inner.is_finished() - } - fn drain_observations(&mut self) -> Vec { - use infotheory::aixi::environment::Environment; - self.inner.drain_observations() - } + fn get_observation(&self) -> u64 { + self.inner.get_observation() + } + + fn get_reward(&self) -> i64 { + self.inner.get_reward() + } + + fn is_finished(&self) -> bool { + self.inner.is_finished() + } + + fn drain_observations(&mut self) -> Vec { + self.inner.drain_observations() + } + } + }; } -#[pyclass(name = "KuhnPokerEnv")] -struct KuhnPokerEnv { - inner: infotheory::aixi::environment::KuhnPoker, +#[cfg(feature = "aixi-gameengine")] +#[pyclass(name = "CoinFlipEnv", unsendable)] +struct CoinFlipEnv { + inner: Box, } +#[cfg(feature = "aixi-gameengine")] #[pymethods] -impl KuhnPokerEnv { +impl CoinFlipEnv { #[new] - #[pyo3(signature = (random_seed=None))] - fn new(random_seed: Option) -> Self { - use infotheory::aixi::environment::Environment; - let mut inner = infotheory::aixi::environment::KuhnPoker::new(); - if let Some(seed) = random_seed { - inner.set_random_seed(seed); - } - Self { inner } + #[pyo3(signature = (p=0.7, random_seed=None))] + fn new(p: f64, random_seed: Option) -> PyResult { + let seed = random_seed.unwrap_or(0); + let (head_numerator, head_denominator) = coin_flip_probability_parts(p)?; + Ok(Self { + inner: infotheory::aixi::gameengine::build_coin_flip_environment( + head_numerator, + head_denominator, + seed, + ) + .map_err(|err| PyRuntimeError::new_err(err.to_string()))?, + }) } fn set_random_seed(&mut self, seed: u64) { - use infotheory::aixi::environment::Environment; self.inner.set_random_seed(seed); } fn get_observation_bits(&self) -> usize { - use infotheory::aixi::environment::Environment; self.inner.get_observation_bits() } fn get_reward_bits(&self) -> usize { - use infotheory::aixi::environment::Environment; self.inner.get_reward_bits() } fn get_action_bits(&self) -> usize { - use infotheory::aixi::environment::Environment; self.inner.get_action_bits() } fn get_num_actions(&self) -> usize { - use infotheory::aixi::environment::Environment; - self.inner.get_num_actions() + self.inner.get_num_actions().get() } fn min_reward(&self) -> i64 { - use infotheory::aixi::environment::Environment; self.inner.min_reward() } fn max_reward(&self) -> i64 { - use infotheory::aixi::environment::Environment; self.inner.max_reward() } + fn perform_action(&mut self, action: u64) { - use infotheory::aixi::environment::Environment; self.inner.perform_action(action); } + fn get_observation(&self) -> u64 { - use infotheory::aixi::environment::Environment; self.inner.get_observation() } + fn get_reward(&self) -> i64 { - use infotheory::aixi::environment::Environment; self.inner.get_reward() } + fn is_finished(&self) -> bool { - use infotheory::aixi::environment::Environment; self.inner.is_finished() } + fn drain_observations(&mut self) -> Vec { - use infotheory::aixi::environment::Environment; self.inner.drain_observations() } } +#[cfg(feature = "aixi-gameengine")] +define_gameengine_env_class!( + BiasedRockPaperScissorEnv, + "BiasedRockPaperScissorEnv", + infotheory::spec::BuiltinEnvironmentSpec::BiasedRockPaperScissor +); +#[cfg(feature = "aixi-gameengine")] +define_gameengine_env_class!( + KuhnPokerEnv, + "KuhnPokerEnv", + infotheory::spec::BuiltinEnvironmentSpec::KuhnPoker +); +#[cfg(feature = "aixi-gameengine")] +define_gameengine_env_class!( + ExtendedTigerEnv, + "ExtendedTigerEnv", + infotheory::spec::BuiltinEnvironmentSpec::ExtendedTiger +); +#[cfg(feature = "aixi-gameengine")] +define_gameengine_env_class!( + TicTacToeEnv, + "TicTacToeEnv", + infotheory::spec::BuiltinEnvironmentSpec::TicTacToe +); +#[cfg(feature = "aixi-gameengine")] +define_gameengine_env_class!( + BlackjackEnv, + "BlackjackEnv", + infotheory::spec::BuiltinEnvironmentSpec::Blackjack +); +#[cfg(feature = "aixi-gameengine-physics")] +define_gameengine_env_class!( + PlatformerEnv, + "PlatformerEnv", + infotheory::spec::BuiltinEnvironmentSpec::Platformer +); + #[pyfunction] fn vm_enabled() -> bool { cfg!(feature = "vm") @@ -5165,6 +4894,10 @@ impl PyNyxVmConfig { fn set_reward_bits(&mut self, bits: usize) { self.inner.reward_bits = bits; } + + fn validate(&self) -> PyResult<()> { + self.inner.validate().map_err(py_value_error) + } } #[cfg(feature = "vm")] @@ -5178,6 +4911,7 @@ struct PyNyxVmEnvironment { impl PyNyxVmEnvironment { #[new] fn new(config: &PyNyxVmConfig) -> PyResult { + config.inner.validate().map_err(py_value_error)?; let env = infotheory::aixi::vm_nyx::NyxVmEnvironment::new(config.inner.clone()) .map_err(|e| PyRuntimeError::new_err(e.to_string()))?; Ok(Self { inner: env }) @@ -5205,12 +4939,14 @@ impl PyNyxVmEnvironment { } } +#[cfg(feature = "backend-rosa")] #[pyclass(name = "SearchGranularity", eq, from_py_object)] #[derive(Clone, Copy, PartialEq)] struct PySearchGranularity { inner: infotheory::search::SearchGranularity, } +#[cfg(feature = "backend-rosa")] #[pymethods] impl PySearchGranularity { #[classattr] @@ -5235,12 +4971,14 @@ impl PySearchGranularity { } } +#[cfg(feature = "backend-rosa")] #[pyclass(name = "Stage2PriorMode", eq, from_py_object)] #[derive(Clone, Copy, PartialEq)] struct PyStage2PriorMode { inner: infotheory::search::Stage2PriorMode, } +#[cfg(feature = "backend-rosa")] #[pymethods] impl PyStage2PriorMode { #[classattr] @@ -5273,6 +5011,7 @@ impl PyStage2PriorMode { } } +#[cfg(feature = "backend-rosa")] #[pyfunction] #[pyo3(signature = ( query, @@ -5280,7 +5019,6 @@ impl PyStage2PriorMode { granularity=None, universal_prior=None, stage2_prior_mode=None, - max_order=8, top_k=50, stage0_keep_frac=0.2, rate_backend=None, @@ -5294,15 +5032,15 @@ fn search( granularity: Option<&PySearchGranularity>, universal_prior: Option, stage2_prior_mode: Option<&PyStage2PriorMode>, - max_order: i64, top_k: usize, stage0_keep_frac: f64, rate_backend: Option<&Bound<'_, PyAny>>, compression_backend: Option<&Bound<'_, PyAny>>, method: Option<&str>, ) -> PyResult> { - let rb = rate_backend_from_py(rate_backend, method)?; - let cb = compression_backend_from_py(compression_backend, method, Some(rb.clone()))?; + let raw_rb = rate_backend_from_py(rate_backend, method)?; + let rb = compile_rate_backend(raw_rb.clone())?; + let cb = compiled_compression_backend_from_py(compression_backend, method, Some(raw_rb))?; let q = query.to_string(); let tp = target_path.to_string(); let gran = granularity @@ -5318,12 +5056,12 @@ fn search( granularity: gran, universal_prior, stage2_prior_mode: s2pm, - max_order, top_k, stage0_keep_frac, ctx: InfotheoryCtx::new(rb, cb), }; - let results = infotheory::search::search_with_options(&q, &tp, &opts); + let results = infotheory::search::search_with_options(&q, &tp, &opts) + .map_err(py_infotheory_error)?; Ok(results .into_iter() .map(|s| { @@ -5348,6 +5086,12 @@ fn _core(_py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; @@ -5356,26 +5100,40 @@ fn _core(_py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; + #[cfg(feature = "backend-ctw")] m.add_class::()?; + #[cfg(feature = "backend-ctw")] m.add_class::()?; + #[cfg(feature = "backend-rosa")] m.add_class::()?; + #[cfg(feature = "backend-zpaq")] m.add_class::()?; #[cfg(feature = "backend-rwkv")] m.add_class::()?; - m.add_class::()?; m.add_class::()?; + #[cfg(feature = "aixi-gameengine")] m.add_class::()?; - m.add_class::()?; + #[cfg(feature = "aixi-gameengine")] m.add_class::()?; + #[cfg(feature = "aixi-gameengine")] + m.add_class::()?; + #[cfg(feature = "aixi-gameengine")] m.add_class::()?; + #[cfg(feature = "aixi-gameengine")] m.add_class::()?; - m.add_class::()?; + #[cfg(feature = "aixi-gameengine")] + m.add_class::()?; + #[cfg(feature = "aixi-gameengine-physics")] + m.add_class::()?; + #[cfg(feature = "backend-rosa")] m.add_class::()?; + #[cfg(feature = "backend-rosa")] m.add_class::()?; #[cfg(feature = "vm")] @@ -5389,25 +5147,8 @@ fn _core(_py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(rate_backend, m)?)?; m.add_function(wrap_pyfunction!(validate_zpaq_rate_method, m)?)?; m.add_function(wrap_pyfunction!(get_compressed_size, m)?)?; - m.add_function(wrap_pyfunction!(get_compressed_size_parallel, m)?)?; m.add_function(wrap_pyfunction!(get_bytes_from_paths, m)?)?; m.add_function(wrap_pyfunction!(get_compressed_sizes_from_paths, m)?)?; - m.add_function(wrap_pyfunction!( - get_sequential_compressed_sizes_from_sequential_paths, - m - )?)?; - m.add_function(wrap_pyfunction!( - get_parallel_compressed_sizes_from_sequential_paths, - m - )?)?; - m.add_function(wrap_pyfunction!( - get_sequential_compressed_sizes_from_parallel_paths, - m - )?)?; - m.add_function(wrap_pyfunction!( - get_parallel_compressed_sizes_from_parallel_paths, - m - )?)?; m.add_function(wrap_pyfunction!(compress_size_backend, m)?)?; m.add_function(wrap_pyfunction!(compress_size_chain_backend, m)?)?; m.add_function(wrap_pyfunction!(compress_bytes_backend, m)?)?; @@ -5429,29 +5170,29 @@ fn _core(_py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(ncd_matrix_bytes, m)?)?; m.add_function(wrap_pyfunction!(ncd_matrix_paths_with_backend, m)?)?; m.add_function(wrap_pyfunction!(ncd_matrix_bytes_with_backend, m)?)?; - m.add_function(wrap_pyfunction!(marginal_entropy_bytes, m)?)?; + m.add_function(wrap_pyfunction!(empirical_entropy_bytes, m)?)?; m.add_function(wrap_pyfunction!(entropy_rate_bytes, m)?)?; m.add_function(wrap_pyfunction!(entropy_rate_backend, m)?)?; m.add_function(wrap_pyfunction!(biased_entropy_rate_bytes, m)?)?; m.add_function(wrap_pyfunction!(biased_entropy_rate_backend, m)?)?; - m.add_function(wrap_pyfunction!(joint_marginal_entropy_bytes, m)?)?; + m.add_function(wrap_pyfunction!(empirical_joint_entropy_bytes, m)?)?; m.add_function(wrap_pyfunction!(joint_entropy_rate_bytes, m)?)?; m.add_function(wrap_pyfunction!(joint_entropy_rate_backend, m)?)?; m.add_function(wrap_pyfunction!(conditional_entropy_rate_bytes, m)?)?; m.add_function(wrap_pyfunction!(conditional_entropy_bytes, m)?)?; m.add_function(wrap_pyfunction!(mutual_information_bytes, m)?)?; - m.add_function(wrap_pyfunction!(mutual_information_marg_bytes, m)?)?; + m.add_function(wrap_pyfunction!(empirical_mutual_information_bytes, m)?)?; m.add_function(wrap_pyfunction!(mutual_information_rate_bytes, m)?)?; m.add_function(wrap_pyfunction!(mutual_information_rate_backend, m)?)?; m.add_function(wrap_pyfunction!(ned_bytes, m)?)?; - m.add_function(wrap_pyfunction!(ned_marg_bytes, m)?)?; + m.add_function(wrap_pyfunction!(empirical_ned_bytes, m)?)?; m.add_function(wrap_pyfunction!(ned_rate_bytes, m)?)?; m.add_function(wrap_pyfunction!(ned_rate_backend, m)?)?; m.add_function(wrap_pyfunction!(ned_cons_bytes, m)?)?; - m.add_function(wrap_pyfunction!(ned_cons_marg_bytes, m)?)?; + m.add_function(wrap_pyfunction!(empirical_ned_cons_bytes, m)?)?; m.add_function(wrap_pyfunction!(ned_cons_rate_bytes, m)?)?; m.add_function(wrap_pyfunction!(nte_bytes, m)?)?; - m.add_function(wrap_pyfunction!(nte_marg_bytes, m)?)?; + m.add_function(wrap_pyfunction!(empirical_nte_bytes, m)?)?; m.add_function(wrap_pyfunction!(nte_rate_bytes, m)?)?; m.add_function(wrap_pyfunction!(nte_rate_backend, m)?)?; m.add_function(wrap_pyfunction!(tvd_bytes, m)?)?; @@ -5495,6 +5236,7 @@ fn _core(_py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(run_agent_with_environment, m)?)?; m.add_function(wrap_pyfunction!(run_aiqi_with_environment, m)?)?; m.add_function(wrap_pyfunction!(search_with_simulator, m)?)?; + #[cfg(feature = "backend-rosa")] m.add_function(wrap_pyfunction!(search, m)?)?; m.add_function(wrap_pyfunction!(vm_enabled, m)?)?; Ok(()) @@ -5503,10 +5245,20 @@ fn _core(_py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { #[cfg(test)] mod tests { use super::*; + use std::sync::Once; + + fn with_python_initialized(f: F) -> R + where + F: for<'py> FnOnce(Python<'py>) -> R, + { + static PYTHON_INIT: Once = Once::new(); + PYTHON_INIT.call_once(Python::initialize); + Python::attach(f) + } #[test] fn parse_observation_key_mode_accepts_pyclass_instance() { - Python::attach(|py| { + with_python_initialized(|py| { let mode_obj = Py::new( py, PyObservationKeyMode { @@ -5522,7 +5274,7 @@ mod tests { #[test] fn parse_observation_key_mode_accepts_string_aliases() { - Python::attach(|py| { + with_python_initialized(|py| { let stream_hash = pyo3::types::PyString::new(py, "stream_hash"); let parsed_hash = PyAgentSimulatorShim::parse_key_mode(stream_hash.as_any()); assert_eq!( @@ -5530,46 +5282,22 @@ mod tests { infotheory::aixi::common::ObservationKeyMode::StreamHash ); - let stream_hash_hyphen = pyo3::types::PyString::new(py, "stream-hash"); - let parsed_hash_hyphen = - PyAgentSimulatorShim::parse_key_mode(stream_hash_hyphen.as_any()); - assert_eq!( - parsed_hash_hyphen, - infotheory::aixi::common::ObservationKeyMode::StreamHash - ); - - let full_stream = pyo3::types::PyString::new(py, "fullstream"); + let full_stream = pyo3::types::PyString::new(py, "full_stream"); let parsed_full = PyAgentSimulatorShim::parse_key_mode(full_stream.as_any()); assert_eq!( parsed_full, infotheory::aixi::common::ObservationKeyMode::FullStream ); - - let full_stream_hyphen = pyo3::types::PyString::new(py, "full-stream"); - let parsed_full_hyphen = - PyAgentSimulatorShim::parse_key_mode(full_stream_hyphen.as_any()); - assert_eq!( - parsed_full_hyphen, - infotheory::aixi::common::ObservationKeyMode::FullStream - ); - - let full = pyo3::types::PyString::new(py, "full"); - let parsed_full_alias = PyAgentSimulatorShim::parse_key_mode(full.as_any()); - assert_eq!( - parsed_full_alias, - infotheory::aixi::common::ObservationKeyMode::FullStream - ); }); } #[test] - fn parse_framing_mode_accepts_aliases() { + fn parse_framing_mode_accepts_canonical_names() { let raw = parse_framing_mode("raw").expect("raw framing"); let framed = parse_framing_mode("framed").expect("framed framing"); - let framed_alias = parse_framing_mode("frame").expect("frame alias"); assert_eq!(raw, infotheory::compression::FramingMode::Raw); assert_eq!(framed, infotheory::compression::FramingMode::Framed); - assert_eq!(framed_alias, infotheory::compression::FramingMode::Framed); + assert!(parse_framing_mode("frame").is_err()); assert!(parse_framing_mode("nope").is_err()); } @@ -5634,6 +5362,31 @@ mod tests { } } + #[cfg(feature = "backend-rwkv")] + #[test] + fn compression_backend_rwkv7_constructor_uses_shared_cfg_lowering() { + let backend = PyCompressionBackend::rwkv7( + Some( + "cfg:hidden=64,intermediate=64,layers=1,train=sgd,lr=0.01;policy:schedule=0..100:infer" + .to_string(), + ), + "ac", + ) + .expect("rwkv7 constructor"); + + match backend.inner { + CompressionBackend::Rate { + rate_backend: RateBackend::Rwkv7Method { .. }, + coder, + framing, + } => { + assert_eq!(coder, infotheory::coders::CoderType::AC); + assert_eq!(framing, infotheory::compression::FramingMode::Framed); + } + _ => panic!("expected rate-coded rwkv7 backend"), + } + } + #[test] fn file_roundtrip_backend_forces_rate_framed() { let backend = CompressionBackend::Rate { @@ -5649,4 +5402,43 @@ mod tests { _ => panic!("expected rate backend"), } } + + #[cfg(all(feature = "aixi-gameengine", not(feature = "aixi-gameengine-physics")))] + #[test] + fn new_gameengine_builtin_maps_missing_feature_to_py_runtime_error() { + with_python_initialized(|_py| { + let err = match new_gameengine_builtin( + infotheory::spec::BuiltinEnvironmentSpec::Platformer, + Some(0), + ) { + Ok(_) => panic!("missing optional builtin feature must surface as a Python error"), + Err(err) => err, + }; + let rendered = err.to_string(); + assert!(rendered.contains("RuntimeError")); + assert!(rendered.contains("requires feature 'aixi-gameengine-physics'")); + }); + } + + #[test] + fn search_with_simulator_rejects_zero_action_alphabet_at_python_boundary() { + with_python_initialized(|py| { + let simulator = py + .eval( + pyo3::ffi::c_str!("type('BadSim', (), {'get_num_actions': lambda self: 0})()"), + None, + None, + ) + .expect("construct zero-action simulator") + .unbind(); + let strategy = PyMctsStrategy::parallel_uct(2, None).expect("valid strategy"); + let err = search_with_simulator(py, simulator, vec![0], 0, 0, 1, Some(&strategy)) + .expect_err("zero-action simulator must be rejected"); + let rendered = err.to_string(); + assert!( + rendered.contains("AgentSimulator.get_num_actions() must be >= 1"), + "unexpected error message: {rendered}" + ); + }); + } } diff --git a/docs/cli.md b/docs/cli.md new file mode 100644 index 00000000..0283affa --- /dev/null +++ b/docs/cli.md @@ -0,0 +1,157 @@ +# CLI Reference + +The `infotheory` binary is available only when the `cli` feature is enabled: + +```bash +cargo build -p infotheory --release --features cli --bin infotheory --locked +``` + +For local development, the same commands can be run through Cargo: + +```bash +cargo run -p infotheory --features cli -- [args...] [options] +``` + +## Help + +The top-level help is intentionally short: + +```bash +infotheory --help +``` + +Use topic help for details: + +```bash +infotheory help backends +infotheory help compression +infotheory help generation +infotheory help batch +infotheory help aixi +infotheory help warmstart +infotheory help tune +infotheory help diagnostics +infotheory help sequitur +infotheory help search +``` + +Representative command help also routes to the matching topic: + +```bash +infotheory ncd --help +infotheory generate --help +infotheory warmstart --help +infotheory tune --help +``` + +## Metrics + +Single-file metrics: + +```bash +infotheory h README.md +infotheory h_rate README.md --rate-backend ctw --method 32 +infotheory id README.md --rate-backend fac-ctw --method 32 --msb-first +``` + +Two-file metrics: + +```bash +infotheory mi a.bin b.bin +infotheory mi a.bin b.bin --rate-backend ctw --method 16 +infotheory ned a.bin b.bin +infotheory nte a.bin b.bin +infotheory kl a.bin b.bin +infotheory js a.bin b.bin +``` + +`h` and the empirical two-file metrics use order-0 byte statistics unless a rate +backend is explicitly selected. `h_rate` always uses the active rate backend. + +## Backend Selection + +Common backend options: + +```bash +--rate-backend +--compression-backend +--method +--rate-backend-json +--compression-backend-json +--expert-spec +--model-export +``` + +Use `infotheory help backends` for the feature-dependent backend list reported +by the binary you built. + +Canonical JSON inputs are preferred when a backend is produced by another +InfoTheory tool, especially tuner output. Relative asset paths inside canonical +backend JSON resolve against the JSON file's directory. + +## Compression + +```bash +infotheory ncd a.bin b.bin --compression-backend zpaq --method 5 +infotheory ncd a.bin b.bin --compression-backend rate-ac --rate-backend ctw --method 16 +infotheory compress in.bin out.itc --compression-backend rate-rans --rate-backend fac-ctw --method 32 +infotheory decompress out.itc restored.bin --compression-backend rate-rans --rate-backend fac-ctw --method 32 +``` + +Use `rate-ac` or `rate-rans` when the compressor should be driven by a +predictive `RateBackend`. + +## Generation + +```bash +cat prompt.txt | infotheory generate --rate-backend ctw --method 32 --bytes 8 +infotheory generate prompt.txt --rate-backend match --bytes 16 --sample --seed 7 +``` + +Generation supports greedy or sampled continuation, top-k/top-p filtering, and +adaptive continuation. See `infotheory help generation`. + +## Spec-Driven Commands + +`aixi` executes canonical `planner_run` documents: + +```bash +infotheory aixi configs/aixi/paper_kuhn_poker.json +``` + +Legacy pre-1.2 AIXI JSON configs are rejected. Convert them with: + +```bash +./projman.sh legacy_aixi_convert +``` + +`warmstart` tools export, convert, and merge exact-J_H teacher datasets: + +```bash +infotheory warmstart teacher planner-run --target target.json --teacher teacher.json --out teacher-dataset.json +infotheory warmstart teacher from-jsonl --target target.json --jsonl run.jsonl --out teacher-dataset.json +infotheory warmstart teacher merge --target target.json --out merged.json --teacher a.json --teacher b.json +``` + +`tune` executes canonical `tune` JSON or binary spec documents: + +```bash +infotheory tune examples/tuner/strict-smoke-spec.json --max-evaluations 1 +``` + +The tuner has many executor and certificate controls; keep those in topic help +and tuner docs rather than top-level help. + +## Diagnostics + +```bash +RAYON_NUM_THREADS=4 infotheory ac-log-loss corpus.bin \ + --mixture configs/bench/mixture.json \ + --out-prefix /tmp/mixture-diagnostic + +infotheory sequitur-debug --hex 616263616263 --alphabet-prefix 8 +``` + +`ac-log-loss` writes `.trace.tsv`, `.nodes.tsv`, and +`.summary.tsv`. `ctw-profile` is available only when the binary is built +with `backend-ctw research-tooling`. diff --git a/docs/developer-canonical-pipeline-invariants.md b/docs/developer-canonical-pipeline-invariants.md new file mode 100644 index 00000000..67939dbf --- /dev/null +++ b/docs/developer-canonical-pipeline-invariants.md @@ -0,0 +1,130 @@ +# Canonical Pipeline Invariants + +This document defines the hard semantic boundary for spec/runtime flow in Infotheory. + +## Normative Pipeline + +The only valid semantic flow is: + +1. Load/parse (JSON/binary wrapper form) +2. Validate/canonicalize +3. Lower/compile into runtime-ready plans +4. Execute via runtime builders + +Executable runtime objects must be derived from canonical lowered plans, not from ad-hoc +raw strings or partially interpreted intermediate representations. + +## Scope Boundaries For Refactor Series + +In-scope: + +- `spec/*` canonicalization, parsing, document, and plan-lowering boundaries +- `runtime/*` dispatch and runtime construction boundaries +- Rust API / CLI / Python route consistency +- converter alias parity with runtime registry authority + +Out-of-scope: + +- backend algorithmic mathematics +- vendor crate internals (`vendor/*`) except required feature wiring/compatibility + +## Bypass Inventory (Tracked Targets) + +The following internal areas are tracked as bypass-sensitive and must not be expanded: + +- `crates/infotheory/src/spec/core.rs` + - `compiled_rate_backend_from_plan` and `compiled_compression_backend_from_plan` + - internal unchecked helpers must remain private and never be exposed outside `spec::core` +- `crates/infotheory/src/runtime/mod.rs` + - runtime constructors for calibrated/rate wrappers must consume checked compiled plans +- `crates/infotheory/src/aixi/agent.rs` + `crates/infotheory/src/aixi/aiqi.rs` + - planner spec construction must route through shared builder utilities and canonical spec + compilation path + +## Error Semantics + +Document loading must remain format-directed and truthful: + +- `.json` -> JSON parse only +- `.itsd` or magic envelope -> binary parse +- no JSON->binary blind fallback that obscures root parse errors + +## Planner-Run Ownership Boundary + +Planner-run execution has a single, layered ownership split. Each concern lives in exactly +one place and downstream layers must not re-implement upstream responsibilities. + +- `crates/infotheory/src/aixi/planner_agent.rs` owns *execution*: controller construction + (`PlannerControllerAgent`), the cycle/phase state machine (`PlannerRunSession`, + `PlannerSchedule`, `PlannerPhase`), the observer trait (`PlannerCycleObserver`, + `PlannerActionProvenance`), and per-cycle outcomes. Nothing here reads the filesystem or + interprets CLI overlays. +- `crates/infotheory/src/aixi/planner_runtime.rs` owns *asset and environment binding*: + resolving `planner_run` documents to compiled plans (`compile_planner_run_document`), + building environments (`build_planner_environment`), and loading/validating warm-start + teacher datasets against a compiled plan. This is the only layer that materializes a + teacher dataset from a resolved asset. +- `crates/infotheory/src/cli/planner_run.rs` owns the *CLI adapter*: it reads the config + path, rejects legacy non-canonical JSON and legacy interface reward-range fields, drives + `run_compiled_planner_run`/`run_aixi_mode`, and renders trace/telemetry output + (`AixiRunLogger`, `PlannerCliObserver`). `crates/infotheory/src/main.rs` only dispatches + the `aixi` subcommand into this adapter; it must not host planner-run execution helpers, + the run logger, or canonical-document shape checks. + +The CLI layer depends on the planner-agent and planner-runtime facades; the dependency +never flows the other way. New planner-run surfaces (including warm-start variants) must +slot into this same three-layer split rather than re-deriving runtime behavior from raw +strings or partially interpreted JSON. + +## Warm-Start Exact-J_H Invariants + +Normative semantics for `aiqi_warmstart_exact_jh`, standalone teacher artifacts, +JSONL conversion, deterministic teacher-trace merge, and task fingerprint binding +live in `docs/warmstart-exact-jh.tex`. The checklist below records implementation +anchors that must stay aligned with that document. + +### Task Fingerprint Binding + +The warm-start task fingerprint +(`warmstart_exact_jh_planner_task_fingerprint` in +`crates/infotheory/src/aixi/warmstart_contract.rs`) is computed by *stripping the teacher +itself out of the task identity* before hashing. Concretely, the fingerprint payload +commits to: + +- `planner_run_task_sha256`: a SHA-256 digest over the canonical planner-run JSON with the + teacher removed from both `assets` and the controller's `teacher_dataset_asset` field; +- the controller kind and backend label; +- the teacher-contract schema version; +- `task_asset_content_sha256`: SHA-256 content commitments for every resolved asset *except* the + teacher dataset asset. + +Invariant: the fingerprint must be invariant to the teacher dataset asset's path and byte +content, and must bind to the content of every non-teacher (`task_input`) asset. This is +what lets a teacher dataset legitimately commit to the fingerprint of the task it teaches +without creating a self-reference. Any change that folds teacher path/content back into the +fingerprint, or that drops a non-teacher asset's content from it, is a regression and is +guarded by the fingerprint golden/asset-mutation tests in +`crates/infotheory/src/aixi/warmstart_contract.rs`. + +### Validation Boundary + +The teacher contract and the full teacher dataset are validated at distinct points: + +- `validate_warmstart_exact_jh_teacher_contract` (in `planner_runtime.rs`) is + *contract-only*: it checks the dataset's declared contract (interface, adapter, scalar, + task fingerprint) against a compiled planner run via + `validate_warmstart_teacher_against_compiled_planner_run`. It does not validate trace + records. +- `load_warmstart_exact_jh_teacher_dataset` runs *full dataset validation* + (`validate_warmstart_teacher_dataset_for_compiled_planner_run`) after parsing the asset + bytes, i.e. contract plus every trace/transition, against the compiled plan. +- Tuner ingestion and the trace-refresh merge path likewise run full dataset validation + after the relevant planner run is compiled; the trace-refresh merge counter + (`warmstart_trace_refresh_merges`) increments only for structurally inserted traces. The + self-improvement report marks `same_task_trace_refresh_rebuild` when same-task trace + refresh is the active self-improvement mode; the actual warm-start agent rebuild remains + insertion-aware and only happens when a merge changed the dataset. + +Invariant: full dataset validation always happens against a *compiled* planner run, never +against raw JSON, and the contract-only check must never be silently substituted for full +dataset validation on a path that ingests teacher traces. diff --git a/docs/developer-testing.md b/docs/developer-testing.md index 2328744e..9b5c3cb1 100644 --- a/docs/developer-testing.md +++ b/docs/developer-testing.md @@ -7,18 +7,18 @@ coverage workflows. ```bash # Rust tests (default features) -cargo test --locked +cargo test -p infotheory --locked -# Rust tests with CLI enabled (includes CLI/API parity + search tests) -cargo test --features cli --locked +# Rust CLI + broad backend parity pass +cargo test -p infotheory --no-default-features --features "cli all-backends" --locked # VM-focused Rust tests -cargo test --features vm --test nyx_vm_tests --locked +cargo test -p infotheory --no-default-features --features "vm backend-ctw" --locked ``` ```bash -# Build Python extension in editable mode -uv run maturin develop --release +# Build Python extension in editable mode using the repo's pyproject/maturin config +uv run maturin develop # Python tests uv run pytest -q python/tests @@ -38,7 +38,7 @@ uv run pytest \ crate (library + CLI tests). ```bash -cargo llvm-cov -p infotheory --tests --features cli --locked --summary-only +cargo llvm-cov -p infotheory --tests --features "cli all-backends" --locked --summary-only ``` CI enforces a minimum line coverage threshold for this command. @@ -53,7 +53,7 @@ cargo +nightly rustdoc -p infotheory --all-features -- \ > /tmp/rustdoc_cov.json ``` -CI enforces a minimum documented-item percentage using this report. +CI currently enforces full documented-item coverage (100%) using this report. ## Golden and Parity Tests @@ -65,20 +65,57 @@ The suite includes: `calibrated`, `mamba`, and `rwkv7` string parsing (`python/tests/test_api_surface.py`) - Compression/decompression roundtrip checks in Rust and Python - VM stats-backend parsing and predictor-backed trace-model coverage for the new - backends (`src/main.rs`, `src/aixi/vm_nyx.rs`) -- Deterministic fixture hash checks (`tests/roundtrip_hashes.rs`, `python/tests/test_golden_hashes.py`) -- RWKV method parsing/canonicalization tests (`tests/rwkv_method_canonicalization.rs`) + backends (`crates/infotheory/src/main.rs`, `crates/infotheory/src/aixi/vm_nyx.rs`) +- Deterministic fixture hash checks (`crates/infotheory/tests/roundtrip_hashes.rs`, `python/tests/test_golden_hashes.py`) +- RWKV method parsing/canonicalization tests (`crates/infotheory/tests/rwkv_method_canonicalization.rs`) These tests are designed to catch semantic drift and output regressions across interfaces. +## Local CI Preflight + +For a local CI-like pass, prefer the project wrapper: + +```bash +./projman.sh test_ci +``` + +Useful controls: + +- `INFOTHEORY_BUILD_MODE=native|portable` +- `INFOTHEORY_CI_INCLUDE_VM=1` +- `INFOTHEORY_CI_SKIP_RUST_LINE_COVERAGE=1` +- `INFOTHEORY_CI_SKIP_RUSTDOC_COVERAGE=1` +- `INFOTHEORY_CI_SKIP_FEATURE_GATES=1` +- `INFOTHEORY_CI_SKIP_PYTHON=1` + +Avoid indiscriminate workspace all-features sweeps; they pull in heavyweight +optional surfaces that are intentionally tested through curated CI slices. + +## Benchmark Provenance Checks + +The `two-json` benchmark suite is pinned to the historical canonical +`configs/bench/two.json` / `examples/two.json` spec with `alpha = 0.03`. + +The benchmark harness and comparator now enforce provenance: + +- `scripts/bench_two_json.sh` records the resolved suite-spec path, suite-spec + SHA-256 digest, build mode, and build features in raw and summary TSVs. +- `scripts/compare_bench_two_json.lua` rejects baseline/current comparisons when + the suite-spec digests differ. +- Rust and Python tests assert that the checked-in `two.json` benchmark specs + stay byte-identical and preserve the historical `alpha = 0.03` setting. + +This is the guardrail against benchmark-subject drift being mistaken for a code +regression. + ## MC-AIXI Competitor Benchmark Validation Use the reproducible benchmark harness to validate cross-implementation parity for MC-AIXI behavior and reporting: ```bash -./projman.sh bench__aixi_competitors --profile default --trials 1 +./projman.sh bench_aixi_competitors --profile default --trials 1 ``` Parity/correctness expectations for this benchmark: diff --git a/docs/infotheory-tuner-v1.tex b/docs/infotheory-tuner-v1.tex new file mode 100644 index 00000000..6110ce79 --- /dev/null +++ b/docs/infotheory-tuner-v1.tex @@ -0,0 +1,4755 @@ + +\documentclass[11pt]{article} +\usepackage[margin=1in]{geometry} + +% Core packages +\usepackage{amsmath,amssymb,amsthm,mathtools} +\usepackage{tikz-cd} +\usepackage{multicol} +\usepackage[T1]{fontenc} +\usepackage{lmodern} +\usepackage{hyperref} + +% Paragraphs +\setlength{\parindent}{0pt} +\setlength{\parskip}{0.7\baselineskip} + +\newtheorem{definition}{Definition} +\newtheorem{theorem}{Theorem} +\newtheorem{proposition}{Proposition} +\newtheorem{corollary}{Corollary} +\newtheorem{remark}{Remark} +\newtheorem{assumption}{Assumption} + +\title{Constrained Dataset Predictive Tuning:\\A Formal Algorithmic Specification} +\author{Noah Cashin} +\date{\today} + +\begin{document} +\maketitle + +\section{Purpose and Scope}\label{sec:purpose-scope} + +This document specifies a constrained predictive/compression tuning capability +for a fixed dataset. The dataset may be either a passive-byte string or a +canonical causal prediction dataset whose context and target channels are typed +and evaluator-normalized. +The specification is intended to be: +\begin{itemize} + \item mathematically precise, + \item directly implementable in \texttt{infotheory}, + \item optimizer-agnostic at the interface layer, + \item semantically unified across passive compression and active + environment-model tuning by one causal-conditional evaluator, + \item runtime-selectable between annealed hill climbing, + MC-AIXI(FAC-CTW), discounted AIQI, and a distinct exact + warm-start controller, and + \item mathematically explicit about objective alignment in each controller path. +\end{itemize} + +This capability is \emph{not} an unconstrained architecture search engine or a +replacement for normal compression execution. It is a bounded optimization +wrapper over the same compression semantics. + +\section{Formal Problem Statement}\label{sec:formal-problem} + +\subsection{Objects} + +Let: +\begin{itemize} + \item $\mathcal{A}_{\mathrm{doc}}$ denote the finite asset-binding map + carried by the validated top-level tune document, + \item $d$ denote the canonical dataset object loaded from the asset + identified by \texttt{input\_asset} through + $\mathcal{A}_{\mathrm{doc}}$, + \item $x \in \{0,\dots,255\}^{n}$ denote the passive-byte specialization + of $d$ when the validated dataset object is classified as passive-byte + data, + \item $N_H(d)\in\mathbb{R}_{>0}$ denote the profile-fixed dataset-size + functional used for throughput normalization, + \item $\mathcal{Z}$ denote the full configuration space of canonical + \texttt{CompressionBackend} candidates admitted by the tuning grammar, + \item $B$ denote the user-provided bounds specification, + \item $H$ denote the fixed deployability profile (hardware/evaluator methodology and resource accounting), + \item $z_0 \in \mathcal{Z}$ denote the validated baseline candidate + embedded in \texttt{baseline\_candidate}, + \item $T_{\mathrm{eval}} > 0$ denote the hard per-candidate wall-clock limit, + \item $T_{\mathrm{total}} > 0$ denotes the hard total tuning wall-clock budget, + \item $\theta_{\min}>0$ denote the required minimum throughput floor in + dataset-size units per second, + \item $\mu_{\max}>0$ denote the required memory cap (bytes). +\end{itemize} + +Define the bounded-valid candidate set: +\[ +\mathcal{F}(B,H):=\{z \in \mathcal{Z}:\ \mathrm{Valid}_{B,H}(z)=1\}. +\] + +Each $z\in\mathcal{Z}$ is a canonical compression candidate. Evaluation is +normatively defined by compiling the same candidate through the shared +\texttt{infotheory} backend/spec pipeline used for ordinary compression +execution. When the compiled candidate exposes an explicit predictive +factorization, write $\rho_z$ for the induced chronological predictor. When the +candidate is given by a direct compression path, $\rho_z$ denotes the +candidate's normative predictive/codelength semantics under profile $H$. + +For each candidate $z$, evaluation returns +\[ +M(z) = (\mathrm{status}(z), c_{\mathrm{phys}}(z), \ell(z), \tau(z), \mu(z), \operatorname{diag}(z)), +\] +where +\begin{itemize} + \item $\mathrm{status}(z) \in \{\mathrm{success},\mathrm{timeout},\mathrm{invalid},\mathrm{error}\}$, + \item $c_{\mathrm{phys}}(z)\in\mathbb{N}_0\sqcup\{\bot_c\}$ is the + physical compressed size in bytes of a material encoded artifact when such + an artifact exists under the selected evaluator path, with distinguished + sentinel $\bot_c$ meaning ``not materialized/undefined'', + \item $\ell(z) \in \mathbb{R}_{\ge 0}$ is cumulative negative log-loss in + bits on the charged target stream under the same conditioning semantics; + this is the normative target-loss quantity for optimization, + \item $\tau(z) \in \mathbb{R}_{\ge 0}$ is elapsed wall-clock time, + \item $\mu(z) \in \mathbb{R}_{\ge 0}$ is measured peak memory usage, + \item $\operatorname{diag}(z)$ is optional diagnostics (including invalid-reason tags). +\end{itemize} + +\begin{remark}[Peak-memory measurement semantics] +Each candidate evaluation is executed in its own isolated evaluator instance. + +For strict v1 theorem-facing execution, that isolated evaluator instance is a +fresh spawned-and-exec'd worker process per candidate evaluation +(\texttt{spawn\_exec\_worker}). The parent controller runtime is sequential +(\texttt{parent\_controller\_threads}=1). Optional multithreading is allowed +inside the evaluator worker only, because its runtime/threadpool state is +initialized after process creation. + +For strict v1 theorem-facing memory accounting, $\mu(z)$ must be measured by an +operating-system or container memory controller that accounts for the evaluator +instance over its lifetime. On Linux, the conforming strict profile is +\texttt{cgroup\_v2\_peak\_required}. If the required controller accounting +cannot be configured, the evaluator profile is invalid and tuning must abort +before the normative baseline evaluation. + +Operationally, cgroup-v2 delegation may require a two-level delegated subtree: +a session cgroup that contains the long-lived tuner parent process, and a +separate evaluation-parent cgroup under which per-evaluation worker cgroups are +created. In delegation-containment terms, placing the first delegated process +into the subtree may require an action by the delegator/root side before +unprivileged tuning begins. The strict evaluator parent path passed to +\texttt{--evaluator-cgroup-parent} should name the evaluation-parent node. + +A single-process peak-RSS fallback may be exposed only as an explicit +non-strict execution profile +(\texttt{single\_process\_rss\_fallback}). In that profile, the evaluator must +run as a single process, $\mu(z)$ is the operating-system reported peak RSS of +that process, and provenance must state that strict theorem-facing memory +certification is not claimed. + +Backend-reported memory may be recorded as a diagnostic or as a supplementary +component of a hybrid check, but backend-reported memory alone is not a strict +v1 deployability memory measurement unless separately justified by the +deployability profile. + +Executing complex runtime/library code in a raw post-\texttt{fork()} child +without an \texttt{exec()} boundary is non-conforming for strict v1 +theorem-facing isolation. + +Naive summation of RSS across a multi-process tree is not normative, because it +can double-count shared pages. +\end{remark} + +Timeout semantics are hard relative to the effective evaluator limit +$T_{\mathrm{eff}}$ used for that call: +\[ +\tau(z) \ge T_{\mathrm{eff}} \implies \mathrm{status}(z)=\mathrm{timeout}. +\] +For full per-candidate evaluations, +$T_{\mathrm{eff}}=T_{\mathrm{eval}}$. In remaining-budget-truncated runtime +calls, $T_{\mathrm{eff}}=\min(T_{\mathrm{eval}},B_t)$. + +For successful candidates, define throughput +\[ +\theta(z):= +\begin{cases} +\dfrac{N_H(d)}{\tau(z)}, & \mathrm{status}(z)=\mathrm{success},\ \tau(z)>0,\\[4pt] ++\infty, & \mathrm{status}(z)=\mathrm{success},\ \tau(z)=0. +\end{cases} +\] +Define the equivalent throughput runtime cap +\[ +\tau_{\theta}:=\frac{N_H(d)}{\theta_{\min}}. +\] +Then, for successful candidates, +\[ +\theta(z)\ge\theta_{\min} +\iff +\tau(z)\le\tau_{\theta}. +\] + +\begin{remark}[Passive opening and causal generalization] +The opening problem statement writes the operational objective first for the +passive-byte specialization, where $d=x$ and $N_H(d)=n$, so throughput is +measured in bytes/s. Section~\ref{sec:causal-evaluator} gives the strictly more +general causal-conditional evaluator, under which passive-byte tuning is +recovered as the singleton-action, target-only special case. +\end{remark} + +\subsection{Primary Objective} + +Define model-code length by canonical bytes +\[ +L_B(z):=\big|\operatorname{ser}(\operatorname{canon}(z))\big|. +\] + +\begin{remark}[Candidate code length versus outer request metadata] +For the canonical tune document, $L_B(z)$ is measured on the canonical bytes +of the candidate object $z$ alone, equivalently the canonical bytes of the +embedded \texttt{CompressionBackend}. Top-level request metadata such as +\texttt{output\_config\_path}, \texttt{report\_path}, asset bindings, dataset +identifiers, and executor-side controls do not contribute to $L_B(z)$. +\end{remark} + +Define the deployable set +\[ +\mathcal{F}^{\mathrm{dep}}(B,H):= +\left\{z\in\mathcal{F}(B,H):\ \mathrm{status}(z)=\mathrm{success}\ \land\ \theta(z)\ge\theta_{\min}\ \land\ \mu(z)\le\mu_{\max}\right\}. +\] + +The normative objective is the totalized deployable two-part MDL criterion in +bits +\[ +\mathcal{J}_H(z;d)= +\begin{cases} +8L_B(z)+\ell(z), & z\in\mathcal{F}^{\mathrm{dep}}(B,H),\\[3pt] ++\infty, & \text{otherwise.} +\end{cases} +\] + +The primary optimization problem is +\[ +\min_{z\in\mathcal{F}(B,H)}\mathcal{J}_H(z;d). +\] + +\begin{remark}[Operational units and reporting] +The optimizer uses the bit-valued objective $\mathcal{J}_H(z;d)=8L_B(z)+\ell(z)$. +Model-code length $L_B(z)$ is still reported in bytes. Physical compressed +bytes $c_{\mathrm{phys}}(z)$ are reporting-only and may be absent +($\bot_c$) for evaluators that compute target log-loss without materializing a +byte artifact. In passive exact-encoding paths, one may additionally report the +finite-coder relation $8c_{\mathrm{phys}}(z)=\ell(z)+\varepsilon_{\mathrm{code}}(z)$, +but this identity is not normative for general causal datasets. +\end{remark} + +\subsection{Deterministic Incumbent Key and Output Semantics} + +For each evaluated candidate $z$, define the deterministic key +\[ +K(z):=\big(\mathcal{J}_H(z;d),\operatorname{ser}(\operatorname{canon}(z))\big), +\] +ordered lexicographically, where $\operatorname{canon}:\mathcal{Z}\to\mathcal{Z}$ +and $\operatorname{ser}:\mathcal{Z}\to\{0,\dots,255\}^*$ are defined in +Section~\ref{sec:canon-cache}. + +For byte strings, use standard lexicographic byte order with strict-prefix +rule: $u\prec_{\mathrm{bytes}} v$ if either (i) at the first differing byte, +$u$ has the smaller byte, or (ii) $u$ is a strict prefix of $v$. + +Let +\[ +\mathcal{S}_t:=\left\{z:\ z\text{ has been evaluated or cache-reused by step }t\ \text{and }z\in\mathcal{F}^{\mathrm{dep}}(B,H)\right\}. +\] +The incumbent is +\[ +b_t:=\arg\min_{z\in\mathcal{S}_t}K(z). +\] +Write $\prec_K$ for lexicographic order on key tuples, i.e. +$K(z_1)\prec_K K(z_2)$ iff the tuple of $z_1$ is lexicographically smaller than that of $z_2$. + +\begin{proposition}[Uniqueness of incumbent under finite deployable evaluated set] +If $\mathcal{S}_t$ is finite and nonempty, then $b_t$ exists and is unique. +\end{proposition} + +\begin{proof} +By construction, $\prec_{\mathrm{bytes}}$ is a total order on +$\{0,\dots,255\}^*$. Hence lexicographic order on +$\big(\mathbb{R}_{\ge 0}\sqcup\{+\infty\}\big)\times\{0,\dots,255\}^*$ is total. A finite +nonempty totally ordered set has a unique minimum. +\end{proof} + +This key preserves the primary optimization semantics: $\mathcal{J}_H$ is +primary, and deterministic canonical bytes are a tie-break only. + +\section{Input Contract and Defaults}\label{sec:input-contract} + +\subsection{Canonical tune document} + +The normative top-level request is a validated \texttt{SpecDocument::Tune} +document. It may be serialized as canonical JSON or as the versioned binary +\texttt{itsd} envelope; both encodings denote the same tune request. + +Required top-level fields are: +\begin{itemize} + \item \texttt{schema\_version: 1} + \item \texttt{kind: "tune"} + \item \texttt{assets: [AssetBinding]} (possibly empty, but required when + an asset identifier is referenced) + \item \texttt{input\_asset: AssetId} + \item \texttt{baseline\_candidate: CompressionBackend} + \item \texttt{controller: TuneControllerSpec} + \item \texttt{bounds: TuneBoundsSpec} + \item \texttt{eval\_time\_limit\_seconds: f64} + \item \texttt{time\_budget\_seconds: f64} + \item \texttt{min\_throughput\_bytes\_per\_second: f64} with constraint $>0$ + \item \texttt{max\_memory\_bytes: u64} with constraint $>0$ + \item \texttt{output\_config\_path: Path} + \item \texttt{seed: u64} +\end{itemize} + +Optional top-level fields are: +\begin{itemize} + \item \texttt{report\_path: Option} +\end{itemize} + +Each \texttt{AssetBinding} is a pair \texttt{(id, path)}. Relative asset paths +are resolved against the tune document base directory; absolute paths are +preserved. + +Define the throughput floor parameter by +\[ +\theta_{\min}:=\texttt{min\_throughput\_bytes\_per\_second}\in\mathbb{R}_{>0}. +\] + +\begin{remark}[Legacy passive-byte field name] +The input field name \texttt{min\_throughput\_bytes\_}\allowbreak\texttt{per\_second} uses legacy +passive-byte terminology. Semantically, $\theta_{\min}$ is measured in the same +dataset-size units per second as $N_H(d)$; in passive-byte mode, $N_H(d)=n$ +and the units are bytes/s. +\end{remark} + +Define the memory-cap parameter by +\[ +\mu_{\max}:=\texttt{max\_memory\_bytes}\in\mathbb{R}_{>0}. +\] + +\subsection{Derived semantic classifiers and controller kinds} + +The canonical tune document does \emph{not} carry independent +\texttt{dataset\_kind}, \texttt{objective\_target}, or +\texttt{planner\_reward\_mode} fields. These are derived semantic classifiers +after document validation. + +Define the internal dataset classifier by +\[ +\operatorname{DatasetKind}(d)\in +\left\{ +\begin{aligned} +&\texttt{"passive\_bytes"},\ \texttt{"interactive\_trace"},\\ +&\texttt{"causal\_prefix\_dataset"} +\end{aligned} +\right\}. +\] + +Define the derived evaluator-target classifier by +\[ +\operatorname{ObjectiveTarget}(d,H)\in +\left\{ +\begin{aligned} +&\texttt{"passive\_ac"},\\ +&\texttt{"interactive\_causal\_ac"},\\ +&\texttt{"planner\_}\allowbreak\texttt{deployable\_}\allowbreak\texttt{model"} +\end{aligned} +\right\}. +\] + +Compatibility and derivation rules are normative: +\begin{itemize} + \item if \texttt{input\_asset} resolves to a raw byte string, then + $\operatorname{DatasetKind}(d)=\texttt{"passive\_bytes"}$ and the default + derived evaluator target is \texttt{"passive\_ac"}, + \item if \texttt{input\_asset} resolves to a canonical typed causal trace, + then + $\operatorname{DatasetKind}(d)=\texttt{"interactive\_trace"}$ and the + default derived evaluator target is \texttt{"interactive\_causal\_ac"}, + \item if \texttt{input\_asset} resolves to a canonical finite causal + prediction dataset, then + $\operatorname{DatasetKind}(d)=\texttt{"causal\_prefix\_dataset"}$ and the + default derived evaluator target is \texttt{"interactive\_causal\_ac"}, + \item \texttt{controller.kind} selects the search controller only; it does + not by itself change the dataset classifier or the primary evaluator + target, + \item a planner-facing controller may optimize candidates on either passive + or causal datasets, because the planner-facing environment is the tuner + search process over candidates rather than necessarily the external dataset + being scored, + \item the derived evaluator target is upgraded to + \texttt{"planner\_deployable\_model"} only when the fixed deployability + profile $H$ requests planner-deployability constraints for the returned + candidate; this upgrade changes feasibility constraints and diagnostics, + but not the primary objective $8L_B(z)+\ell(z)$. +\end{itemize} + +Define runtime controller kind by +\[ +\texttt{controller.kind}\in +\left\{ +\begin{aligned} +&\texttt{"annealed\_hill\_climbing"},\ \texttt{"mc\_aixi\_fac\_ctw"},\\ +&\texttt{"aiqi\_discounted"},\ \texttt{"aiqi\_warmstart\_exact\_jh"} +\end{aligned} +\right\}. +\] + +Runtime dispatch is normative: +\begin{itemize} + \item if \texttt{controller.kind = "annealed\_hill\_climbing"}, execute + Section~\ref{sec:ahc-runtime}, + \item if \texttt{controller.kind} is + \texttt{mc\_aixi\_fac\_ctw}, \texttt{aiqi\_discounted}, or + \texttt{aiqi\_warmstart\_exact\allowbreak\_jh}, execute + Section~\ref{sec:aixi-runtime}. +\end{itemize} + +Planner-facing controllers additionally carry a +\texttt{PlannerInterfaceSpec} subobject +\[ +\texttt{interface}= +(\texttt{observation\_bits},\texttt{observation\_stream\_len}, +\texttt{observation\_key\_mode},\texttt{reward\_bits}, +\texttt{agent\_actions}), +\] +where: +\begin{itemize} + \item \texttt{agent\_actions} is a non-zero action-alphabet cardinality, + \item \texttt{observation\_key\_mode} is one of + \texttt{"first"}, \texttt{"last"}, \texttt{"stream\_hash"}, or + \texttt{"full\_stream"}, + \item \texttt{reward\_bits} declares the finite encoded reward-alphabet + width; write $b_R:=\texttt{reward\_bits}$. +\end{itemize} + +The shared planner interface does not contain +\texttt{min\_reward}, \texttt{max\_reward}, or +\texttt{reward\_offset}. Those fields belonged to the unreleased affine +integer reward-encoding design, where objective improvements were encoded by a +shifted integer interval. That design is not part of the v1 public surface. + +The field \texttt{observation\_key\_mode} specifies the planner-visible +branching key derived from the encoded observation stream $o_{1:m}$. This +branching key is a runtime search-tree representation, distinct from the fixed +encoded observation stream itself and distinct from the exact-state encoder +$\Psi_H$ of Section~\ref{sec:aixi-embedding}. For the built-in modes, write +\[ +\widehat{\mathcal O}_B:=\{0,\dots,2^{64}-1\}, +\qquad +\mathcal{K}_B^{(L_O)}:=\bigsqcup_{\ell=1}^{L_O}\widehat{\mathcal O}_B^\ell. +\] +Then each built-in \texttt{observation\_key\_mode} deterministically maps a +validated encoded observation stream to an element of +$\mathcal{K}_B^{(L_O)}$: +\begin{itemize} + \item \texttt{"full\_stream"} returns the full stream + $(o_1,\dots,o_m)$ unchanged and is the paper-accurate expectimax mode, + \item \texttt{"first"} returns the singleton key $(o_1)$, or $(0)$ on the + empty stream, + \item \texttt{"last"} returns the singleton key $(o_m)$, or $(0)$ on the + empty stream, + \item \texttt{"stream\_hash"} masks each symbol to the declared + \texttt{observation\_bits}, initializes $h_0:=0$, updates + $h_{i+1}:=\operatorname{rotl}_7(h_i)\oplus(o_i\wedge \mathrm{mask})$, and + returns the singleton key $(h_m)$. +\end{itemize} +Here $\operatorname{rotl}_7$ denotes 64-bit left rotation by seven positions. +For the hash mode, write $b_O:=\texttt{observation\_bits}$; then +$\mathrm{mask}=0$ when $b_O=0$, $\mathrm{mask}=2^{b_O}-1$ when +$1\le b_O<64$, and $\mathrm{mask}=2^{64}-1$ when $b_O\ge 64$. +Under the standard planner interface, a conforming upstream observation adapter +emits exactly $L_O$ encoded observation symbols per external decision step, so +the built-in \texttt{"full\_stream"} mode returns an element of +$\widehat{\mathcal O}_B^{L_O}$ while the other built-in modes return singleton +keys in $\widehat{\mathcal O}_B^1$. All non-\texttt{"full\_stream"} modes are +lossy single-key projections of a possibly longer observation stream. Among the +built-in modes, only \texttt{"full\_stream"} is eligible for the exact-state +observation path by default; any other mode requires a proof that the induced +projection is injective on the planner state space relevant to the theorem +being claimed. The precise observation adapter specification $\eta_O$, +including any upstream encoding of raw environment observations into the stream +consumed here and any fixed-length validation rule, must be recorded in +provenance. This public observation/keying interface does not by itself +certify the exact-state observation path; that requires a theorem-side +encoder $\Psi_H$ together with the injectivity/certification contract stated +later in Section~\ref{sec:aixi-embedding}. + +For exact-objective-difference semantics, the implementation must additionally +fix or derive a finite canonical scalar representation +\[ +\mathbb{V}_H^{\mathcal J} +\] +for exact objective values and objective differences. This representation may +be supplied by fixed evaluator numeric semantics, by an implementation-internal +declaration, or by the deployability profile $H$, but it must be recorded in +provenance, together with an injective reward encoder $\Omega_H$ and semantic +decoder $V_H$. The exact reward semantics are therefore not defined by any +interval/offset triple. + +For \texttt{aiqi\_discounted}, normalized-clipped-improvement semantics are +parameterized by the explicit controller clipping fields +\[ +\begin{aligned} +r_{\min}^{\mathrm{clip}}&:=\texttt{controller.min\_improvement},\\ +r_{\max}^{\mathrm{clip}}&:=\texttt{controller.max\_improvement},\\ +r_{\max}^{\mathrm{clip}}&>r_{\min}^{\mathrm{clip}}. +\end{aligned} +\] +These parameters are specific to the discounted controller. They determine only +the discounted controller's normalization interval and are not part of exact +objective-difference semantics. + +Controller-specific required fields are: +\begin{itemize} + \item for \texttt{controller.kind = "annealed\_hill\_climbing"}: + \texttt{controller.max\_mutation\_radius}\((\in\mathbb{N},\ge 1)\), + \item for \texttt{controller.kind = "mc\_aixi\_fac\_ctw"}: + \texttt{controller.interface} and + \texttt{controller.planner\_simulations\_per\_step}\((\in\mathbb{N},\ge 1)\). + This tuning controller uses the canonical sequential $\rho$UCT path; the + tune document does not expose a separate \texttt{mcts\_strategy} field, + \item for \texttt{controller.kind = "aiqi\_discounted"}: + \texttt{controller.interface}, + \texttt{controller.planner\_simulations\_per\_step}\((\in\mathbb{N},\ge 1)\), + \texttt{controller.return\_horizon}\((\in\mathbb{N},\ge 1)\), + \texttt{controller.return\_bins}\((\in\mathbb{N},\text{power of two})\), + \texttt{controller.discount\_factor}\((\in[0,1))\), + \texttt{controller.min\_improvement}, and + \texttt{controller.max\_improvement} with constraint + \texttt{controller.max\_improvement > controller.min\_improvement}, + \item for \texttt{controller.kind = "aiqi\_warmstart\_exact\allowbreak\_jh"}: + \texttt{controller.interface}, + \texttt{controller.planner\_simulations\_per\_step}\((\in\mathbb{N},\ge 1)\), + \texttt{controller.return\_horizon}\((\in\mathbb{N},\ge 1)\), + \texttt{controller.warmstart\_teacher\_dataset\_asset: AssetId}, + and + \texttt{controller.label\_phase\_period}\((\in\mathbb{N},\ge\texttt{return\_horizon})\). + The canonical tune document does not surface a separate + \texttt{return\_bins} field for this controller; its exact label alphabet + is induced by the exact scalar representation and the declared interface + reward encoding. +\end{itemize} + +Planner reward semantics are derived from the controller family: +\begin{itemize} + \item \texttt{"mc\_aixi\_fac\_ctw"} and + \texttt{"aiqi\_warmstart\_exact\_jh"} induce + \texttt{"exact\_objective\_difference"} semantics, + \item \texttt{"aiqi\_discounted"} induces + \texttt{"normalized\_clipped\_improvement"} semantics. +\end{itemize} + +If an implementation additionally offers a quantized objective-difference +variant for an exact-family planner environment, that choice is an internal +execution-profile declaration recorded in provenance rather than a distinct +top-level tune-document field. + +\subsection{Normative input precondition: deployable baseline}\label{subsec:baseline-precondition} + +After validating the top-level tune document, loading $d$ from +\texttt{input\_asset}, and canonicalizing the embedded baseline candidate +\[ +z_0^{\mathrm{can}}:=\operatorname{canon}(z_0), +\] +the runtime must perform the normative baseline evaluation before entering any +controller loop and require +\[ +\mathrm{status}(z_0^{\mathrm{can}})=\mathrm{success} +\quad\land\quad +\theta(z_0^{\mathrm{can}})\ge\theta_{\min} +\quad\land\quad +\mu(z_0^{\mathrm{can}})\le\mu_{\max}. +\] +If this precondition fails, execution must abort with no optimization steps +performed. This precondition is normative for all controller choices. + +\subsection{Execution controls} + +The canonical tune document above is the normative candidate-facing input +surface. An implementation may additionally expose executor-side controls that +do not change candidate identity, such as: +\begin{itemize} + \item \texttt{max\_evaluations} + \item \texttt{annealer\_kernel\_profile} + \item \texttt{cpu\_affinity} + \item \texttt{threads} (interpreted as evaluator-worker-internal thread count) + \item \texttt{warmup\_baseline\_runs} + \item \texttt{self\_improvement\_rounds} + \item \texttt{stagnation\_reset\_evals} + \item \texttt{log\_path} + \item \texttt{diagnostic\_chunk\_bytes} + \item executor-local logging destinations +\end{itemize} + +For strict v1 theorem-facing execution, the parent controller is sequential: +\[ +\texttt{parent\_controller\_threads}=1. +\] +When \texttt{threads} is exposed, it controls only evaluator-worker-internal +threading: +\[ +\texttt{evaluator\_threads}:=\texttt{threads}\ \text{(default 1)}. +\] +The required process-isolation mode is +\texttt{worker\_isolation\_mode="spawn\_exec\_worker"}. + +For live worker evaluation, strict theorem-facing memory accounting is Linux +only (\texttt{hybrid\_strict\_max} with delegated cgroup-v2 parent). The weak +operational memory profiles (\texttt{process\_rss\_peak} and +\texttt{backend\_reported} diagnostic mode) remain Unix-only. + +For deployment hygiene, root usage should be limited to cgroup delegation and, +when required by host containment rules, first-process placement into the +delegated session cgroup. The tuner process itself should run as the +unprivileged tuner user. + +These controls belong to the executor profile rather than to the canonical +candidate/request surface, unless an implementation explicitly lifts them into +an outer wrapper format. They must be recorded in provenance whenever they can +affect metrics or runtime behavior. + +For \texttt{controller.kind = "annealed\_hill\_climbing"}, +\texttt{controller.max\_mutation\_radius} is normative and required. + +Define baseline warmup count by +\[ +w_{\mathrm{base}}:=\texttt{warmup\_baseline\_runs}\in\mathbb{N}_0. +\] +When this executor-side control is not exposed, the default is +$w_{\mathrm{base}}=0$. +Warmup semantics are normative: +\begin{itemize} + \item execute exactly $w_{\mathrm{base}}$ baseline warmup evaluations before + the single normative baseline evaluation, + \item warmup runs use the same evaluator path and timeout + $\min(T_{\mathrm{eval}},T_{\mathrm{total}})$ as baseline evaluation, + \item warmup runs are excluded from optimization metrics, incumbent updates, + cache insertion, and the evaluation counter $k$, + \item only the post-warmup baseline evaluation defines baseline + $M(z_0^{\mathrm{can}})$ for optimization. +\end{itemize} +Warmup policy is evaluator-relevant and must be included in +$\eta_{\mathrm{eval}}$ and provenance. + +\subsection{Evaluation counting semantics} + +When exposed, \texttt{max\_evaluations} must satisfy +\[ +E_{\max}:=\texttt{max\_evaluations}\in\mathbb{N},\qquad E_{\max}\ge 1. +\] +A value of zero is invalid and must be rejected during executor-profile +validation. + +The executor-side control \texttt{max\_evaluations} determines the maximum +number of non-warmup candidate results admitted into the search/controller +state. This count includes the single normative baseline result and includes +post-baseline candidate results obtained either by fresh evaluator execution or +by cache reuse. It excludes warmup baseline runs, self-loop proposals, +inapplicable proposals that do not produce a candidate result, and static +proposal attempts rejected before candidate-result admission. + +Let $k$ be this non-warmup admitted-candidate-result counter. Warmup baseline +runs do not increment $k$. After the single post-warmup normative baseline +evaluation succeeds and is admitted as the initial current/incumbent candidate, +initialize +\[ +k\leftarrow 1. +\] +Before admitting any post-baseline candidate result into the controller +transition, terminate if +\[ +k\ge E_{\max}. +\] +After each admitted post-baseline candidate result, whether obtained by fresh +evaluator execution or cache reuse, increment +\[ +k\leftarrow k+1. +\] + +Therefore the maximum number of post-baseline candidate results admitted by the +executor is +\[ +E_{\mathrm{opt}}=E_{\max}-1. +\] +When $E_{\max}=1$, the executor performs the normative baseline evaluation and +admits zero post-baseline candidate results. When the executor stops because +$k\ge E_{\max}$, that stop condition refers to the baseline-inclusive +non-warmup admitted-candidate-result counter. + +Actual evaluator executions are reported separately from $k$. In particular, +cache-reused candidate results increment $k$ but do not increment the fresh +evaluator-execution/cache-miss counter. Warmup baseline runs are reported +separately and are excluded from both $k$ and cache insertion. + +An implementation that counts only post-baseline optimization results under +the same \texttt{max\_evaluations} field is non-conforming to this v1 +semantics. Such behavior may be exposed only under a distinct executor-profile +field or compatibility wrapper whose provenance states the translation to the +normative baseline-inclusive convention. + +Define stagnation reset threshold by +\[ +\kappa_{\mathrm{stag}}:= +\begin{cases} +\texttt{stagnation\_reset\_evals}, & \text{if exposed by the executor profile},\\ ++\infty, & \text{otherwise.} +\end{cases} +\] +When $\kappa_{\mathrm{stag}}=+\infty$, stagnation reset is disabled. + +Define the mutation-radius cap by +\[ +r_{\mathrm{mut}}^{\max}:=\texttt{controller.max\_mutation\_radius}\in\mathbb{N}, +\qquad +r_{\mathrm{mut}}^{\max}\ge 1. +\] +This is an annealer execution-control parameter. + +If an implementation exposes an executor-side annealer-kernel selector, +interpret it as an execution-profile choice rather than as part of the +canonical candidate/request document. When this selector is not exposed, the +default profile is +\[ +\texttt{reversible\_elementary\_metropolis}. +\] +This default kernel uses only certified reversible elementary edits from +Section~\ref{sec:reversible-annealer-kernel} and ordinary Metropolis +acceptance. It does not require explicit computation of proposal masses +$q_T$. + +An implementation may additionally expose +\[ +\texttt{compiled\_uniform\_metropolis\_hastings} +\] +as a certified variant, but only if it computes exact canonical proposal +masses $q_T$ for the selected mutation grammar. + +Any alternative annealer kernel, including non-uniform action weights, +adaptive proposal mixtures, incumbent-substructure copy, +reset-to-incumbent, greedy repair, or a different acceptance law, is a +heuristic-mode execution variant. It must be recorded in provenance and does +not inherit the default annealer detailed-balance or proposal-kernel formulas +unless separately specified and re-derived. + +For the optional bounded same-task teacher-refresh wrapper around +\texttt{aiqi\_warmstart\_exact\allowbreak\_jh}, define +\[ +R_{\mathrm{si}}:=\texttt{self\_improvement\_rounds}\in\mathbb{N}, +\qquad +R_{\mathrm{si}}\ge 1, +\] +with default $R_{\mathrm{si}}=1$. +If $R_{\mathrm{si}}>1$, interpret $T_{\mathrm{total}}$ as the single +outer-loop wall-clock budget and use deterministic equal-split round deadlines +\[ +\Delta_r^{\mathrm{si}}:=\frac{r}{R_{\mathrm{si}}}T_{\mathrm{total}}, +\qquad r=0,\dots,R_{\mathrm{si}}. +\] +The warm-start exact-$\mathcal{J}_H$ controller then executes the bounded +same-task trace-refresh wrapper of +Section~\ref{sec:aiqi-warmstart-self-improve}; all rounds remain on the same +$(B,H,d)$ task and are evaluated against the same deployable objective +$\mathcal{J}_H$. + +These controls affect runtime behavior and reproducibility, not the candidate search space. + +\section{Dataset modes and causal-conditional evaluator} +\label{sec:causal-evaluator} + +This section gives the unified evaluator abstraction behind passive-byte +compression and active environment-model tuning. It does not change the +searched object: every candidate still denotes a canonical compression backend +whose predictive semantics are scored by one causal-conditional evaluator. +What changes is the dataset lowering and, when requested, the deployability +profile. + +\subsection{Unified chronological compression candidate and induced predictor} + +Let $\mathcal{A}$ be a finite action alphabet and let $\mathcal{X}$ be a +finite percept alphabet. In the interactive setting, a percept is a coded tuple +\[ +x_t=(o_t,r_t,d_t)\in\mathcal{X}, +\] +where $o_t$ is observation, $r_t$ is reward, and $d_t$ is the terminal flag. + +A candidate $z\in\mathcal{Z}$ denotes a compiled compression backend with an +induced chronological conditional predictor +\[ +\rho_z(x_t\mid h_{0}$ is the example weight or +multiplicity. Define the idealized causal code length in bits by +\[ +C_D^{\mathrm{bit}}(z) +:= +\sum_{i=1}^{N} +w_i\bigl[-\log_2 \rho_z(x_i\mid h_i,a_i)\bigr]. +\] +The operational evaluator reports the corresponding target loss in bits +$\ell_D(z)$ through the same six-field contract as +Section~\ref{sec:formal-problem}; when a material encoded artifact exists, it +may additionally report its physical byte size +$c_{D,\mathrm{phys}}(z)=c_{\mathrm{phys}}(z)$. When the dataset is clear from +context, we write $\ell(z)$ and $c_{\mathrm{phys}}(z)$. + +Define the dataset-relative deployable score by +\[ +\mathcal{J}_H^{\mathrm{causal}}(z;D) +:= +\begin{cases} +8L_B(z)+\ell_D(z), & z\in\mathcal{F}_{D}^{\mathrm{dep}}(B,H),\\[3pt] ++\infty, & \text{otherwise.} +\end{cases} +\] +Here $\mathcal{F}_{D}^{\mathrm{dep}}(B,H)$ is the deployable set induced by the +same status/time/memory interface as before, with the dataset-specific target +stream and any profile-fixed throughput convention attached to $H$. + +\begin{proposition}[Passive and interactive tuning share one MDL schema] +\label{prop:causal-mdl-unification} +Let $D$ be a finite causal prediction dataset and let each candidate +$z\in\mathcal{Z}$ induce a chronological conditional predictor $\rho_z$. +Then the deployability-gated two-part score +$\mathcal{J}_H^{\mathrm{causal}}(z;D)$ is the exact analogue of the ordinary +passive compression tuner objective. If $\mathcal{A}=\{\bot\}$ and $D$ is the +passive-byte sequence $x_{1:n}$ lowered as target-only byte events, then +\[ +C_D^{\mathrm{bit}}(z) += +-\sum_{t=1}^{n}\log_2\rho_z(x_t\mid x_{0, +\qquad +M_U>0. +\] + +Let +\[ +\mathcal{P}_{K,T_U,M_U}^H +\] +denote the set of all programs $p$ such that: +\begin{enumerate} + \item $|p|\le K$, + \item under the fixed profile $H$, program $p$ induces a computable + predictive semimeasure $\nu_{p,H}^{\sigma}$ on every $H$-admissible + structural skeleton $\sigma$, + \item the induced predictor is itself deployable under the same resource + contract encoded by $H$ (including throughput and memory requirements), + \item the program's universal-family execution remains within truncation + limits $T_U$ and $M_U$. +\end{enumerate} + +For the fixed dataset $D$, abbreviate +\[ +\nu_p^H(D):=\nu_{p,H}^{\sigma_H(D)}\bigl(y_H(D)\bigr). +\] + +\begin{assumption}[Constant-overhead embedding of the truncated universal family] +\label{ass:universal-embed} +There exist a constant $c_U\ge 0$ and an injective map +\[ +\iota:\mathcal{P}_{K,T_U,M_U}^H +\to +\mathcal{F}^{\rho,\mathrm{dep}}(B,H), +\qquad +p\mapsto z_p, +\] +such that for every $p\in\mathcal{P}_{K,T_U,M_U}^H$, +\[ +\rho_{z_p,H}^{\sigma}=\nu_{p,H}^{\sigma} +\qquad\text{for every $H$-admissible structural skeleton } \sigma, +\qquad\text{and}\qquad +L_B^{\mathrm{bit}}(z_p)\le |p|+c_U. +\] +\end{assumption} + +\begin{theorem}[Truncated-universal dominance] +\label{thm:truncated-universal-dominance} +Under Assumption~\ref{ass:universal-embed}, for every canonical dataset object +$D$ admissible under $H$, +\[ +\widetilde{\xi}_{B,H}(D) +\ge +2^{-c_U} +\sum_{p\in\mathcal{P}_{K,T_U,M_U}^H} +2^{-|p|}\nu_p^H(D). +\] +Consequently, +\[ +\xi_{B,H}(D) +\ge +2^{-c_U} +\sum_{p\in\mathcal{P}_{K,T_U,M_U}^H} +2^{-|p|}\nu_p^H(D). +\] +\end{theorem} + +\begin{proof} +By definition of $\widetilde{\xi}_{B,H}$ and the inclusion +$z_p=\iota(p)\in\mathcal{F}^{\rho,\mathrm{dep}}(B,H)$, +\[ +\widetilde{\xi}_{B,H}(D) += +\sum_{z\in\mathcal{F}^{\rho,\mathrm{dep}}(B,H)} +2^{-L_B^{\mathrm{bit}}(z)}\rho_{z,H}(D) +\ge +\sum_{p\in\mathcal{P}_{K,T_U,M_U}^H} +2^{-L_B^{\mathrm{bit}}(z_p)}\rho_{z_p,H}(D). +\] +Using Assumption~\ref{ass:universal-embed}, +\[ +2^{-L_B^{\mathrm{bit}}(z_p)}\rho_{z_p,H}(D) +\ge +2^{-(|p|+c_U)}\nu_p^H(D) += +2^{-c_U}2^{-|p|}\nu_p^H(D). +\] +Summing over $p$ proves the first inequality. + +For the normalized mixture, +\[ +\xi_{B,H}(D)=\widetilde{\xi}_{B,H}(D)/Z_B^{\rho,\mathrm{dep}} +\ge \widetilde{\xi}_{B,H}(D), +\] +because $Z_B^{\rho,\mathrm{dep}}\le 1$ by +Proposition~\ref{prop:kraft-semantic}. Hence the same lower bound holds for +$\xi_{B,H}$. +\end{proof} + +\begin{corollary}[Truncated-universal deployable MDL baseline] +\label{cor:truncated-universal-baseline} +Under the exact-coding assumption of the previous subsection and +Assumption~\ref{ass:universal-embed}, for every canonical dataset object $D$ +admissible under $H$, +\[ +\inf_{z\in\mathcal{F}^{\rho,\mathrm{dep}}(B,H)}\mathcal{J}_H^{\mathrm{bit}}(z;D) +\le +c_U ++ +\inf_{p\in\mathcal{P}_{K,T_U,M_U}^H} +\bigl(|p|-\log_2 \nu_p^H(D)\bigr). +\] +\end{corollary} + +\begin{proof} +For each $p\in\mathcal{P}_{K,T_U,M_U}^H$, let $z_p=\iota(p)$. Then +\[ +\mathcal{J}_H^{\mathrm{bit}}(z_p;D) += +L_B^{\mathrm{bit}}(z_p)-\log_2\rho_{z_p,H}(D) +\le +|p|+c_U-\log_2 \nu_p^H(D). +\] +Taking the infimum over $p$ gives +\[ +\inf_{z\in\mathcal{F}^{\rho,\mathrm{dep}}(B,H)}\mathcal{J}_H^{\mathrm{bit}}(z;D) +\le +\inf_{p\in\mathcal{P}_{K,T_U,M_U}^H} +\mathcal{J}_H^{\mathrm{bit}}(z_p;D) +\le +c_U ++ +\inf_{p\in\mathcal{P}_{K,T_U,M_U}^H} +\bigl(|p|-\log_2 \nu_p^H(D)\bigr). +\] +\end{proof} + +\begin{remark}[Meaning of the universality claim] +Theorem~\ref{thm:truncated-universal-dominance} is a class-richness statement. +It does not claim that the runtime optimizer must explicitly execute the +universal family on every run. It states that if the deployable candidate class +contains a constant-overhead encoding of the truncated universal family, then +the induced bounded mixture and the deployable MDL optimum inherit a formal +truncated-universality baseline. +\end{remark} + +\section{Mutation Grammar and Search Neighborhood} + +Let $\mathcal{N}_B(z)$ denote the legal finite neighborhood induced by bounds compiler $B$. +Mutations are generated from this grammar only, never by unconstrained raw JSON edits. + +The bounded mutation grammar is partitioned into two classes: +\begin{enumerate} + \item \emph{certified reversible elementary edits}, used by the default + annealed Metropolis kernel, + \item \emph{heuristic macro moves}, useful for search but outside the + default detailed-balance contract. +\end{enumerate} +Certified reversible elementary edits may include local integer-parameter +steps, finite enum replacements with explicit inverse descriptors, +expert add/remove pairs carrying slot and expert identity, swaps, +optional-component toggles, and other local edits whose inverse is compiled +explicitly. Heuristic macro moves may include incumbent-substructure copy, +reset-to-incumbent, greedy repair, adaptive large jumps, and any mutation whose +reverse probability is not certified by the grammar compiler. + +For integer parameter $p\in[L,U]$, the certified elementary descriptor set +contains signed integer-step descriptors +\[ +\Delta p\in +\{-r_{\mathrm{mut}}^{\max},\ldots,-1,+1,\ldots,+r_{\mathrm{mut}}^{\max}\}. +\] +The descriptor with step $\Delta p$ has inverse descriptor $-\Delta p$. + +At elapsed fraction $u\in[0,1]$, define the active local radius +\[ +r(u)= +\max\!\left( +1, +\left\lfloor +r_{\mathrm{mut}}^{\max}\frac{T(u)}{T_0} +\right\rfloor +\right). +\] +For the default certified reversible kernel, at elapsed fraction $u$ a signed +integer-step descriptor is applicable at state $z$ only if both +\[ +|\Delta p|\le r(u) +\qquad\text{and}\qquad +p+\Delta p\in[L,U]. +\] +If either condition fails, then +\[ +A_{B,u}(e,z)=z. +\] +Thus inactive or boundary-crossing integer edits become self-loop proposals. + +A non-self integer edit is therefore permitted only at fixed elapsed fraction +$u$ when +\[ +|\Delta p|\le r(u) +\quad\text{and}\quad +p+\Delta p\in[L,U], +\] +in which case its inverse is the descriptor with step $-\Delta p$. Since +$|\Delta p|\le r(u)$ iff $|-\Delta p|\le r(u)$, and +\[ +p+\Delta p\in[L,U] +\quad\Longleftrightarrow\quad +(p+\Delta p)-\Delta p=p\in[L,U], +\] +the elementary reversibility condition is preserved. + +Clipped integer mutation is permitted only in heuristic annealer profiles or +in the optional Metropolis--Hastings profile with exact proposal-mass +accounting. It is not part of the default certified reversible elementary +kernel. + +\subsection{Certified reversible elementary annealer kernel} +\label{sec:reversible-annealer-kernel} + +The default v1 annealer kernel is a reversible elementary-edit kernel. Define +the canonical candidate image +\[ +\mathcal{Z}^{\mathrm{can}}:=\operatorname{canon}(\mathcal{Z}). +\] +The bounds compiler constructs a finite edit-descriptor set +\[ +\mathcal{E}_B +\] +equipped with an involution +\[ +\operatorname{inv}:\mathcal{E}_B\to\mathcal{E}_B, +\qquad +\operatorname{inv}(\operatorname{inv}(e))=e. +\] +The edit sampling law $p_B$ satisfies +\[ +p_B(e)=p_B(\operatorname{inv}(e)) +\qquad\text{for all }e\in\mathcal{E}_B. +\] + +Each edit descriptor induces a total canonical transition at elapsed fraction +$u$: +\[ +A_{B,u}:\mathcal{E}_B\times\mathcal{Z}^{\mathrm{can}} +\to +\mathcal{Z}^{\mathrm{can}}. +\] +If an edit is inapplicable at $z$, inactive at elapsed fraction $u$, or would +leave the certified schema/bounds-valid elementary-edit family, then +\[ +A_{B,u}(e,z)=z. +\] +Here ``valid'' refers only to static candidate validity under +$\mathrm{Valid}_{B,H}$ before evaluator execution. It does not assert +deployability; timeout, throughput, memory, and success status are still +determined only by the evaluator. +Thus inapplicable, inactive, or uncertified elementary edits become self-loop +proposals rather than invalid evaluated candidates. + +For every fixed $u\in[0,1]$ and every non-self transition, the edit compiler +must satisfy +\[ +A_{B,u}(e,z)=z'\ne z +\quad\Longleftrightarrow\quad +A_{B,u}(\operatorname{inv}(e),z')=z. +\] + +For fixed elapsed fraction $u$, the induced proposal kernel is +\[ +q_{B,u}(z'\mid z) +:= +\sum_{e\in\mathcal{E}_B} +p_B(e)\mathbf{1}\!\left\{A_{B,u}(e,z)=z'\right\}. +\] +By inverse symmetry and the fixed-$u$ reversibility condition, +\[ +q_{B,u}(z'\mid z)=q_{B,u}(z\mid z') +\] +for all canonical candidates $z,z'$. Therefore the default annealer uses the +ordinary Metropolis acceptance rule at the current elapsed fraction $u$ +without a Hastings correction. + +Because $u$ changes over runtime, the annealer is a time-inhomogeneous +Metropolis process. No stationary-distribution claim is made for the full +cooling schedule. + +\begin{remark}[Heuristic macro moves] +An implementation may additionally expose non-reversible macro moves such as +reset-to-incumbent, incumbent-substructure copy, greedy repair, adaptive large +jumps, or arbitrary-valid-subcandidate replacement. These moves are permitted +only under a heuristic annealer profile or as a separately reported heuristic +phase. They do not participate in the certified reversible Metropolis kernel +and do not support detailed-balance or stationary-distribution claims. They +may still be useful for anytime optimization, but their guarantees are limited +to feasibility safety and incumbent monotonicity under the surrounding runtime +contract. +\end{remark} + +\subsection{Finite action compilation for planner exposure}\label{sec:action-compile} + +For planner exposure, the bounded mutation grammar is compiled by a +deterministic compiler $\operatorname{compile}_B$ parameterized by $B$. +Define compiled action-cardinality +\[ +N_B^{\mathrm{compile}}(B):=\big|\operatorname{compile}_B(B)\big|\in\mathbb{N}, +\qquad +N_B^{\mathrm{compile}}(B)\ge 1, +\] +and planner-visible indexed action alphabet +\[ +\mathcal{A}_B=\{0,\dots,N_B^{\mathrm{compile}}(B)-1\}, +\qquad +|\mathcal{A}_B|=N_B^{\mathrm{compile}}(B). +\] +For planner-exposed controller configurations, cardinality consistency is +normative: +\[ +N_B^{\mathrm{compile}}(B)=\texttt{agent\_actions}. +\] +After compiling $\mathcal{A}_B$ from $B$, initialization must verify +\[ +|\mathcal{A}_B|=\texttt{agent\_actions}. +\] +On mismatch, initialization aborts with configuration error +\texttt{action\_alphabet\_mismatch}. +Each action index selects one compiled mutation template. The template decoder +depends on state, so grammar families such as incumbent copying, +reset-to-incumbent moves, and temperature-scaled local mutations remain +representable. + +Let $\mathcal{S}_B$ denote the annealer proposal-state space (including +proposal source, incumbent, elapsed time, and mutation RNG state), and let +$s_t\in\mathcal{S}_B$ denote the current proposal state. Action decoding is +specified by totalized state-aware semantics +\[ +\mu_B^{\mathrm{tot}}:\mathcal{A}_B\times\mathcal{S}_B\to\mathcal{Z}\times\{0,1\}, +\qquad +(a_t,s_t)\mapsto(\tilde z_t,\iota_t), +\] +where $\iota_t=1$ marks an inapplicable mutation primitive at state $s_t$. +If an implementation models decoder randomness explicitly rather than through +RNG state inside $s_t$, the equivalent form is a total Markov kernel +$\mu_B^{\mathrm{tot}}(\cdot\mid a_t,s_t)\in\Delta(\mathcal{Z}\times\{0,1\})$. + +For this specification, the action-selection distribution is fixed to +\emph{uniform} over the compiled finite alphabet: +\[ +\pi_T(a\mid s,B):=\frac{1}{N_B^{\mathrm{compile}}(B)}, +\qquad +a\in\mathcal{A}_B. +\] +The subscript $T$ is retained for interface compatibility; in this +specification temperature dependence enters through state in +$\mu_B^{\mathrm{tot}}$ rather than through non-uniform action weights. + +This subsection defines the finite indexed action alphabet used by +planner-exposed controller configurations. The same compilation may also be +used by the optional certified annealer profile +\texttt{compiled\_uniform\_metropolis\_hastings}, but it is not the default +v1 annealer kernel. + +Define proposal-outcome kernel +\[ +\widehat q_T(z,\iota\mid s,B) +:= +\sum_{a\in\mathcal{A}_B} +\pi_T(a\mid s,B) +\mathbf{1}\!\left\{\mu_B^{\mathrm{tot}}(a,s)=(z,\iota)\right\}, +\] +and induced canonical-candidate marginal +\[ +q_T(z'\mid s,B) +:= +\sum_{(z,\iota)\in\mathcal{Z}\times\{0,1\}} +\widehat q_T(z,\iota\mid s,B) +\mathbf{1}\!\left\{\operatorname{canon}(z)=z'\right\}. +\] +Here $s$ is the current annealer proposal state. +If stochastic decoder form is used, replace indicator terms above by the +corresponding kernel masses from $\mu_B^{\mathrm{tot}}(\cdot\mid a,s)$. + +\begin{remark}[Optional Metropolis--Hastings annealer kernel] +The more general compiled-uniform Metropolis--Hastings kernel is permitted +only when the implementation computes exact canonical proposal masses +$q_T(z'\mid s,B)$. This certified variant may include state-aware macro moves, +but its correctness depends on exact accounting for inapplicable actions, +canonicalization collisions, stochastic decoder choices, and reverse-source +states. It is therefore an optional certified profile rather than the default +v1 annealer kernel. +\end{remark} + +\section{Annealed Hill Climbing Algorithm}\label{sec:ahc-runtime} + +\subsection{State and initialization} + +State variables: current candidate $z_{\mathrm{cur}}$, best candidate +$z_{\mathrm{best}}$, elapsed wall time $t$, evaluation counter $k$, and +stagnation counter $n_{\mathrm{stag}}\in\mathbb{N}_0$. + +Initialization requirements: +\begin{enumerate} + \item load and canonicalize baseline candidate $z_0^{\mathrm{can}}:=\operatorname{canon}(z_0)$, + \item execute $w_{\mathrm{base}}$ warmup baseline runs per the execution-control semantics above, + \item evaluate $z_0^{\mathrm{can}}$ once normatively with timeout $\min(T_{\mathrm{eval}},T_{\mathrm{total}})$, + \item enforce the baseline deployability precondition from + Section~\ref{subsec:baseline-precondition}; abort immediately if baseline + is non-deployable (fails success status, throughput floor, or memory cap), + \item set $z_{\mathrm{cur}}=z_{\mathrm{best}}=z_0^{\mathrm{can}}$ and $n_{\mathrm{stag}}\leftarrow 0$. +\end{enumerate} + +\subsection{Cooling schedule} + +With elapsed fraction +\[ +u=\min\!\left(1,\frac{t}{T_{\mathrm{total}}}\right), +\] +use log-linear cooling +\[ +T(u)=T_{\min}\left(\frac{T_0}{T_{\min}}\right)^{1-u}, +\] +default $T_0=1.0$, $T_{\min}=10^{-3}$. + +\begin{remark}[Temperature units] +The annealing temperature $T(u)$ has the same units as the objective: +bits. Therefore the ratio +\[ +\Delta\mathcal{J}/T(u) +\] +is dimensionless. +\end{remark} + +\subsection{Transition rule} + +At each loop iteration: +\begin{enumerate} + \item compute remaining budget $B_t:=T_{\mathrm{total}}-t$; + if $B_t\le 0$ then terminate immediately, + \item compute $T(u)$, + \item sample elementary edit $e_t\sim p_B$ and set + \[ + \hat z':=A_{B,u}(e_t,z_{\mathrm{cur}}), + \] + \item if $\hat z'=z_{\mathrm{cur}}$, count a self-loop proposal and + continue to the next loop iteration with no evaluation, + \item else if $\hat z'$ fails $\mathrm{Valid}_{B,H}$, assign + $M(\hat z')$ status invalid with diagnostic reason + \texttt{candidate\_invalid} without starting compression execution, + reject it as current, and continue to the next loop iteration, + \item else define effective evaluation timeout + \[ + T_t^{\mathrm{eval}}:=\min(T_{\mathrm{eval}},B_t), + \] + and for cache lookup use + $\mathrm{Key}_{\mathrm{cache}}(\hat z',\eta_{\mathrm{eval}}^{(t)},d)$, + where $\eta_{\mathrm{eval}}^{(t)}$ is the evaluator profile including + $T_t^{\mathrm{eval}}$; on cache miss evaluate with per-candidate timeout + $T_t^{\mathrm{eval}}$, + \item set $K_{\mathrm{pre}}:=K(z_{\mathrm{best}})$; if + $\mathcal{J}_H(\hat z';d)<+\infty$ and + $K(\hat z')\prec_K K_{\mathrm{pre}}$, update + $z_{\mathrm{best}}\leftarrow \hat z'$, + \item if $\mathcal{J}_H(\hat z';d)<+\infty$, accept as current via the + ordinary Metropolis criterion: + \[ + P_{\mathrm{accept}}= + \min\!\left( + 1, + \exp\!\left(-\dfrac{\mathcal{J}_H(\hat z';d)-\mathcal{J}_H(z_{\mathrm{cur}};d)}{T(u)}\right) + \right), + \] + and non-deployable candidates ($\mathcal{J}_H(\hat z';d)=+\infty$) are rejected, + setting $z_{\mathrm{cur}}\leftarrow\hat z'$ on acceptance, + \item update stagnation counter by + \[ + n_{\mathrm{stag}}\leftarrow + \begin{cases} + 0, + & \mathcal{J}_H(\hat z';d)<+\infty\ \land\ K(\hat z')\prec_K K_{\mathrm{pre}},\\ + n_{\mathrm{stag}}+1, + & \mathcal{J}_H(\hat z';d)<+\infty\ \land\ \neg\big(K(\hat z')\prec_K K_{\mathrm{pre}}\big),\\ + n_{\mathrm{stag}}, + & \mathcal{J}_H(\hat z';d)=+\infty, + \end{cases} + \] + \item if $n_{\mathrm{stag}}\ge\kappa_{\mathrm{stag}}$, set + $z_{\mathrm{cur}}\leftarrow z_{\mathrm{best}}$ and + $n_{\mathrm{stag}}\leftarrow 0$. +\end{enumerate} + +\subsection{Termination} + +Terminate when any condition holds: +\begin{itemize} + \item elapsed wall-clock reaches the global deadline + $T_{\mathrm{total}}$, + \item \texttt{max\_evaluations} reached, + \item neighborhood exhaustion is proven, + \item unrecoverable evaluator failure occurs. +\end{itemize} + +Hard-budget enforcement is normative: no candidate evaluation may run with a +timeout exceeding remaining global budget, and no new blocking evaluation may +start once $B_t\le 0$. + +Output must include: +\begin{itemize} + \item deterministic serialized $z_{\mathrm{best}}$ at \texttt{output\_config\_path}, + \item structured report at optional \texttt{report\_path}. +\end{itemize} + +\section{Reward and Observation (Future Environment Compatibility)}\label{sec:reward-observation} + +This section specifies the normative planner-reward semantics. +Candidate validity, evaluator semantics, objective-feasibility semantics +(including throughput and memory deployability constraints), and output +definition remain shared across controller choices. + +\subsection{Planner reward semantics and encoded reward symbols} + +Planner-facing controllers induce one of the following reward semantics: +\begin{itemize} + \item \texttt{exact\_objective\_difference}: exact incumbent-objective + decrease in the same bit-valued objective $\mathcal{J}_H$ used by the tuner, + encoded by a finite injective reward-symbol map; + \item \texttt{normalized\_clipped\_improvement}: normalized and clipped + incumbent-improvement reward used by the discounted AIQI controller; + \item \texttt{quantized\_objective\_difference}: declared deterministic + quantization of objective-difference reward for finite-alphabet controller + interfaces. +\end{itemize} + +The canonical tune document does not expose these as an independent top-level +field. In the normative tuning surface, +\texttt{"mc\_aixi\_fac\_ctw"} and +\texttt{"aiqi\_warmstart\_exact\_jh"} induce +\texttt{"exact\_objective\_difference"} semantics, whereas +\texttt{"aiqi\_discounted"} induces +\texttt{"normalized\_clipped\_improvement"} semantics. A quantized +objective-difference variant, if an implementation offers one internally, is an +execution-profile declaration recorded in provenance rather than a distinct +tune-document field. + +Let +\[ +r_H^{\mathcal J}(s,a) +:= +\mathcal{J}_H(\pi_b(s);d) +- +\mathcal{J}_H(\pi_b(U_H(s,a));d) +\] +denote the underlying objective-improvement signal. Along reachable +trajectories this quantity is nonnegative and finite. + +A planner-facing implementation must define a controller-specific reward +encoding domain +\[ +\mathcal{D}_H^R\subseteq\mathbb{R}_{\ge 0}, +\] +a reward encoder +\[ +\Omega_H:\mathcal{D}_H^R\to \mathcal{R}_B^{\mathrm{enc}} +\] +and a semantic reward decoder +\[ +V_H:\mathcal{R}_B^{\mathrm{enc}}\to\mathbb{R}_{\ge 0} +\] +according to the controller-induced reward semantics. For +exact-objective-difference semantics, \(\mathcal{D}_H^R\) is the represented +reachable subset of the finite scalar representation +\(\mathbb{V}_H^{\mathcal J}\) defined below. For +normalized-clipped-improvement semantics, \(\mathcal{D}_H^R=[0,1]\) after +clipping and normalization. For quantized objective-difference semantics, one +may take \(\mathcal{D}_H^R=\mathbb{R}_{\ge 0}\). When later formulas write +\(\Omega_H(r)\) directly on the underlying objective-improvement signal, that +notation denotes the controller-specific composite map obtained after any +required deterministic preprocessing. + +For exact-objective-difference semantics, the implementation must fix a +finite canonical scalar representation +\[ +\mathbb{V}_H^{\mathcal J}\subset\mathbb{R}_{\ge 0} +\] +for objective values and objective differences. Examples include a fixed-width +IEEE-754 representation with non-finite values forbidden, a fixed-point rational +lattice, or another deterministic finite numeric code declared by $H$. + +\begin{remark}[Exactness is relative to the declared scalar semantics] +For exact-objective-difference semantics, ``exact'' means exact with respect to +the finite scalar representation $\mathbb{V}_H^{\mathcal J}$ declared by $H$. +If the evaluator uses canonical IEEE-754 arithmetic, then non-finite values are +forbidden, rounding behavior must be deterministic, and theorem claims concern +the induced finite floating-point objective. If exact rational or fixed-point +arithmetic is selected instead, theorem claims concern that representation. +Implementations must not silently mix scalar representations within one run. +\end{remark} + +Conformance requires that every deployable objective value emitted by the +evaluator lies in this representation: +\[ +\mathcal{J}_H(z;d)\in\mathbb{V}_H^{\mathcal J} +\qquad +\text{for every planner-exposed deployable candidate }z. +\] +The representation must also be closed under the objective-difference operation +used by the planner on reachable incumbent transitions: +\[ +\mathcal{J}_H(b;d)-\mathcal{J}_H(b';d)\in\mathbb{V}_H^{\mathcal J} +\] +whenever $b,b'$ are reachable incumbents and +$\mathcal{J}_H(b';d)\le \mathcal{J}_H(b;d)$. + +The exact reward encoder is then a finite injective code for this scalar +representation: +\[ +\Omega_H:\mathbb{V}_H^{\mathcal J}\hookrightarrow +\{0,\dots,2^{b_R}-1\}, +\] +with semantic decoder +\[ +V_H:\{0,\dots,2^{b_R}-1\}\to\mathbb{R}_{\ge 0} +\] +satisfying +\[ +V_H(\Omega_H(r))=r +\qquad +\text{for every represented reachable reward }r. +\] +Initialization must abort with configuration error +\texttt{reward\_encoding\_unsafe} if the declared representation is not finite, +if $2^{b_R}$ is too small for the declared representation, or if the evaluator +cannot guarantee that all exact-mode objective values and differences are +represented exactly. + +For quantized objective-difference semantics, the implementation must declare +a deterministic quantizer +\[ +Q_H^{\mathrm{quant}}:\mathbb{R}_{\ge 0}\to\{0,\dots,2^{b_R}-1\} +\] +and a corresponding representative-value decoder +\[ +V_H^{\mathrm{quant}}:\{0,\dots,2^{b_R}-1\}\to\mathbb{R}_{\ge 0}. +\] +The emitted reward symbol is +\[ +\Omega_H(r):=Q_H^{\mathrm{quant}}(r), +\] +and the planner's semantic reward value is +\[ +V_H(\Omega_H(r)):=V_H^{\mathrm{quant}}(Q_H^{\mathrm{quant}}(r)). +\] +This mode is implementable for arbitrary real-valued objectives, but exact +$\mathcal{J}_H$ planner theorems apply only to the quantized objective induced +by $V_H^{\mathrm{quant}}\circ Q_H^{\mathrm{quant}}$, not to the original +real-valued objective unless the quantizer is exact on the reachable reward set. + +For normalized-clipped-improvement semantics, define the unit-interval clamp +\[ +\operatorname{clamp}_{[0,1]}(x):=\min\!\bigl(\max(x,0),1\bigr) +\] +and the controller-specific clipping parameters +\[ +\begin{aligned} +r_{\min}^{\mathrm{clip}}&:=\texttt{controller.min\_improvement},\\ +r_{\max}^{\mathrm{clip}}&:=\texttt{controller.max\_improvement},\\ +r_{\max}^{\mathrm{clip}}&>r_{\min}^{\mathrm{clip}}. +\end{aligned} +\] +Then define the normalized clipped improvement by +\[ +\widehat r_H(r) +:= +\operatorname{clamp}_{[0,1]} +\left( +\frac{r-r_{\min}^{\mathrm{clip}}}{r_{\max}^{\mathrm{clip}}-r_{\min}^{\mathrm{clip}}} +\right), +\qquad r_{\max}^{\mathrm{clip}}>r_{\min}^{\mathrm{clip}}. +\] +The implementation must define a finite immediate-reward encoder +\[ +\Omega_H^{\mathrm{clip}}:[0,1]\to\{0,\dots,2^{b_R}-1\} +\] +and representative-value decoder +\[ +V_H^{\mathrm{clip}}:\{0,\dots,2^{b_R}-1\}\to[0,1]. +\] +The emitted reward symbol is +\[ +\Omega_H(r):=\Omega_H^{\mathrm{clip}}(\widehat r_H(r)), +\] +and the semantic one-step reward value consumed by the discounted controller is +\[ +V_H(\Omega_H(r)) +:= +V_H^{\mathrm{clip}}\!\left(\Omega_H^{\mathrm{clip}}(\widehat r_H(r))\right). +\] +This mode is the normative reward semantics for +\texttt{aiqi\_discounted}. It does not require an exact reward alphabet large +enough to injectively encode $\mathbb{V}_H^{\mathcal J}$, and it does not +support exact $\mathcal{J}_H$ planner theorems. + +Only \texttt{exact\_objective\_difference}, with the injective finite reward +encoder above, supports the exact $\mathcal{J}_H$ planner theorems below. + +\subsection{Exact objective-aligned planner reward path} + +For exact objective alignment under planner controllers, use +incumbent-objective-difference reward +\[ +R_t^{\mathcal{J}}:=\mathcal{J}_H(b_t;d)-\mathcal{J}_H(b_{t+1};d), +\] +where $\mathcal{J}_H$ is the deployable objective from +Section~\ref{sec:formal-problem}. This preserves exact objective ordering. + +For fixed finite horizon $m\in\mathbb{N}$, the exact cumulative-improvement +target is +\[ +G_t^{(m)}:=\sum_{k=0}^{m-1}R_{t+k}^{\mathcal{J}} +=\mathcal{J}_H(b_t;d)-\mathcal{J}_H(b_{t+m};d), +\] +so finite-horizon planner targets are exact objective decreases over $m$ +executed steps. + +With incumbent $b$ before evaluating $z$, define diagnostic deltas whenever +physical compressed bytes are available for both candidates: +\[ +\delta_c(z\mid b)=\frac{c_{\mathrm{phys}}(b)-c_{\mathrm{phys}}(z)}{\max(c_{\mathrm{phys}}(b),1)}, +\qquad +\delta_v(z\mid b)=\frac{\tau(b)-\tau(z)}{\max(\tau(b),\varepsilon)}, +\] +with fixed $\varepsilon>0$ (default $\varepsilon=10^{-9}$). + +For total observation semantics, define +\[ +N_{H,+}(d):=\max(N_H(d),1). +\] +Use reserved failure symbols +\[ +\bot_c,\bot_\ell,\bot_\tau,\bot_{\delta_c},\bot_{\delta_v}. +\] + +Let $\Sigma_B$ be a finite signature alphabet and define deterministic +signature map +\[ +\sigma:\mathcal{Z}\to\Sigma_B. +\] + +Define totalized diagnostics: +\[ +\tilde c(z)= +\begin{cases} + c_{\mathrm{phys}}(z)/N_{H,+}(d), + & \mathrm{status}(z)=\mathrm{success}\ \land\ c_{\mathrm{phys}}(z)\neq\bot_c,\\ + \bot_c, & \text{otherwise,} +\end{cases} +\] +\[ +\tilde \ell(z)= +\begin{cases} + \ell(z)/N_{H,+}(d), & \mathrm{status}(z)=\mathrm{success},\\ + \bot_\ell, & \text{otherwise,} +\end{cases} +\] +\[ +\tilde \tau(z)= +\begin{cases} +\tau(z)/T_{\mathrm{eval}}, & \mathrm{status}(z)=\mathrm{success},\\ +1, & \mathrm{status}(z)=\mathrm{timeout},\\ +\bot_\tau, & \text{otherwise,} +\end{cases} +\] +\[ +\tilde\delta_c(z\mid b)= +\begin{cases} +\delta_c(z\mid b), +& \mathrm{status}(z)=\mathrm{success} +\ \land\ c_{\mathrm{phys}}(z)\neq\bot_c +\ \land\ c_{\mathrm{phys}}(b)\neq\bot_c,\\ +\bot_{\delta_c}, & \text{otherwise,} +\end{cases} +\qquad +\tilde\delta_v(z\mid b)= +\begin{cases} +\delta_v(z\mid b), & \mathrm{status}(z)=\mathrm{success},\\ +\bot_{\delta_v}, & \text{otherwise.} +\end{cases} +\] + +Define typed value spaces +\[ +\mathcal{V}_c:=\mathbb{R}_{\ge 0}\sqcup\{\bot_c\}, +\quad +\mathcal{V}_{\ell}:=\mathbb{R}_{\ge 0}\sqcup\{\bot_\ell\}, +\quad +\mathcal{V}_{\tau}:=[0,1]\sqcup\{\bot_\tau\}, +\] +\[ +\mathcal{V}_{\delta_c}:=\mathbb{R}\sqcup\{\bot_{\delta_c}\}, +\qquad +\mathcal{V}_{\delta_v}:=\mathbb{R}\sqcup\{\bot_{\delta_v}\}. +\] +Then raw observation codomain is +\[ +\mathcal{O}^{\mathrm{raw}}_B:= +\{0,1\}\times\mathcal{V}_c\times\mathcal{V}_{\ell}\times\mathcal{V}_{\tau} +\times\mathcal{V}_{\delta_c}\times\mathcal{V}_{\delta_v}\times\Sigma_B\times\{0,1\}. +\] + +The raw observation payload is +\[ +o^{\mathrm{raw}}(z\mid b,d_t)= +\Big( +\mathbf{1}_{\mathrm{fail}}, +\tilde c(z), +\tilde \ell(z), +\tilde \tau(z), +\tilde\delta_c(z\mid b), +\tilde\delta_v(z\mid b), +\sigma(z), +d_t +\Big), +\qquad +o^{\mathrm{raw}}(z\mid b,d_t)\in\mathcal{O}^{\mathrm{raw}}_B, +\] +where $\mathbf{1}_{\mathrm{fail}}=1$ for timeout/invalid/error or for successful candidates with +$\theta(z)<\theta_{\min}$ or $\mu(z)>\mu_{\max}$, and $0$ otherwise, +and $d_t\in\{0,1\}$ is terminal indicator emitted by this environment. + +Define deterministic observation adapter +\[ +\Psi_O:=\Psi_O^{\eta_O}, +\qquad +\Psi_O^{\eta_O}:\mathcal{O}^{\mathrm{raw}}_B\to\mathcal{O}_B^{L_O}, +\] +for a fixed observation-adapter specification +\[ +\eta_O:=(\text{field order},\text{sentinel coding},\text{quantization/clipping}, +\text{packing rule},L_O,b_O). +\] +Admissibility requires +\[ +0\le (\Psi_O(o))_i\le 2^{b_O}-1 +\quad\text{for all }o\in\mathcal{O}^{\mathrm{raw}}_B,\ i\in\{1,\dots,L_O\}, +\] +with the degenerate case $(\Psi_O(o))_i=0$ when $b_O=0$. +For reproducible planner behavior, $\eta_O$ is fixed for a run and must be +recorded or hash-committed in provenance. + +\begin{remark}[Adapter boundary] +The generic raw-observation adapter $\Psi_O$ is separate from the planner +reward encoder $\Omega_H$ and semantic decoder $V_H$. The exact planner-state +embedding in Section~\ref{sec:aixi-embedding} may use the stronger specialized +state encoder $\Psi_H$ for observations, while reward symbols are always +emitted through the selected reward-mode encoder $\Omega_H$. No separate +integer reward adapter $\Pi_R$ or affine encoder $Q_R$ is part of the general +semantics. +\end{remark} + +\section{Core Properties} + +\begin{proposition}[Feasibility safety] +Any returned $z_{\mathrm{best}}$ is a deployable evaluated or cache-reused candidate satisfying +bounds, timeout feasibility, minimum-throughput, and memory-cap constraints. +\end{proposition} + +\begin{proof} +$z_{\mathrm{best}}$ starts at a deployable baseline and is updated only with +deployable candidates under $K$. Timeout/invalid/error and successful but +non-deployable candidates never enter $\mathcal{S}_t$. +\end{proof} + +\begin{proposition}[Monotone incumbent key] +Across best updates, the incumbent key $K(z_{\mathrm{best}})$ is strictly decreasing in lexicographic order. +\end{proposition} + +\begin{proof} +By update rule, $z_{\mathrm{best}}$ changes only when $K(z')\prec_K K(z_{\mathrm{best}})$. Hence each update is a strict decrease in $K$. +\end{proof} + +\begin{proposition}[Objective monotonicity under key updates] +Across best updates, incumbent objective is non-increasing: if +$z_{\mathrm{best}}$ changes from $z$ to $z'$, then +$\mathcal{J}_H(z';d)\le\mathcal{J}_H(z;d)$. +\end{proposition} + +\begin{proof} +The incumbent updates only when +$K(z')=(\mathcal{J}_H(z';d),\operatorname{ser}(\operatorname{canon}(z')))$ is +lexicographically smaller than +$K(z)=(\mathcal{J}_H(z;d),\operatorname{ser}(\operatorname{canon}(z)))$. +Hence $\mathcal{J}_H(z';d)\le\mathcal{J}_H(z;d)$. +\end{proof} + +\begin{remark}[Conditional replay determinism guarantee] +If evaluator behavior, timeout clock semantics, RNG stream, build/features, and +execution controls are fixed and deterministic, then the tuning trace and final +output are deterministic. +\end{remark} + +\begin{remark} +Wall-clock jitter can perturb timeout boundaries in practice. Therefore reproducibility is a normative requirement and should be recorded with timing and environment metadata. +\end{remark} + +\section{Annealed-Controller Assumptions and Non-Asymptotic Guarantees}\label{sec:annealed-guarantees} + +Unless explicitly stated otherwise, this section applies to the +\texttt{"annealed\_hill\_climbing"} runtime path from +Section~\ref{sec:ahc-runtime}. + +\begin{assumption}[Finite grammar and computable validity]\label{ass:finite-grammar-validity} +For every valid candidate $z$, the neighborhood $\mathcal{N}_B(z)$ is finite and effectively enumerable; canonicalization and validity checks terminate. +\end{assumption} + +\begin{assumption}[Positive reversible proposal support for default annealer]\label{ass:positive-proposal-support} +For each reachable canonical current candidate $z$, elapsed fraction $u$, and +certified active neighboring candidate $z'$ reachable by the elementary-edit +compiler at that elapsed fraction, there exists at least one descriptor +$e\in\mathcal{E}_B$ such that +\[ +A_{B,u}(e,z)=z'. +\] +Equivalently, +\[ +q_{B,u}(z'\mid z)>0. +\] +By symmetry of the certified reversible kernel at fixed $u$, +\[ +q_{B,u}(z'\mid z)=q_{B,u}(z\mid z'). +\] +\end{assumption} + +\begin{remark}[Symmetry removes the Hastings correction] +Under the default reversible elementary kernel of +Section~\ref{sec:reversible-annealer-kernel}, positive proposal support for a +non-self transition at fixed elapsed fraction $u$ automatically implies the +same positive support in the reverse direction at that same $u$. Therefore the +acceptance rule depends only on the objective difference and temperature; no +reverse-source proposal-mass accounting is required. +\end{remark} + +\begin{assumption}[Per-iteration overhead lower bound]\label{ass:iter-overhead-lb} +There exists $h_{\min}>0$ such that each loop iteration consumes at least $h_{\min}$ wall-clock seconds including proposal, canonicalization, validation, cache lookup, and bookkeeping. +\end{assumption} + +\begin{proposition}[Finite-horizon iteration bound]\label{prop:finite-iter-bound} +Let $N_{\mathrm{iter}}$ be total loop iterations before termination by time budget $T_{\mathrm{total}}$. Under Assumption~\ref{ass:iter-overhead-lb}, +\[ +N_{\mathrm{iter}}\le\left\lfloor\frac{T_{\mathrm{total}}}{h_{\min}}\right\rfloor. +\] +Hence all diagnostic counts (attempted proposals, invalids, cache hits, cache misses, successes) are finite and budget-bounded. +\end{proposition} + +\begin{proposition}[Annealed finite-horizon improvement probability lower bound]\label{prop:finite-horizon-improve} +Fix horizon $H\in\mathbb{N}$. Suppose at each of the first $H$ deployable +evaluation opportunities, conditional on past history, the probability of +producing a strictly better incumbent is at least $p_*>0$. Then +\[ +\mathbb{P}(\text{at least one incumbent improvement by }H)\ge 1-(1-p_*)^H. +\] +Consequently, the probability of no improvement by horizon $H$ decays at least geometrically. +\end{proposition} + +\begin{proposition}[Non-asymptotic Metropolis acceptance envelope]\label{prop:accept-envelope} +For a deployable proposal $z'$ from current candidate $z_{\mathrm{cur}}$ at +elapsed fraction $u$ under the default reversible elementary kernel, let +\[ +\Delta\mathcal{J}:=\mathcal{J}_H(z';d)-\mathcal{J}_H(z_{\mathrm{cur}};d). +\] +Then +\[ +\mathbb{P}_{\mathrm{accept}}(u)= +\min\!\left(1, +\exp\!\left(-\dfrac{\Delta\mathcal{J}}{T(u)}\right) +\right). +\] + +For an uphill proposal with $\Delta\mathcal{J}>0$, +\[ +\mathbb{P}_{\mathrm{accept}}(u)= +\exp\!\left(-\frac{\Delta\mathcal{J}}{T(u)}\right). +\] +If $T_0>T_{\min}>0$ and +\[ +T(u)=T_{\min}\left(\frac{T_0}{T_{\min}}\right)^{1-u}, +\] +then $\mathbb{P}_{\mathrm{accept}}(u)$ is non-increasing in $u$, and +\[ +\exp\!\left(-\frac{\Delta\mathcal{J}}{T_{\min}}\right) +\le +\mathbb{P}_{\mathrm{accept}}(u) +\le +\exp\!\left(-\frac{\Delta\mathcal{J}}{T_0}\right). +\] +\end{proposition} + +\begin{remark} +These are finite-time statements: they do not claim global optimality under finite budgets. They characterize the expected behavior envelope and make engineering trade-offs explicit. +\end{remark} + +\subsection{Idealized asymptotic theorem (explicitly out-of-budget scope)}\label{sec:annealed-asymptotic-scope} + +The guarantees above are finite-time and budget-respecting. The theorem below +is deliberately scope-delimited: it describes an idealized infinite-opportunity +annealed process and is not a production guarantee under finite-budget runtime. + +\begin{assumption}[Finite feasible candidate key space]\label{ass:finite-success-key-space} +The deployable set $\mathcal{F}^{\mathrm{dep}}(B,H)$ is finite and +nonempty. +\end{assumption} + +\begin{assumption}[Infinite deployable opportunities]\label{ass:infinite-success-opportunities} +Index deployable evaluation opportunities by $h\in\mathbb{N}$ and let $b_h$ +denote the incumbent after opportunity $h$. The process admits infinitely many +such opportunities almost surely. +\end{assumption} + +\begin{assumption}[Uniform one-step improvement support]\label{ass:uniform-improvement-support} +There exists $p_{\mathrm{imp}}\in(0,1]$ such that for every deployable opportunity +index $h$, conditional on any realized history with incumbent $b_h$ not globally +minimal in $K$ over $\mathcal{F}^{\mathrm{dep}}(B,H)$, the probability of a strict +incumbent-key improvement at the next deployable opportunity is at least +$p_{\mathrm{imp}}$. +\end{assumption} + +Define incumbent rank by +\[ +\operatorname{rank}_K(b):= +\big|\{z\in\mathcal{F}^{\mathrm{dep}}(B,H):K(z)\prec_K K(b)\}\big|. +\] +Then $\operatorname{rank}_K(b)=0$ iff $b$ is globally $K$-minimal. + +\begin{theorem}[Idealized almost-sure eventual global-optimum hit]\label{thm:idealized-asymptotic-hit} +Under Assumptions~\ref{ass:finite-success-key-space}--\ref{ass:uniform-improvement-support}, +\[ +\mathbb{P}\!\big(\exists h\in\mathbb{N}:\operatorname{rank}_K(b_h)=0\big)=1. +\] +Moreover, for any realized history with $\operatorname{rank}_K(b_h)>0$ and any +$H\in\mathbb{N}$, +\[ +\mathbb{P}\!\big(\operatorname{rank}_K(b_{h+j})=\operatorname{rank}_K(b_h)\ \forall j=1,\dots,H\mid\mathcal{H}_h\big) +\le (1-p_{\mathrm{imp}})^H, +\] +where $\mathcal{H}_h$ denotes history up to deployable opportunity $h$. +\end{theorem} + +\begin{proof} +If $\operatorname{rank}_K(b_h)>0$, Assumption~\ref{ass:uniform-improvement-support} +implies conditional probability at least $p_{\mathrm{imp}}$ of a strict rank +drop at the next deployable opportunity. Therefore the probability of no rank +drop for $H$ consecutive deployable opportunities is at most +$(1-p_{\mathrm{imp}})^H$, proving the finite-horizon bound. Letting +$H\to\infty$ gives probability $0$ of never dropping rank from any positive +rank state. Since rank is a nonnegative integer that strictly decreases on +every incumbent improvement and is bounded above by +$|\mathcal{F}^{\mathrm{dep}}(B,H)|-1$, only finitely many drops are needed to +reach rank $0$. Hence rank $0$ is reached almost surely. +\end{proof} + +\begin{remark} +Theorem~\ref{thm:idealized-asymptotic-hit} is an idealized asymptotic statement. +Finite-budget runtime with hard deadline $T_{\mathrm{total}}$ and optional +\texttt{max\_evaluations} remains governed by the non-asymptotic statements in +this section. +\end{remark} + +\section{Computational Complexity}\label{sec:computational-complexity} + +Let: +\begin{itemize} + \item $d$ the fixed canonical dataset object, + \item $S_H(d)$ the size of the lowered charged event stream under profile + $H$, + \item $N_H(d)$ the declared throughput denominator, + \item $N$ attempted proposals, + \item $U\le N$ unique canonical candidates evaluated or cache-hit, + \item $|z|$ canonical candidate representation size, + \item $C_{\mathrm{eval}}(d,z)$ evaluator runtime on dataset $d$. +\end{itemize} + +Per proposal attempt, overhead is +\[ +O\big(C_{\mathrm{mut}}(z,B)+C_{\mathrm{canon}}(|z|)+C_{\mathrm{valid}}(z,B)+C_{\mathrm{hash}}(|z|)\big), +\] +with $C_{\mathrm{canon}}(|z|)=O(|z|\log|z|)$ under key sorting, and $C_{\mathrm{hash}}(|z|)=O(|z|)$. + +For cache misses, evaluation cost is bounded by hard timeout: +\[ +C_{\mathrm{eval}}(z) \le +\min\{C_{\mathrm{eval}}(d,z),T_t^{\mathrm{eval}}\}+C_{\mathrm{kill}}, +\qquad +T_t^{\mathrm{eval}}\le T_{\mathrm{total}}-t. +\] + +Hence evaluation runtime is globally deadline-bounded by design, with only +constant finalization overhead after the deadline check: +\[ +\mathrm{Runtime} \le T_{\mathrm{total}} + O(C_{\mathrm{finalize}}). +\] + +Memory complexity is dominated by cache: +\[ +O\big(U\cdot(|\operatorname{ser}(\operatorname{canon}(z))|+|M(z)|)\big). +\] + +For the MC-AIXI/AIQI-family runtime path, let $N_{\mathrm{dec}}$ be the number of +real decision steps and let +\[ +N_{\mathrm{sim}}:=\sum_{t=0}^{N_{\mathrm{dec}}-1}N_{\mathrm{sim},t} +\] +be total planner-internal simulations, where $N_{\mathrm{sim},t}$ is +simulations at decision step $t$ (bounded by +\texttt{planner\_simulations\_per\_step}). If +$C_{\mathrm{sim}}(t)$ denotes per-simulation planner compute at step $t$, then +controller-internal planning overhead is +\[ +O\!\left(\sum_{t=0}^{N_{\mathrm{dec}}-1}N_{\mathrm{sim},t}\,C_{\mathrm{sim}}(t)\right) +=O\big(N_{\mathrm{sim}}\,\overline C_{\mathrm{sim}}\big), +\] +with $\overline C_{\mathrm{sim}}$ the average per-simulation planner cost. +This term affects search-control compute only and does not alter evaluator, +validity, or output semantics. + +\section{Implementation Constraints for \texttt{infotheory}} + +\subsection{Semantic equivalence to the shared spec/runtime pipeline} + +Evaluation must preserve the same semantics as compiling and executing the same +candidate through the shared \texttt{CompressionBackend}/\texttt{SpecDocument} +validation and runtime pipeline. The tuner must not introduce a second +candidate parser, a second method-normalization path, or a second canonicalizer +for backend graphs. Library-internal calls are acceptable only if they are +behaviorally equivalent to the standalone compiled backend path. + +\subsection{CLI exposure} + +Expose capability as +\begin{verbatim} +infotheory tune +\end{verbatim} +with one canonical \texttt{"kind": "tune"} top-level document, consistent +with the shared spec-document loader. + +\subsection{Required report and provenance fields} + +The structured report should record at minimum: +\begin{itemize} + \item baseline and best metrics + ($c_{\mathrm{phys}},\ell,\tau,\mu,\mathcal{J}_H$), + \item operational reporting quantities are recorded with units explicitly: + physical compressed bytes $c_{\mathrm{phys}}$ when defined, + model-code bytes $L_B$, target-loss bits $\ell$, and objective bits + $\mathcal{J}_H$, + \item if a physical encoded artifact exists, any finite-coder relation + between $8c_{\mathrm{phys}}$ and $\ell$ must be labeled as diagnostic only + and not treated as the normative objective identity, + \item deltas and evaluation counters, + \item timeout, invalid, and successful-but-nondeployable counts, + \item final best objective, + \item final best-move reward, + \item crate/tool version, + \item feature set (if available), + \item seed, + \item timing-certification tier and determinism/deadline certificate (if + theorem certification is claimed), + \item parent-controller thread count, evaluator-worker thread count, and + worker isolation mode, + \item evaluator determinism profile declaration (for exact theorem claims: + deterministic under $H$), + \item configured throughput floor $\theta_{\min}$ and derived runtime cap + $\tau_{\theta}=N_H(d)/\theta_{\min}$, with $\tau_{\theta}=n/\theta_{\min}$ + in passive-byte mode, + \item configured memory cap $\mu_{\max}$, + \item input asset identifier, resolved dataset path, and dataset content hash, + \item baseline candidate canonical hash and baseline candidate model bytes + $L_B(z_0)$, + \item initial teacher-dataset asset identifier and provenance/hash (if used), + \item warmup policy and warmup count $w_{\mathrm{base}}$, + \item self-improvement round count $R_{\mathrm{si}}$ and realized per-round trace counts (if enabled), + \item bounds hash, + \item candidate canonicalization classification identifier/version (or hash commitment), + \item observation-adapter specification (or hash commitment), theorem-side + exact-state encoder specification (or hash commitment), and provenance flag + for exact-state observation certification, + \item output candidate canonical hash and output config path. +\end{itemize} + +\subsection{Cache key requirements} + +Use the canonical key definition from Section~\ref{sec:canon-cache}: +\[ +\mathrm{Key}_{\mathrm{cache}}(z,\eta_{\mathrm{eval}},d)= +\Big(\operatorname{ser}(\operatorname{canon}(z)),\eta_{\mathrm{eval}},\operatorname{Hash}(d)\Big). +\] + +In runtime loops with remaining-budget truncation, +$\eta_{\mathrm{eval}}$ must include the effective evaluation timeout +$T_{\mathrm{eff}}=\min(T_{\mathrm{eval}},B_t)$ used for that call. + +For all controller paths, before evaluating a canonical candidate $\hat z$, the +runtime must query the cache key above instantiated at +$(\hat z,\eta_{\mathrm{eval}},d)$. On cache hit, the cached evaluator output +$M(\hat z)$ is reused and no new compression execution occurs. Any +state-relative reward or observation components are recomputed from that cached +evaluator output and the current runtime state. This ensures reasonable +performance, without affecting behaviour under hardware determinism +assumptions. + +\section{Expected Theoretical Behavior}\label{sec:expected-theoretical-behavior} + +Under fixed hard budgets and bounded mutation grammar, this method is a practical anytime algorithm: +\begin{itemize} + \item it is guaranteed to return a deployable incumbent (baseline or better), + \item it enforces throughput and memory limits as hard deployability constraints, + \item it never promotes infeasible outcomes, + \item it trades early exploration for late exploitation through temperature decay, + \item it preserves exact objective semantics via $\mathcal{J}_H$ on the deployable region. +\end{itemize} + +The exact-$\mathcal{J}_H$ statement applies to the annealed hill-climbing +controller and to planner controllers under the objective-aligned +objective-difference reward path from +Section~\ref{sec:reward-observation}. + +No claim of global optimality is made under finite budgets. Under the default +reversible elementary Metropolis rule, any deployable uphill move reachable by +the certified proposal grammar has strictly positive acceptance probability: +\[ +\Delta\mathcal{J}>0,\ T(u)>0,\ q_{B,u}(z'\mid z_{\mathrm{cur}})>0 +\implies +\mathbb{P}_{\mathrm{accept}}(u)= +\exp(-\Delta\mathcal{J}/T(u))>0. +\] +Thus the default certified annealer retains the usual Metropolis uphill-move +mechanism, whereas deterministic hill climbing assigns probability $0$ to +uphill moves. + +The asymptotic claims in this document are: +\begin{itemize} + \item the annealed idealized eventual-hit theorem + (Theorem~\ref{thm:idealized-asymptotic-hit}), conditional on + Section~\ref{sec:annealed-asymptotic-scope}, and + \item the planner-side MC-AIXI asymptotic convergence result + (Theorem~\ref{thm:planner-convergence-mcaixi}, with + Proposition~\ref{prop:planner-convergence-optset} giving the broader + set-valued non-unique optimal-sequence variant), conditional on + Section~\ref{sec:planner-convergence-mcaixi}. +\end{itemize} +Both are assumption-conditional asymptotic statements and are outside +finite-budget runtime guarantees. + +\section{Exact planner-state embedding for MC-AIXI(FAC-CTW) under a fixed hardware profile}\label{sec:aixi-embedding} + +This section formalizes a planner-facing environment whose dynamics are Markov +by construction, while separating two logically distinct use-cases: +\begin{enumerate} + \item a budget-free finite-horizon planning variant for direct + $\rho$UCT / MC-AIXI consistency statements, and + \item an optional reset-augmented continuing variant for stationary/ergodic + finite-state Markov analysis. +\end{enumerate} +All finite-MDP and observed-Markov claims in this section are conditional on +Assumptions~\ref{ass:finite-Z}--\ref{ass:no-hidden}. + +\subsection{Fixed-profile semantics} + +Fix a hardware/evaluator profile +\[ +\begin{aligned} +H=\bigl(&\text{machine class},\text{CPU pinning policy},\text{RAM cap},\\ +&\text{build/features},\eta_{\mathrm{eval}},\\ +&\text{wall-clock timing methodology},\text{timing certification tier},\\ +&\text{determinism/deadline certificate},\text{seed policy}\bigr). +\end{aligned} +\] +All planner-facing semantics below are defined relative to $H$. + +\begin{definition}[Timing-certification tier] +The evaluator profile $H$ declares one of the following timing tiers: +\begin{enumerate} + \item \texttt{best\_effort}: ordinary operating-system timing; suitable for + heuristic tuning and reporting, but not for theorem-certified + wall-clock-dependent claims; + \item \texttt{isolated}: CPU affinity, fixed governor, controlled + background load, memory limits, and reproducibility metadata; + \item \texttt{real\_time}: real-time scheduler or hard-real-time operating + system with declared deadline, preemption, memory-locking, and interrupt + isolation policy; + \item \texttt{deterministic\_table}: evaluator metrics are taken from a + frozen semantic table $M_H^\circ$, not measured online. +\end{enumerate} +Planner theorems involving wall-clock-dependent observations, rewards, or +transitions may be certified only under \texttt{real\_time} with a recorded +determinism/deadline certificate, or under +\texttt{deterministic\_table}. +\end{definition} + +Unless explicitly stated otherwise, this section applies to the fixed canonical +dataset object $d$ selected by the input contract. The planner-facing exact +objective is always +\[ +\mathcal{J}_H(z;d)=8L_B(z)+\ell_H(z) +\] +on deployable candidates, where $\ell_H(z)$ is the evaluator's target loss in +bits under the selected passive or causal lowering. Physical compressed bytes, +when present, are diagnostic only. The passive-byte exact-encoding case is the +special case in which a material encoder also reports +$c_{\mathrm{phys}}(z)$ and a finite-coder overhead relation between +$8c_{\mathrm{phys}}(z)$ and $\ell_H(z)$. + +The exact MC-AIXI convergence claims below require a finite planner-exposed +candidate domain, deterministic evaluator semantics, theorem-certified timing +for any wall-clock-dependent quantities, exact-state observation when +observed-Markov claims are invoked, and exact finite reward encoding of +objective differences as specified in +Remark~\ref{rem:reward-encoding-safety}. They do not require $d$ to be a +passive-byte dataset. + +If evaluator-worker-internal multithreading is enabled +(\texttt{evaluator\_threads}>1), any floating-point reduction order or runtime +effect that could change evaluator outputs must either be made deterministic +under $H$ or be declared as non-theorem-certified profile behavior. Exact +theorem claims require deterministic evaluator semantics under the declared +$H$ profile. + +\begin{assumption}[Finite planner-exposed candidate domain]\label{ass:finite-Z} +The planner-exposed canonical candidate set +\[ +\mathcal{Z}_H^{\mathrm{can}} \subseteq \mathcal{Z} +\] +is finite. +\end{assumption} + +\begin{remark} +Assumption~\ref{ass:finite-Z} is normative for this embedding. Any parameter +that would otherwise range over an infinite or continuous set must be +explicitly discretized before planner exposure; this discretization is part of +$B$ and part of the canonicalization contract. +\end{remark} + +Define status alphabet +\[ +\mathcal{S}_{\mathrm{status}} +:= +\{\mathrm{success},\mathrm{timeout},\mathrm{invalid},\mathrm{error}\}. +\] +Let $\mathcal{D}_H$ be a finite diagnostic codomain containing distinguished +token +\[ +d_{\mathrm{inapp}}:=\texttt{inapplicable\_action}, +\] +and let $\bot_c,\bot_\ell,\bot_\tau,\bot_\mu$ denote failure sentinels for +undefined size/log-loss/time/memory fields. Define evaluator codomain +\[ +\mathcal{Y}_H +:= +\mathcal{S}_{\mathrm{status}} +\times (\mathbb{N}_0\sqcup\{\bot_c\}) +\times (\mathbb{R}_{\ge 0}\sqcup\{\bot_\ell\}) +\times (\mathbb{R}_{\ge 0}\sqcup\{\bot_\tau\}) +\times (\mathbb{R}_{\ge 0}\sqcup\{\bot_\mu\}) +\times \mathcal{D}_H. +\] + +\begin{assumption}[Profile-fixed semantic evaluator]\label{ass:det-eval} +There exists a deterministic semantic evaluator +\[ +M_H^\circ:\mathcal{Z}_H^{\mathrm{can}}\to\mathcal{Y}_H, +\qquad +M_H^\circ(z)=\big(\mathrm{status}_H(z),c_H(z),\ell_H(z),\tau_H(z),\mu_H(z),\operatorname{diag}_H(z)\big), +\] +with: +\begin{itemize} + \item $\mathrm{status}_H(z)\in\mathcal{S}_{\mathrm{status}}$, + \item $c_H(z)\in\mathbb{N}_0\sqcup\{\bot_c\}$, with $c_H(z)\in\mathbb{N}_0$ + exactly when a material encoded artifact is produced under profile $H$, + \item $\ell_H(z)\in\mathbb{R}_{\ge 0}$ on success and + $\ell_H(z)=\bot_\ell$ otherwise, + \item $\tau_H(z)\in\mathbb{R}_{\ge 0}$ on success and + $\tau_H(z)=\bot_\tau$ otherwise, + \item $\mu_H(z)\in\mathbb{R}_{\ge 0}$ on success and + $\mu_H(z)=\bot_\mu$ otherwise, + \item $\operatorname{diag}_H(z)\in\mathcal{D}_H$. +\end{itemize} +This is the fixed-profile specialization of the six-field evaluator contract +from Section~\ref{sec:formal-problem}. Here $\tau_H(z)$ denotes elapsed +wall-clock time in the same unit family as $\tau(z)$, $T_{\mathrm{eval}}$, +and $T_{\mathrm{total}}$. Any additional deterministic profile costs (for +example, PMU cycles) may be recorded in $\operatorname{diag}_H(z)$, but are +not denoted by $\tau_H(z)$. No equality between $\ell_H(z)$ and physical +compressed bytes is assumed. When a material encoded artifact exists, its +physical byte length $c_H(z)$ is diagnostic/reporting only. The exact planner +objective uses $\ell_H(z)$ directly. +\end{assumption} + +\begin{remark}[Deterministic timing requirement for theorem mode] +The exact finite-MDP, exact observed-Markov, and planner-convergence claims +require $M_H^\circ$ to be a deterministic semantic map. Ordinary wall-clock +measurements are admissible for operational tuning, but they are not by +themselves theorem-certified semantic values. + +For theorem-certified runs, the implementation must therefore use either: +\begin{enumerate} + \item timing tier \texttt{real\_time} with a recorded + determinism/deadline certificate, meaning a real-time scheduler or hard + real-time operating system together with declared deadline, interrupt, + memory-locking, and preemption policy; or + \item timing tier \texttt{deterministic\_table}, where evaluator outputs + are taken from a frozen semantic table fixed before planner interaction. +\end{enumerate} +Timing tiers \texttt{best\_effort} and \texttt{isolated} remain conforming for +operational tuning and reporting, but any wall-clock-dependent exact-MDP, +exact observed-Markov, or planner-convergence claim is then reported as +uncertified. +\end{remark} + +For successful candidates under profile $H$, define throughput +\[ +\theta_H(z):= +\begin{cases} +\dfrac{N_H(d)}{\tau_H(z)}, & \mathrm{status}_H(z)=\mathrm{success},\ \tau_H(z)>0,\\[4pt] ++\infty, & \mathrm{status}_H(z)=\mathrm{success},\ \tau_H(z)=0. +\end{cases} +\] + +\begin{remark} +Assumption~\ref{ass:det-eval} does not remove hardware from the model. It +incorporates hardware methodology into $H$; once $H$ is fixed, evaluator +semantics are required to be deterministic relative to that profile. +\end{remark} + +\begin{assumption}[Deployable baseline initialization]\label{ass:baseline-succ} +This is the fixed-profile restatement of the normative input precondition from +Section~\ref{subsec:baseline-precondition}. + +In a conforming runtime, $b_0$ is the incumbent produced by the single +post-warmup evaluation of $z_0^{\mathrm{can}}$. + +The initial incumbent $b_0\in\mathcal{Z}_H^{\mathrm{can}}$ satisfies +\[ +\mathrm{status}_H(b_0)=\mathrm{success} +\quad\land\quad +\theta_H(b_0)\ge\theta_{\min} +\quad\land\quad +\mu_H(b_0)\le\mu_{\max}. +\] +\end{assumption} + +Define the deployable canonical subset +\[ +\mathcal{Z}_{H,\mathrm{dep}}^{\mathrm{can}}:= +\left\{z\in\mathcal{Z}_H^{\mathrm{can}}:\ \mathrm{status}_H(z)=\mathrm{success}\ \land\ \theta_H(z)\ge\theta_{\min}\ \land\ \mu_H(z)\le\mu_{\max}\right\}. +\] +By Assumption~\ref{ass:baseline-succ}, this set is nonempty. + +Define the profile-fixed two-part objective +\[ +L_B(z):=\big|\operatorname{ser}(z)\big|, +\] +and objective +\[ +\mathcal{J}_H(z;d):= +\begin{cases} +8L_B(z)+\ell_H(z), +& z\in\mathcal{Z}_{H,\mathrm{dep}}^{\mathrm{can}},\\[3pt] ++\infty, +& \text{otherwise.} +\end{cases} +\] +This enforces a single deployability-gated objective in the planner path: +non-deployable candidates receive $+\infty$ exactly as timeout/invalid/error +outcomes. On the deployable region, physical byte size $c_H(z)$, when present, +is a diagnostic/reporting quantity. The planner objective is +\[ +\mathcal{J}_H(z;d)=8L_B(z)+\ell_H(z), +\] +with no general equality assumed between $\ell_H(z)$ and $8c_H(z)$. + +Define incumbent key +\[ +K_H(z):=\big(\mathcal{J}_H(z;d),\operatorname{ser}(z)\big), +\] +ordered lexicographically. + +\subsection{State, action, and omitted-variable discipline} + +Fix planner-interface integer parameters +\[ +\begin{aligned} +N_B&:=\texttt{agent\_actions}=N_B^{\mathrm{compile}}(B)\in\mathbb{N},\ N_B\ge 1,\\ +b_O&:=\texttt{observation\_bits}\in\mathbb{N}_0,\\ +L_O&:=\max(1,\texttt{observation\_stream\_len}). +\end{aligned} +\] +Define +\[ +\mathcal{A}_B=\{0,\dots,N_B-1\}, +\qquad +\mathcal{O}_B=\{0,\dots,2^{b_O}-1\} +\] +(with $\mathcal{O}_B=\{0\}$ when $b_O=0$). + +For all planner reward semantics, define +\[ +b_R:=\texttt{reward\_bits}\in\mathbb{N},\qquad b_R\ge 1, +\qquad +\mathcal{R}_B^{\mathrm{enc}}:=\{0,\dots,2^{b_R}-1\}. +\] +For \texttt{normalized\_clipped\_improvement}, additionally fix the +controller-specific clipping parameters +\[ +\begin{aligned} +r_{\min}^{\mathrm{clip}}&:=\texttt{controller.min\_improvement},\\ +r_{\max}^{\mathrm{clip}}&:=\texttt{controller.max\_improvement},\\ +r_{\max}^{\mathrm{clip}}&>r_{\min}^{\mathrm{clip}}. +\end{aligned} +\] +These parameters determine only the normalization interval used by +Section~\ref{sec:aiqi-discounted-contract}; they are not part of exact +objective-difference semantics. +Initialization must verify cardinality consistency +\[ +|\mathcal{A}_B|=\texttt{agent\_actions} +\] +after compiling the action alphabet from $B$; on mismatch, initialization aborts +with configuration error \texttt{action\_alphabet\_mismatch}. + +Let $\Theta_H$ be the complete \emph{finite} auxiliary-state set containing +exactly the remaining variables, if any, that influence future planner-facing +dynamics and are not functions of proposal source and incumbent alone. + +\begin{remark}[Cache and budget variables are state unless theorem-inert] +For runtime planner-controller execution, cache contents, the set of evaluated +or cache-reused candidates, remaining wall-clock budget, evaluation counters, +controller RNG state, proposal RNG state, and any pending delayed-label +buffers can influence future emitted observations, rewards, termination, or +final output. + +Therefore, for any theorem claiming the finite-MDP or observed-Markov path, a +conforming implementation must do exactly one of the following: +\begin{enumerate} + \item include these variables explicitly in $\Theta_H$; or + \item prove and record in provenance that they are planner-facing inert + under the theorem semantics, for example because the theorem model uses a + fixed semantic evaluator table $M_H^\circ$, ignores cache-hit timing as an + observation, and uses the budget-free finite-horizon variant. +\end{enumerate} +Absent one of these two conditions, the finite-MDP and exact observed-Markov +claims are disabled for that run. +\end{remark} + +\begin{assumption}[No hidden state outside chosen state tuple]\label{ass:no-hidden} +Every implementation variable that influences future emitted observation, +emitted reward, or decoded candidate is either: +\begin{enumerate} + \item included in $\Theta_H$, or + \item planner-facing inert: changing it while holding the chosen state tuple + fixed leaves all future emitted transitions/rewards/observations unchanged. +\end{enumerate} +\end{assumption} + +Define planner state space +\[ +\mathcal{S}_H:= +\mathcal{Z}_H^{\mathrm{can}}\times \mathcal{Z}_{H,\mathrm{dep}}^{\mathrm{can}}\times \Theta_H, +\] +with state $s=(u,b,\theta)$ where $u$ is proposal source, $b$ incumbent, +$\theta$ auxiliary state. + +Action decoding is the total deterministic map +\[ +\mu_B^{\mathrm{tot}}:\mathcal{A}_B\times\mathcal{S}_H\to +\mathcal{Z}_H^{\mathrm{can}}\times\{0,1\}, +\qquad +(a,s)\mapsto(\tilde z,\iota), +\] +followed by canonicalization +\[ +z(a,s):=\operatorname{canon}(\tilde z). +\] +Canonical inapplicable-action semantics are invalid-without-evaluation: +when $\iota=1$, no evaluator execution is performed and no compression run is +started. + +Let +\[ +y_H(a,s):= +\begin{cases} +\bigl(\mathrm{invalid},\bot_c,\bot_\ell,\bot_\tau,\bot_\mu,d_{\mathrm{inapp}}\bigr), +& \iota=1,\\[3pt] +M_H^\circ\bigl(z(a,s)\bigr), +& \iota=0. +\end{cases} +\] +Define exact-objective feasibility predicate +\[ +\mathrm{Feas}_H(z):=\mathbf{1}\{\mathrm{status}_H(z)=\mathrm{success}\ \land\ \theta_H(z)\ge\theta_{\min}\ \land\ \mu_H(z)\le\mu_{\max}\}. +\] +Define incumbent update +\[ +\operatorname{Inc}_H(s,a):= +\begin{cases} +z(a,s), +& \iota=0\ \land\ \mathrm{Feas}_H(z(a,s))=1 +\ \land\ +K_H(z(a,s))\prec_K K_H(b),\\[3pt] +b, +& \text{otherwise.} +\end{cases} +\] + +Let +\[ +U_H:\mathcal{S}_H\times\mathcal{A}_B\to\mathcal{S}_H +\] +be the deterministic next-state map, required to satisfy +\[ +\pi_b\bigl(U_H(s,a)\bigr)=\operatorname{Inc}_H(s,a), +\] +where $\pi_b(u,b,\theta)=b$ extracts the incumbent component. + +\subsection{Objective-aligned reward} + +Define planner-semantic reward by incumbent objective decrease: +\[ +r_H(s,a):=\mathcal{J}_H\!\bigl(\pi_b(s);d\bigr)-\mathcal{J}_H\!\bigl(\pi_b(U_H(s,a));d\bigr). +\] +On any realized planner trajectory with $b_t=\pi_b(s_t)$, this agrees +stepwise with Section~\ref{sec:reward-observation}'s reward +$R_t^{\mathcal{J}}$. +This is finite and well-defined on reachable trajectories because incumbent +components are always deployable. + +\begin{proposition}[Undiscounted telescoping identity]\label{prop:telescoping} +For every finite trajectory +\[ +s_0,a_0,s_1,a_1,\dots,s_m, +\qquad +s_{t+1}=U_H(s_t,a_t), +\] +we have +\[ +\sum_{t=0}^{m-1} r_H(s_t,a_t) += +\mathcal{J}_H\!\bigl(\pi_b(s_0);d\bigr)-\mathcal{J}_H\!\bigl(\pi_b(s_m);d\bigr). +\] +\end{proposition} + +\begin{proof} +Immediate from summing +\[ +r_H(s_t,a_t)=\mathcal{J}_H\!\bigl(\pi_b(s_t);d\bigr)-\mathcal{J}_H\!\bigl(\pi_b(s_{t+1});d\bigr). +\] +\end{proof} + +\subsection{Observation encoding and exact observed-Markov path} + +\begin{proposition}[Baseline-derived reachable reward bound] +\label{prop:reward-bound} +Under Assumption~\ref{ass:baseline-succ}, every reachable planner-semantic +reward satisfies +\[ +0\le r_H(s,a)\le \overline R_H, +\qquad +\overline R_H:=\mathcal{J}_H(b_0;d). +\] +\end{proposition} + +\begin{proof} +Along every reachable trajectory from $s_0$, incumbent updates never worsen the +key and therefore never increase the incumbent objective. Hence for every +reachable state $s$, +\[ +0\le \mathcal{J}_H\!\bigl(\pi_b(s);d\bigr)\le \mathcal{J}_H(b_0;d)=\overline R_H. +\] +Also, +\[ +\mathcal{J}_H\!\bigl(\pi_b(U_H(s,a));d\bigr) +\le +\mathcal{J}_H\!\bigl(\pi_b(s);d\bigr), +\] +so +\[ +0 +\le +r_H(s,a) += +\mathcal{J}_H\!\bigl(\pi_b(s);d\bigr)-\mathcal{J}_H\!\bigl(\pi_b(U_H(s,a));d\bigr) +\le +\overline R_H. +\] +\end{proof} + +Choose deterministic state encoder +\[ +\Psi_H:\mathcal{S}_H\to\mathcal{O}_B^{L_O}. +\] + +\begin{definition}[Exact-state observation path] +The embedding uses the \emph{exact-state observation path} if $\Psi_H$ is +injective. +\end{definition} + +\begin{definition}[Exact-state observation certificate] +A run has an \emph{exact-state observation certificate} if the implementation +constructs, records in provenance, and validates a theorem-side encoder +\[ +\Psi_H:\mathcal{S}_H\to\mathcal{O}_B^{L_O} +\] +and verifies that $\Psi_H$ is injective on the planner state space relevant to +the claimed theorem. + +The public observation pipeline +\[ +(\texttt{observation\_bits}, +\texttt{observation\_stream\_len}, +\texttt{observation\_key\_mode}, +\Psi_O) +\] +is the runtime observation/keying interface. It does not by itself license the +exact observed-Markov claims unless it is also used to construct an injective +$\Psi_H$, or unless an implementation supplies a separate injective $\Psi_H$ +and records the relationship between the public pipeline and $\Psi_H$ in +provenance. +\end{definition} + +\begin{remark}[Runtime bridge for exact-state claims] +At planner-controller initialization, the implementation must set a provenance +flag +\[ +\texttt{exact\allowbreak\_state\allowbreak\_observation\allowbreak\_certified} +\in\{\texttt{true},\texttt{false}\}. +\] +It may set this flag to \texttt{true} only after constructing a concrete +$\Psi_H$ and verifying its injectivity on the relevant finite state space, or +after loading a proof/certificate accepted by the implementation. + +If the flag is \texttt{false}, the run may still execute normally, but reports +must not claim the exact observed-$1$-Markov path or any theorem that depends +on injective $\Psi_H$. + +Among built-in public key modes, \texttt{"full\_stream"} is the only default +candidate for this certificate. The modes \texttt{"first"}, +\texttt{"last"}, and \texttt{"stream\_hash"} are lossy unless a separate +injectivity proof is supplied for the concrete finite state space. +\end{remark} + +A sufficient cardinality condition is +\[ +2^{b_OL_O}\ge |\mathcal{S}_H|. +\] + +\begin{remark}[Normative exact reward-encoding safety check] +\label{rem:reward-encoding-safety} +For exact-objective-difference semantics, +initialization must verify that the declared canonical scalar representation +$\mathbb{V}_H^{\mathcal J}$ is finite, that the encoded reward alphabet has +sufficient cardinality to injectively encode it, +and that the evaluator guarantees exact representability of all exact-mode +objective values and reachable incumbent objective differences. It must then +construct the corresponding injective encoder $\Omega_H$ and decoder $V_H$. +If this check fails, initialization aborts with configuration error +\texttt{reward\_encoding\_unsafe}. + +A shifted integer-interval encoder is merely a sufficient implementation +shortcut for the special case where all reachable objective differences are +already known to lie in a finite integer interval. It is not the general exact +reward-encoding contract and is not part of the normative document surface. + +For quantized objective-difference semantics, the +runtime uses the declared quantizer contract from +Section~\ref{sec:reward-observation}; exact $\mathcal{J}_H$ theorem claims do +not apply unless that quantizer is injective on the reachable reward set and its +decoder is exact there. + +For normalized-clipped-improvement semantics, the +runtime verifies only the normalized-clipping preconditions required by +Section~\ref{sec:aiqi-discounted-contract}. +\end{remark} + +For any planner reward semantics with finite encoded reward alphabet, the emitted percept is +\[ +x_{t+1}^{\mathrm{enc}}= +\bigl(\Psi_H(s_{t+1}),\Omega_H(r_H(s_t,a_t))\bigr), +\] +and the planner's semantic reward value for that symbol is +\[ +V_H(\Omega_H(r_H(s_t,a_t))). +\] +In exact-objective-difference mode this semantic value equals +$r_H(s_t,a_t)$ on every reachable transition. + +\begin{proposition}[Finite MDP by construction]\label{prop:finite-mdp} +Under Assumptions~\ref{ass:finite-Z}--\ref{ass:no-hidden}, the induced +planner-facing process is a finite time-homogeneous MDP +\[ +\mathfrak{M}_H:=(\mathcal{S}_H,\mathcal{A}_B,T_H,R_H), +\] +with deterministic transition kernel +\[ +T_H(s'\mid s,a):=\mathbf{1}\{U_H(s,a)=s'\}, +\] +and reward function +\[ +R_H(s,a,s'):=r_H(s,a). +\] +\end{proposition} + +\begin{proof} +By Assumption~\ref{ass:no-hidden}, every variable affecting future +planner-facing emissions is either part of $s$ or planner-facing inert. Hence +$s_{t+1}$ and $r_t$ are functions only of $(s_t,a_t)$. Since +$\mathcal{S}_H$ and $\mathcal{A}_B$ are finite, the process is a finite +time-homogeneous MDP. +\end{proof} + +\begin{proposition}[Exact observed $1$-Markov path]\label{prop:obs-1-markov} +If the exact-state observation path is used, then the emitted percept process is +observed $1$-Markov: +\[ +\Pr\!\bigl(x_{t+1}^{\mathrm{enc}}\mid a_t,x_{1:t}^{\mathrm{enc}},a_{1:t-1}\bigr) += +\Pr\!\bigl(x_{t+1}^{\mathrm{enc}}\mid a_t,x_t^{\mathrm{enc}}\bigr). +\] +\end{proposition} + +\begin{proof} +Injectivity of $\Psi_H$ implies $x_t^{\mathrm{enc}}$ determines $s_t$ +uniquely. Apply Proposition~\ref{prop:finite-mdp}. +\end{proof} + +\subsection{Budget-free finite-horizon planning variant} + +In the budget-free variant, no countdown or terminal flag is part of +environment state. The planning horizon $m\in\mathbb{N}$ is an external planner +parameter. Proposition~\ref{prop:finite-mdp} then yields a finite +(time-homogeneous) MDP for every fixed planning horizon $m$. + +\subsection{Conditional planner convergence on exact finite-horizon MC-AIXI path}\label{sec:planner-convergence-mcaixi} + +Define the objective-optimal deployable set +\[ +\mathcal{Z}_{H,\mathrm{obj}}^{\star}:= +\arg\min_{z\in\mathcal{Z}_{H,\mathrm{dep}}^{\mathrm{can}}}\mathcal{J}_H(z;d), +\] +and its deterministic tie-broken representative +\[ +z_H^{\star}:=\arg\min_{z\in\mathcal{Z}_{H,\mathrm{obj}}^{\star}}K_H(z), +\] +which is unique by finiteness and total ordering of $K_H$. + +For fixed horizon $m\in\mathbb{N}$ and action sequence +$a_{0:m-1}\in\mathcal{A}_B^m$, define the deterministic rollout +\[ +s_0^{a}:=s_0, +\qquad +s_{t+1}^{a}:=U_H(s_t^{a},a_t), +\quad t=0,\dots,m-1, +\] +and deterministic undiscounted return +\[ +G_m(s_0,a_{0:m-1}) +:= +\sum_{t=0}^{m-1} r_H(s_t^{a},a_t). +\] + +\begin{assumption}[Planner-convergence regime]\label{ass:planner-conv-regime} +The planner is MC-AIXI(FAC-CTW) with $\rho$UCT on the exact model +$\mathfrak{M}_H=(\mathcal{S}_H,\mathcal{A}_B,T_H,R_H)$, +using objective-aligned reward $r_H$, deterministic undiscounted +finite-horizon return $G_m$, +and the budget-free finite-horizon variant of this section. +\end{assumption} + +\begin{assumption}[Horizon-$m$ attainability with unique optimal action path]\label{ass:planner-conv-unique-path} +There exists $m\in\mathbb{N}$ and a unique action sequence +$a_{0:m-1}^{\star}\in\mathcal{A}_B^m$. Define optimal-prefix states by +\[ +s_0^{\star}:=s_0, +\qquad +s_{t+1}^{\star}:=U_H(s_t^{\star},a_t^{\star}), +\quad t=0,\dots,m-1. +\] +Then: +\begin{enumerate} + \item $\pi_b(s_m^{\star})\in\mathcal{Z}_{H,\mathrm{obj}}^{\star}$, and + \item for every other sequence $a_{0:m-1}\neq a_{0:m-1}^{\star}$, + \[ + G_m(s_0,a_{0:m-1}^{\star}) > G_m(s_0,a_{0:m-1}), + \] +\end{enumerate} +\end{assumption} + +\begin{assumption}[Per-decision asymptotic root-action consistency]\label{ass:planner-conv-root-consistency} +For each simulation budget $N\in\mathbb{N}$, let +$S_t^{(N)}\in\mathcal{S}_H$ be the closed-loop state process defined by +\[ +S_0^{(N)}:=s_0, +\qquad +S_{t+1}^{(N)}:=U_H\big(S_t^{(N)},a_t^{(N)}\big), +\] +where $a_t^{(N)}$ is the root action selected at step $t$ using $N$ +planner-internal simulations. For each $t\in\{0,\dots,m-1\}$, +\[ +\mathbb{P}\!\big(a_t^{(N)}=a_t^{\star}\mid S_t^{(N)}=s_t^{\star}\big) +\xrightarrow[N\to\infty]{}1. +\] +\end{assumption} + +\begin{theorem}[Conditional MC-AIXI convergence to the objective-optimal set]\label{thm:planner-convergence-mcaixi} +Under Assumptions~\ref{ass:planner-conv-regime}--\ref{ass:planner-conv-root-consistency} +and Proposition~\ref{prop:finite-mdp}, +consider the closed-loop process indexed by simulation budget $N$, and define +$b_t^{(N)}:=\pi_b\big(S_t^{(N)}\big)$. Then +\[ +\mathbb{P}\!\big(b_m^{(N)}\in\mathcal{Z}_{H,\mathrm{obj}}^{\star}\big) +\xrightarrow[N\to\infty]{}1. +\] +\end{theorem} + +For the non-unique maximizer case, see Proposition~\ref{prop:planner-convergence-optset}. + +\begin{proof} +We prove by induction that +\[ +\mathbb{P}\!\big(S_t^{(N)}=s_t^{\star}\big)\xrightarrow[N\to\infty]{}1 +\quad\text{for each }t=0,\dots,m. +\] +Base case $t=0$ is immediate from $S_0^{(N)}=s_0=s_0^{\star}$. + +Induction step: assume +$\mathbb{P}(S_t^{(N)}=s_t^{\star})\to 1$ for some $tr_{\min}^{\mathrm{clip}}, + \] + equivalently + \[ + \texttt{controller.max\_improvement} + > + \texttt{controller.min\_improvement}. + \] + \end{itemize} + \item construct or classify the theorem-side observation encoder + $\Psi_H:\mathcal{S}_H\to\mathcal{O}_B^{L_O}$ and record provenance flag + \texttt{exact\allowbreak\_state\allowbreak\_observation\allowbreak\_certified}. This flag may be set to + \texttt{true} only if the implementation has constructed a concrete + $\Psi_H$ and verified injectivity on the relevant finite state space, or + has loaded an accepted proof/certificate of that injectivity. If the flag + is \texttt{false}, runtime may continue, but reports must not claim the + exact observed-$1$-Markov path or any theorem requiring injective + $\Psi_H$. If theorem-facing reports request such claims without the + certificate, initialization must either abort with configuration error + \texttt{exact\allowbreak\_state\allowbreak\_observation\allowbreak\_uncertified} or continue with those + claims disabled, + \item if theorem-facing reports request exact finite-MDP, + exact observed-Markov, or planner-convergence certification, verify that + the timing-certification tier in $H$ is either + \texttt{real\_time} with a recorded determinism/deadline certificate or + \texttt{deterministic\_table}. Otherwise initialization must either abort + with configuration error + \texttt{timing\allowbreak\_theorem\allowbreak\_uncertified} or continue + with those theorem-facing claims disabled, + \item instantiate planner-facing environment state $s_0\in\mathcal{S}_H$ + with the post-warmup baseline incumbent satisfying the normative baseline + precondition from Section~\ref{subsec:baseline-precondition} + (equivalently Assumption~\ref{ass:baseline-succ} in the fixed-profile + analysis). +\end{enumerate} + +\paragraph{Controller-specific internal policy contract.} +Initialize controller-specific internal state by +\[ +\begin{aligned} +u_0^{\mathrm{mc}}&\ \text{(MC-AIXI/FAC-CTW)},\\ +u_0^{\mathrm{aiqi,disc}}&\ \text{(discounted variant)},\\ +u_0^{\mathrm{aiqi,warm}}&\ \text{(warm-start exact variant)}. +\end{aligned} +\] +Apply seeded randomized tie-breaking for MC-AIXI; apply deterministic +lowest-index greedy tie-breaking for \texttt{aiqi\_discounted} +(Section~\ref{sec:aiqi-discounted-contract}) and for the exact-greedy +warm-start path (Section~\ref{sec:aiqi-warmstart-exact-jh}); explicit +exploration parameters are heuristic. At each decision step $t$, allocate +planner-internal simulation count +\[ +N_{\mathrm{sim},t}\in\{1,\dots,N_{\mathrm{sim}}^{\max}\} +\] +as permitted by remaining wall-clock budget, run those internal simulations, +and then select external action $a_t\in\mathcal{A}_B$ from the current +controller state. + +For \texttt{controller.kind = aiqi\_warmstart\_exact\allowbreak\_jh} with +$R_{\mathrm{si}}>1$, this single-pass execution is used as the per-round +kernel inside the bounded same-task trace-refresh wrapper of +Section~\ref{sec:aiqi-warmstart-self-improve}. The global wall-clock budget +remains $T_{\mathrm{total}}$, the next round starts from the current outer +incumbent, and the final output is still the minimum-$K_H$ deployable candidate +over all real evaluations performed across all rounds. + +\paragraph{Shared environment, termination, and output contract.} +\begin{enumerate} + \item environment applies deterministic transition + $s_{t+1}=U_H(s_t,a_t)$ and emits encoded percept + \[ + x_{t+1}^{\mathrm{enc}}= + \bigl(\Psi_H(s_{t+1}),\Omega_H(r_H(s_t,a_t))\bigr), + \] + with canonical inapplicable-action semantics inherited from + Section~\ref{sec:aixi-embedding}: when decoder flag $\iota=1$, the emitted + outcome is invalid with diagnostic token + \texttt{inapplicable\_action}, no evaluator execution occurs, and no + compression run is started, + \item planner updates its internal state using + $(a_t,x_{t+1}^{\mathrm{enc}})$ and repeats until termination criteria are + met, + \item termination criteria are the same normative criteria used by the + tuning runtime (global deadline, \texttt{max\_evaluations} if set, + neighborhood exhaustion when provable, unrecoverable evaluator failure), + \item final output candidate is + \[ + z^\star=\arg\min_{z\in\mathcal{S}_{\mathrm{term}}^{\mathrm{dep}}}K_H(z), + \] + where + \[ + \mathcal{S}_{\mathrm{term}}^{\mathrm{dep}}:= + \left\{z\in\mathcal{Z}_H^{\mathrm{can}}:\ + \begin{aligned}[t] + &z\text{ evaluated or cache-reused before termination},\\ + &\mathrm{status}_H(z)=\mathrm{success},\\ + &\theta_H(z)\ge\theta_{\min},\ \mu_H(z)\le\mu_{\max} + \end{aligned} + \right\}. + \] +\end{enumerate} +Controller-specific search-control dynamics may differ, but scoring/output +semantics and candidate-validity semantics remain shared. + +\begin{remark}[Heuristic-mode boundary for controller execution] +Within the planner-controller family, two kinds of runs are permitted by this +specification. Exact-theorem-facing runs are those that satisfy the explicit +certification conditions stated later for reward encoding, state exposure, +timing, and the exact warm-start controller path. Heuristic-mode runs are +still conforming controller executions, but they are outside those theorem +certifications; this includes the discounted/bin-quantized +\texttt{aiqi\_discounted} path, any planner run using explicit exploration +parameters, and any annealer run that leaves the certified reversible +elementary Metropolis kernel or the optional certified +\texttt{compiled\_uniform\_metropolis\_hastings} profile in favor of +heuristic macro moves, adaptive proposals, or non-certified acceptance laws. +This boundary affects theorem-facing claims only: deployability semantics, +candidate scoring, invalid/inapplicable handling, and final output selection +remain shared unless a section explicitly states otherwise. +\end{remark} + +\subsection{Normative discounted AIQI controller contract} +\label{sec:aiqi-discounted-contract} + +This subsection is normative for +\texttt{controller.kind = aiqi\_discounted}. It defines a +discounted, bin-quantized heuristic controller path. It does \emph{not} +provide exact-$\mathcal{J}_H$ finite-horizon objective-alignment guarantees. +The controller uses the induced +\texttt{"normalized\_clipped\_improvement"} reward semantics. + +Define controller parameters +\[ +H_{\mathrm{ret}}:=\texttt{return\_horizon}\in\mathbb{N},\ H_{\mathrm{ret}}\ge 1, +\qquad +\gamma:=\texttt{discount\_factor}\in[0,1), +\] +\[ +M:=\texttt{return\_bins}\in\mathbb{N}, +\qquad +M\text{ is a power of two}. +\] +\[ +\begin{aligned} +r_{\min}^{\mathrm{clip}}&:=\texttt{controller.min\_improvement},\\ +r_{\max}^{\mathrm{clip}}&:=\texttt{controller.max\_improvement},\\ +r_{\max}^{\mathrm{clip}}&>r_{\min}^{\mathrm{clip}}. +\end{aligned} +\] +These are the explicit clipping parameters for the discounted controller. + +Define clamp on unit interval +\[ +\operatorname{clamp}_{[0,1]}(x):=\min\!\bigl(\max(x,0),1\bigr). +\] +Let +\[ +r_t:=\mathcal{J}_H(b_t;d)-\mathcal{J}_H(b_{t+1};d) +\] +denote the underlying incumbent-improvement source signal. Under the +normalized-clipped-improvement reward semantics, the discounted controller +clips and normalizes this signal and then observes the finite reward symbol +\[ +\Omega_H(r_t) += +\Omega_H^{\mathrm{clip}} +\left( +\operatorname{clamp}_{[0,1]} +\left( +\frac{r_t-r_{\min}^{\mathrm{clip}}}{r_{\max}^{\mathrm{clip}}-r_{\min}^{\mathrm{clip}}} +\right) +\right) +\] +and uses the decoded representative value +\[ +\hat r_t:=V_H(\Omega_H(r_t))\in[0,1]. +\] +The discounted finite-horizon target is +\[ +G_{t,H_{\mathrm{ret}}}^{(\gamma)}:= +\operatorname{clamp}_{[0,1]}\!\left( +(1-\gamma)\sum_{k=0}^{H_{\mathrm{ret}}-1}\gamma^k\hat r_{t+k} +\right). +\] + +Define return-bin quantizer +\[ +Q_{\mathrm{return}}(G):= +\min\!\left(\left\lfloor M G\right\rfloor,\ M-1\right), +\qquad +G\in[0,1]. +\] +The discounted controller predicts/optimizes the distribution of +$Q_{\mathrm{return}}(G_{t,H_{\mathrm{ret}}}^{(\gamma)})$, not exact +$\mathcal{J}_H$-improvement targets. + +Greedy action ties are broken deterministically by lowest action index; any +explicit exploration parameter is heuristic and outside exact-$\mathcal{J}_H$ +claims. + +\subsection{MC-AIXI-warm-started model-free exact-\texorpdfstring{$\mathcal{J}_H$}{J\_H} controller} +\label{sec:aiqi-warmstart-exact-jh} + +This subsection defines a model-free controller path whose $m$-step prediction +target is exactly aligned with the deployable tuning objective $\mathcal{J}_H$ +via terminal-incumbent improvement, while using MC-AIXI-generated teacher data +to initialize the label predictor. It is inspired by the delayed-label +phase-predictor structure of AIQI, but it does \emph{not} use discounted return +targets and does \emph{not} invoke the discounted AIQI theorem directly. +It is normative for +\texttt{controller.kind = aiqi\_warmstart\_exact\allowbreak\_jh}. +Its exact finite-horizon objective-alignment claims are stated for the same +deployable objective $\mathcal{J}_H(\cdot;d)$ as the tuning runtime on the +fixed same-task environment induced by $(B,H,d)$. +Conforming implementations may also expose +an internal \texttt{"quantized\_objective\_difference"} variant, but +the exact theorem statements below concern +\texttt{"exact\_objective\_difference"}. + +\paragraph{Setup.} +Fix the same planner-facing finite interaction contract as in +Section~\ref{sec:reward-observation}: finite action alphabet +$\mathcal{A}_B$, finite encoded observation alphabet, deterministic incumbent +key, and the objective-aligned reward +\[ +R_t^{\mathcal{J}}:=\mathcal{J}_H(b_t;d)-\mathcal{J}_H(b_{t+1};d). +\] +Define +\[ +m:=\texttt{return\_horizon}\in\mathbb{N},\qquad m\ge 1. +\] + +Define the exact $m$-step cumulative improvement target +\[ +G_t^{(m)} +:= +\sum_{k=0}^{m-1} R_{t+k}^{\mathcal{J}}. +\] +Because the reward is incumbent-objective difference, this telescopes exactly: +\[ +G_t^{(m)} += +\mathcal{J}_H(b_t;d)-\mathcal{J}_H(b_{t+m};d). +\] +Hence maximizing expected $G_t^{(m)}$ is equivalent to minimizing the expected +terminal incumbent objective after $m$ further planner steps. + +\begin{remark}[Finite exact target encoding] +The exact $m$-step improvement label +\[ +G_t^{(m)} += +\mathcal{J}_H(b_t;d)-\mathcal{J}_H(b_{t+m};d) +\] +is represented using the same finite canonical scalar representation +$\mathbb{V}_H^{\mathcal J}$ used by the exact reward encoder. Define the exact +label alphabet by the encoded scalar values +\[ +\mathcal{G}_H^{\mathrm{enc}}:=\Omega_H(\mathbb{V}_H^{\mathcal J}). +\] +A realized label is stored as the pair +\[ +\bigl(\Omega_H(G_t^{(m)}),\;V_H(\Omega_H(G_t^{(m)}))\bigr), +\] +or equivalently as the encoded symbol together with the declared decoder +$V_H$. It is not necessary to enumerate all deployable incumbent pairs before +runtime. Conforming implementations may represent predictors over +$\mathcal{G}_H^{\mathrm{enc}}$ sparsely or through a byte/symbol sequence model. + +For analysis, let +\[ +\overline G_m:=\mathcal{J}_H(b_0;d), +\] +which is a valid upper bound because incumbents never worsen and objectives are +nonnegative. +\end{remark} + +\paragraph{Initial teacher dataset and admissible same-task trace sources.} +Let +\[ +\mathcal{D}_{\mathrm{teach}} += +\bigl\{ +(h_{ +\Delta_t(h_{\widehat Q_m(h_{0$, then the executed +policy is exploratory and therefore is not exactly greedy at that decision +step. If one wants the executed action to coincide with the greedy rule +analyzed here, use $\varepsilon_t=0$ at deployment; exact optimality at that +step additionally requires the predictor-equality premise in +Proposition~\ref{prop:aiqi-warmstart-exact-jh}. +\end{remark} + +\begin{remark}[Implementation-conformance split] +Conforming implementations must keep the controller kinds +\texttt{aiqi\_discounted} and \texttt{aiqi\_warmstart\_exact\allowbreak\_jh} +distinct. Reusing discounted return-bin targets under the exact warm-start +controller key is non-conforming to this specification. Likewise, any bounded +self-improvement trace-refresh wrapper may merge only admissible same-task +exact-label traces as defined above; reusing differently encoded or surrogate- +labeled traces is non-conforming. +\end{remark} + +\subsection{Optional bounded same-task trace-refresh self-improvement loop} +\label{sec:aiqi-warmstart-self-improve} + +This subsection defines an optional bounded wrapper around +\texttt{aiqi\_warmstart\_exact\allowbreak\_jh}. It reuses the same exact +label mechanism on the original tuning task; it does \emph{not} introduce any +auxiliary objective. + +Let +\[ +R_{\mathrm{si}}:=\texttt{self\_improvement\_rounds}\in\mathbb{N}, +\qquad +R_{\mathrm{si}}\ge 1, +\] +with default $R_{\mathrm{si}}=1$. +When $R_{\mathrm{si}}>1$, interpret $T_{\mathrm{total}}$ as the single +outer-loop wall-clock budget and define deterministic round deadlines +\[ +\Delta_r^{\mathrm{si}}:=\frac{r}{R_{\mathrm{si}}}T_{\mathrm{total}}, +\qquad r=0,\dots,R_{\mathrm{si}}. +\] + +Let the round-$r$ teacher dataset be $\mathcal{D}^{(r)}$, with initial dataset +$\mathcal{D}^{(0)}:=\mathcal{D}_{\mathrm{teach}}$, and define +\[ +\beta_0:=b_0, +\qquad +\mathcal{S}_0^{\mathrm{meta}}:=\{\beta_0\}. +\] +For each round $r\in\{0,\dots,R_{\mathrm{si}}-1\}$: +\begin{enumerate} + \item initialize the phase predictors from $\mathcal{D}^{(r)}$, + \item initialize the round-$r$ controller with incumbent $\beta_r$ and + execute the exact-$\mathcal{J}_H$ controller on the same $(B,H,d)$ task + until the earlier of global termination and wall-clock deadline + $\Delta_{r+1}^{\mathrm{si}}$, + \item let $L_r\in\mathbb{N}_0$ be the number of executed actions in round + $r$, and define the realized labeled round-trace set + \[ + \mathcal{T}^{(r)} + := + \left\{ + (h_{ \ + --teacher \ + --out +\end{verbatim} +The target planner run must use \texttt{aiqi\_warmstart\_exact\_jh}. The +teacher planner run may use any CLI-executable planner controller. The export +command: +\begin{enumerate} + \item compiles both planner-run documents through the canonical spec + pipeline; + \item verifies that target and teacher have identical planner interface + values and identical canonical environment JSON; + \item executes the teacher controller for its configured learning and + evaluation cycles; + \item records realized transitions after each action; + \item rejects the run if fewer than \(H\) transitions were produced; + \item builds the target standalone teacher contract; + \item validates the resulting artifact by constructing the target + \Warm{} agent from it; + \item writes pretty canonical teacher JSON to \texttt{--out}. +\end{enumerate} +This realizes the offline-to-online path: any controller can act as teacher, +and the resulting artifact initializes a \Warm{} student for the target task. + +The reproducibility CLI also exposes: +\begin{verbatim} +infotheory warmstart teacher from-jsonl \ + --target --jsonl --out + +infotheory warmstart teacher merge \ + --target --out \ + --teacher --teacher ... +\end{verbatim} +The first command implements the deterministic telemetry converter from +Section~\ref{sec:telemetry}. The second performs the deterministic +same-contract trace union defined there and validates the merged result by +constructing the target student. + +\section{Online Trace Refresh} +\label{sec:refresh} + +During online deployment, \Warm{} can export its own live same-task trace after +at least \(H\) transitions. The live trace is not itself a teacher artifact; it +becomes teacher data only when it is wrapped under the unchanged target +contract, merged into the teacher dataset, and validated by the same full +dataset gate used for ordinary teacher artifacts. A refresh wrapper may then +reinitialize the student from the merged dataset. + +A bounded refresh loop is specified by an initial teacher dataset +\(\Teach^{(0)}\), a maximum number of refresh rounds \(R\), and a finite run +budget per round. Round \(r\) builds \(W^{(r)}\) from \(\Teach^{(r)}\), executes +it on the same compiled bridge, extracts a live trace \(\tau^{(r)}\) if one has +at least \(H\) transitions, and sets +\[ +\Teach^{(r+1)} += +\operatorname{Merge}(\Teach^{(r)},\tau^{(r)}). +\] +\(\operatorname{Merge}\) is the deterministic trace-content union of +Section~\ref{sec:teacher-artifact}. The loop stops after \(R\) rounds, or +earlier if no admissible live trace is produced. + +\begin{proposition}[Refresh trace payload admissibility] +If a live \Warm{} trace of length at least \(H\) is produced by a target +runtime that validates every emitted action, observation, and reward against +the compiled target bridge, then wrapping that trace under the unchanged target +contract yields transition payloads whose complete horizon windows are +admissible teacher labels for that same target contract. +\end{proposition} + +\begin{proof} +Every recorded action, observation, and reward has already passed the target +runtime's transition validator. Each label is the deterministic \(H\)-step sum +of rewards in that same trace. The trace length premise gives at least one +complete delayed-label window, and the unchanged wrapper contract supplies the +same fingerprint, interface fields, horizon, phase period, and provenance +fields required by Definition~\ref{def:admissible-teacher}. Therefore the +wrapped live trace can pass the teacher-artifact validator as part of the +merged dataset. +\end{proof} + +Refresh is a data mechanism, not a monotonic-improvement theorem. It improves +the theorem-facing local bound only to the extent that the refreshed predictor +is closer to the target conditional label law on visited histories. + +\begin{theorem}[Conditional local refresh improvement] +Fix a history \(h\) and suppose two same-bridge students differ only in their +teacher artifact: an initial student \(W_0\) and a refreshed student \(W_1\). +If, for every action, +\[ +\|\widehat p_{W_1}(\cdot\mid h,a)-p(\cdot\mid h,a)\|_{\TV} +< +\|\widehat p_{W_0}(\cdot\mid h,a)-p(\cdot\mid h,a)\|_{\TV}, +\] +then the local finite-horizon value-error bound of +Theorem~\ref{thm:tv-robustness} is strictly smaller for \(W_1\) than for +\(W_0\) at \(h\). +\end{theorem} + +\begin{proof} +The bound in Theorem~\ref{thm:tv-robustness} is monotone in the uniform +total-variation radius. A strict decrease for every action gives a strict +decrease of the supremum radius at \(h\), and therefore a strict decrease of +the displayed value-error bound. +\end{proof} + +For bridges with monotone incumbent semantics, such as the tuner exact-\(\Jh\) +bridge, the bridge may prove additional best-so-far or objective-monotonicity +claims. Those claims belong to the bridge specification; they are not implied +by refresh alone for arbitrary environments. + +\section{Core Theorems} +\label{sec:theorems} + +Let \(p_t(\ell\mid h,a)\) be the true conditional law of the exact +\(H\)-step return label at history \(h\) under action \(a\). Define +\[ +Q^\star_H(h,a)=\sum_{\ell\in\Lab}\Vlab(\ell)p_t(\ell\mid h,a). +\] +Let +\[ +D_G=\max_{\ell\in\Lab}\Vlab(\ell)-\min_{\ell\in\Lab}\Vlab(\ell). +\] +For the direct integer-reward bridge, \(D_G=Hr_{\max}-Hr_{\min}=Hr_{\max}\). + +\begin{theorem}[Exact greedy alignment] +If, at a greedy decision history \(h\), +\[ +\widehat p_t(\cdot\mid h,a)=p_t(\cdot\mid h,a) +\quad\text{for every }a\in\Act, +\] +then \Warm{} selects an action maximizing the true expected exact +\(H\)-step return \(Q^\star_H(h,a)\), with deterministic lowest-index +tie-breaking. +\end{theorem} + +\begin{proof} +Under the premise, \(\widehat Q_H(h,a)=Q^\star_H(h,a)\) for every action. +The implemented rule chooses the lowest-index maximizer of these same values. +\end{proof} + +\begin{theorem}[Total-variation robustness] +\label{thm:tv-robustness} +If +\[ +\sup_{a\in\Act} +\|\widehat p_t(\cdot\mid h,a)-p_t(\cdot\mid h,a)\|_{\TV} +\le \delta, +\] +then the greedy \Warm{} action \(\hat a\) satisfies +\[ +Q^\star_H(h,a^\star)-Q^\star_H(h,\hat a)\le 2D_G\delta, +\] +where \(a^\star\) is any true optimal action. +\end{theorem} + +\begin{proof} +For any bounded function with range diameter \(D_G\), expectation changes by +at most \(D_G\) times total variation. Thus +\[ +|\widehat Q_H(h,a)-Q^\star_H(h,a)|\le D_G\delta +\] +for every action. Since \(\hat a\) maximizes \(\widehat Q_H\), +\[ +Q^\star_H(h,a^\star) +\le \widehat Q_H(h,a^\star)+D_G\delta +\le \widehat Q_H(h,\hat a)+D_G\delta +\le Q^\star_H(h,\hat a)+2D_G\delta. +\] +\end{proof} + +\begin{corollary}[Teacher action-gap preservation] +If the teacher value \(\QT\) has a unique greedy action \(a_T\) at \(h\) and +\[ +\sup_{a\in\Act}|\widehat Q_W(h,a)-\QT(h,a)|<\Delta_T(h)/2, +\] +then the greedy warmstart action equals \(a_T\). In particular, the condition +holds whenever +\[ +\sup_{a\in\Act}\|\widehat p_W(\cdot\mid h,a)-\PT(\cdot\mid h,a)\|_{\TV} +\le\delta +\quad\text{and}\quad +D_G\delta<\Delta_T(h)/2. +\] +\end{corollary} + +\begin{proof} +For every action, the value-error premise gives an interval of radius +strictly less than \(\Delta_T(h)/2\) around \(\QT(h,a)\). The teacher-best +interval remains strictly above every competitor interval, so the warmstart +maximizer is the teacher maximizer. The distributional condition implies the +value-error premise by the same total-variation bound used in +Theorem~\ref{thm:tv-robustness}. +\end{proof} + +\section{Tuner Bridge Instantiation} +\label{sec:tuner-bridge} + +The tuner bridge instantiates the generic reward \(r_t\) as exact improvement +in the deployable two-part objective: +\[ +\Jh(z;d)=8L_B(z)+\ell_H(z). +\] +For incumbent sequence \(b_t\), the exact reward is +\[ +r_t=\Jh(b_{t-1};d)-\Jh(b_t;d) +\] +after the bridge's finite reward encoder has certified representability and +injectivity. Therefore the \(H\)-step label telescopes: +\[ +G_i^{(H)} +=\sum_{j=i}^{i+H-1}r_j +=\Jh(b_{i-1};d)-\Jh(b_{i+H-1};d). +\] +This is the tuner theorem use case. Its dataset lowering, deployability, +timing, exact reward certificate, and observation adapter are governed by the +tuner specification. \Warm{} consumes the resulting externally validated +teacher artifact through the same contract fields, but this standalone +document does not weaken or replace the tuner theorem assumptions. + +\section{Implementation Conformance} +\label{sec:implementation} + +A conforming implementation must: +\begin{itemize} + \item route planner-run documents through the canonical spec + validate/compile pipeline; + \item reject zero return horizon, zero return bins, phase period below + horizon, planner simulation budgets other than the direct-evaluator marker + \(1\), unsupported predictor conditioning, and malformed teacher contracts; + \item validate every teacher and live transition against action, + observation, and reward bounds before learning from it; + \item use deterministic LSB-first fixed-width field encodings for actions, + observations, and rewards, and MSB-first value-monotone label codewords; + \item reset transient predictor conditioning history between independent + teacher traces; + \item expose canonical teacher JSON serialization and parsing as inverse + operations on valid artifacts; + \item keep JSONL telemetry separate from canonical teacher artifacts; + \item merge same-contract traces by deterministic content deduplication; + \item record whether actions were greedy or exploratory in planner JSONL + telemetry when provenance is present. +\end{itemize} + +\section{Traceability Obligations} +\label{sec:traceability} + +The implementation and tests should maintain the following traceability map: +\begin{itemize} + \item compiled bridge contract: parser, canonical JSON, binary round-trip, + compile, and fingerprint tests for \texttt{aiqi\_warmstart\_exact\_jh}; + \item admissible teacher artifacts: schema parse/serialize inverse tests, + contract mismatch rejection tests, standalone planner-run export tests, and + tuner-bridge validation tests owned by the tuner specification; + \item JSONL conversion: same-step action/percept pairing tests, + pre-action percept next-step pairing tests, mixed-convention rejection + tests, malformed/duplicate rejection tests, and final target-agent + construction from converted data; + \item deterministic merge and refresh: structural trace-content deduplication + tests, same-contract validation tests, and rebuilt-agent construction from + merged data; + \item online delayed labels: within-run transition observation tests showing + a newly closed \(H\)-window updates exactly one phase model; + \item implementation anchors: +\begin{verbatim} +PlannerActionProvenance +warmstart_teacher_trace_from_jsonl_reader +warmstart_teacher_trace_from_jsonl_path +merge_warmstart_teacher_traces_deterministic +standalone_warmstart_teacher_contract_for_compiled_planner_run +warmstart_exact_jh_planner_task_fingerprint +WarmStartExactJhAgent::from_compiled_planner_run +validate_warmstart_exact_jh_teacher_contract +load_warmstart_exact_jh_teacher_dataset +PlannerControllerAgent::from_compiled +PlannerEnvironment +PlannerRunSession +run_warmstart_teacher_planner_run_export +run_warmstart_teacher_from_jsonl_export +run_warmstart_teacher_merge +run_compiled_planner_run +\end{verbatim} + \item theorem claims: total-variation/action-gap claims are mathematical + obligations; implementation tests must verify their preconditions are + exposed and not silently inferred from unvalidated data. +\end{itemize} + +\section{Claim Boundary} +\label{sec:claims} + +\Warm{} does not claim universal optimality, discounted-return optimality, +wall-clock dominance over its teacher, or automatic repair of missing +counterfactual action coverage. The exact theorem is local and finite-horizon: +if the conditional label law is exact at the current history, greedy +\Warm{} is exact for the current \(H\)-step return objective. Offline teacher +data, online refresh, and exploration are mechanisms for making that local +model useful and reproducible; their stronger empirical claims must be +measured and reported as experimental results. + +\end{document} diff --git a/examples/config.json b/examples/config.json index 3e81ceff..e32c392e 100644 --- a/examples/config.json +++ b/examples/config.json @@ -3,18 +3,18 @@ "alpha": 0.03, "experts": [ { - "name": "ctw", - "kind": "ctw", + "log_prior": 0, + "name": "fac-ctw", "base_depth": 32, - "encoding_bits": 8, + "kind": "fac-ctw", "num_percept_bits": 8, - "log_prior": 0.0 + "encoding_bits": 8 }, { - "name": "rosa", + "log_prior": 0, "kind": "rosaplus", "max_order": -1, - "log_prior": 0.0 + "name": "rosa" } ] } diff --git a/examples/mixture_spec.json b/examples/mixture_spec.json index 7f7ef0f0..05f185ad 100644 --- a/examples/mixture_spec.json +++ b/examples/mixture_spec.json @@ -1,40 +1,42 @@ { - "kind": "switching", - "alpha": 0.02, "experts": [ { "name": "rosa-6", + "log_prior": 0, "kind": "rosaplus", - "max_order": 6, - "log_prior": 0.0 + "max_order": 6 }, { + "encoding_bits": 8, + "log_prior": 0, + "base_depth": 16, "name": "ctw-16", - "kind": "ctw", - "depth": 16, - "log_prior": 0.0 + "kind": "fac-ctw", + "num_percept_bits": 8 }, { "name": "nested-bayes", - "kind": "mixture", - "log_prior": -0.2, "spec": { "kind": "bayes", "experts": [ { + "encoding_bits": 8, "name": "fac-ctw-12", - "kind": "fac-ctw", "base_depth": 12, - "encoding_bits": 8, + "kind": "fac-ctw", "num_percept_bits": 8 }, { "name": "zpaq-2", - "kind": "zpaq", - "method": "2" + "method": "2", + "kind": "zpaq" } ] - } + }, + "kind": "mixture", + "log_prior": -0.2 } - ] + ], + "kind": "switching", + "alpha": 0.02 } diff --git a/examples/rosa_ctw.json b/examples/rosa_ctw.json index 1ed64d63..ce6aa9ca 100644 --- a/examples/rosa_ctw.json +++ b/examples/rosa_ctw.json @@ -1,20 +1,20 @@ { "kind": "neural", - "alpha": 0.04, "experts": [ { - "name": "rosa", "kind": "rosaplus", - "max_order": -1, - "log_prior": 0.0 + "log_prior": 0, + "name": "rosa", + "max_order": -1 }, { - "name": "ctw", - "kind": "ctw", - "base_depth": 32, - "encoding_bits": 8, + "log_prior": 0, + "name": "fac-ctw", + "kind": "fac-ctw", "num_percept_bits": 8, - "log_prior": 0.0 + "encoding_bits": 8, + "base_depth": 32 } - ] + ], + "alpha": 0.04 } diff --git a/examples/simple.json b/examples/simple.json index 82bb706c..e1f23e8a 100644 --- a/examples/simple.json +++ b/examples/simple.json @@ -3,8 +3,8 @@ "alpha": 0.04, "experts": [ { - "name": "rwkv", - "kind": "rwkv", + "name": "rwkv7", + "kind": "rwkv7", "method": "cfg:hidden=64,layers=1,intermediate=64,decay_rank=16,a_rank=16,v_rank=16,g_rank=16,seed=22,train=adam,lr=0.0009,stride=1;policy:schedule=0..50%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=1,clip=1.0,momentum=0.9)", "log_prior": 0.0 diff --git a/examples/three.json b/examples/three.json index b2c20885..7b8248a0 100644 --- a/examples/three.json +++ b/examples/three.json @@ -1,26 +1,26 @@ { - "kind": "neural", - "alpha": 0.03, "experts": [ { - "name": "ctw", - "kind": "ctw", - "base_depth": 32, - "encoding_bits": 8, "num_percept_bits": 8, - "log_prior": 0.0 + "encoding_bits": 8, + "log_prior": 0, + "name": "fac-ctw", + "kind": "fac-ctw", + "base_depth": 32 }, - { + { + "log_prior": 0, "name": "rosa", - "kind": "rosaplus", "max_order": -1, - "log_prior": 0.0 + "kind": "rosaplus" }, - { - "name": "mamba", - "kind": "mamba", - "method": "cfg:hidden=128,layers=1,intermediate=128,seed=26,train=adam,lr=0.001,stride=1;policy:schedule=0..10%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=1,clip=0,momentum=0.9)", - "log_prior": 0.0 - } - ] + { + "log_prior": 0, + "name": "mamba", + "method": "cfg:hidden=128,layers=1,intermediate=128,seed=26,train=adam,lr=0.001,stride=1;policy:schedule=0..10%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=1,clip=0,momentum=0.9)", + "kind": "mamba" + } + ], + "alpha": 0.03, + "kind": "neural" } diff --git a/examples/tuner/README.md b/examples/tuner/README.md new file mode 100644 index 00000000..edad3ae9 --- /dev/null +++ b/examples/tuner/README.md @@ -0,0 +1,150 @@ +# Tuner Validation Command Suite + +This README is a depot for manual validation, as much of it requires root and is not automated. + +Before copy-pasting the commands below, set the repository root once: + +```bash +export INFOTHEORY_REPO=/path/to/infotheory +``` + +The tuner, at least for passive compression, outputs a raw CompressionBackend JSON Object. By the library, you can use parse_compression_backend_json directly on this output. + +You can compress with the CompressionBackend object like this: +```bash +infotheory compress /input.bin /output.bin --compression-backend-json /path/to/compression_backend.json +``` + +## 1) One-time delegated cgroup-v2 setup + +```bash +cd "$INFOTHEORY_REPO" +sudo ./scripts/delegate_tuner_cgroup_v2.sh setup theo infotheory-tuner +``` + +## 2) Strict smoke checks + +Strict non-MC-AIXI smoke: + +```bash +cd "$INFOTHEORY_REPO" +sudo --preserve-env=PATH ./scripts/delegate_tuner_cgroup_v2.sh run-in-session theo infotheory-tuner -- \ + env INFOTHEORY_TUNER_EVAL_CGROUP_PARENT=/sys/fs/cgroup/infotheory-tuner/evals \ + cargo run -p infotheory --no-default-features --features 'tuner cli backend-ctw' -- \ + tune examples/tuner/strict-smoke-spec.json \ + --rss-mode hybrid_strict_max \ + --evaluator-cgroup-parent /sys/fs/cgroup/infotheory-tuner/evals \ + --max-evaluations 1 +``` + +Strict MC-AIXI smoke: + +```bash +cd "$INFOTHEORY_REPO" +sudo --preserve-env=PATH ./scripts/delegate_tuner_cgroup_v2.sh run-in-session theo infotheory-tuner -- \ + env INFOTHEORY_TUNER_EVAL_CGROUP_PARENT=/sys/fs/cgroup/infotheory-tuner/evals \ + cargo run -p infotheory --no-default-features --features 'tuner cli backend-ctw' -- \ + tune examples/tuner/strict-smoke-mc-aixi-spec.json \ + --exact-reward-encoding-certificate examples/tuner/strict-smoke-mc-aixi-reward-cert.json \ + --rss-mode hybrid_strict_max \ + --evaluator-cgroup-parent /sys/fs/cgroup/infotheory-tuner/evals \ + --max-evaluations 1 +``` + +## 3) Strict TSV examples + +Annealed simple TSV: + +```bash +cd "$INFOTHEORY_REPO" +sudo --preserve-env=PATH ./scripts/delegate_tuner_cgroup_v2.sh run-in-session theo infotheory-tuner -- \ + env INFOTHEORY_TUNER_EVAL_CGROUP_PARENT=/sys/fs/cgroup/infotheory-tuner/evals \ + cargo run -p infotheory --no-default-features --features 'tuner cli backend-ctw' -- \ + tune examples/tuner/strict-annealed-simple-tsv-spec.json \ + --rss-mode hybrid_strict_max \ + --evaluator-cgroup-parent /sys/fs/cgroup/infotheory-tuner/evals \ + --max-evaluations 1 +``` + +Annealed advanced neural-mixture TSV: + +```bash +cd "$INFOTHEORY_REPO" +sudo --preserve-env=PATH ./scripts/delegate_tuner_cgroup_v2.sh run-in-session theo infotheory-tuner -- \ + env INFOTHEORY_TUNER_EVAL_CGROUP_PARENT=/sys/fs/cgroup/infotheory-tuner/evals \ + cargo run -p infotheory --no-default-features --features 'tuner cli backend-ctw backend-mixture' -- \ + tune examples/tuner/strict-annealed-advanced-neural-mixture-tsv-spec.json \ + --rss-mode hybrid_strict_max \ + --evaluator-cgroup-parent /sys/fs/cgroup/infotheory-tuner/evals \ + --max-evaluations 1 +``` + +MC-AIXI advanced neural-mixture TSV: + +```bash +cd "$INFOTHEORY_REPO" +sudo --preserve-env=PATH ./scripts/delegate_tuner_cgroup_v2.sh run-in-session theo infotheory-tuner -- \ + env INFOTHEORY_TUNER_EVAL_CGROUP_PARENT=/sys/fs/cgroup/infotheory-tuner/evals \ + cargo run -p infotheory --no-default-features --features 'tuner cli backend-ctw backend-mixture' -- \ + tune examples/tuner/strict-mcaixi-advanced-neural-mixture-tsv-spec.json \ + --exact-reward-encoding-certificate examples/tuner/strict-mcaixi-advanced-neural-mixture-tsv-exact-reward-cert-hybrid-strict-max.json \ + --rss-mode hybrid_strict_max \ + --evaluator-cgroup-parent /sys/fs/cgroup/infotheory-tuner/evals \ + --max-evaluations 1 +``` + +## 4) Strict three-mode `two.json` comparison artifact + +Walkthrough: + +- [walkthrough_improving_config.md](walkthrough_improving_config.md) + +Script: + +- [benchmark_tuner_two_json_modes.sh](../../scripts/benchmark_tuner_two_json_modes.sh) + +Run default strict comparison (subject = `git show HEAD:README.md`): + +```bash +cd "$INFOTHEORY_REPO" +sudo --preserve-env=PATH ./scripts/delegate_tuner_cgroup_v2.sh run-in-session theo infotheory-tuner -- \ + env INFOTHEORY_TUNER_EVAL_CGROUP_PARENT=/sys/fs/cgroup/infotheory-tuner/evals \ + ./scripts/benchmark_tuner_two_json_modes.sh +``` + +Run strict comparison on explicit subject file: + +```bash +cd "$INFOTHEORY_REPO" +sudo --preserve-env=PATH ./scripts/delegate_tuner_cgroup_v2.sh run-in-session theo infotheory-tuner -- \ + env INFOTHEORY_TUNER_EVAL_CGROUP_PARENT=/sys/fs/cgroup/infotheory-tuner/evals \ + ./scripts/benchmark_tuner_two_json_modes.sh /path/to/input.bin 2 1 +``` + +## 5) Emit strict exact reward certificates only + +Strict MC-AIXI smoke cert: + +```bash +cd "$INFOTHEORY_REPO" +sudo --preserve-env=PATH ./scripts/delegate_tuner_cgroup_v2.sh run-in-session theo infotheory-tuner -- \ + env INFOTHEORY_TUNER_EVAL_CGROUP_PARENT=/sys/fs/cgroup/infotheory-tuner/evals \ + cargo run -p infotheory --no-default-features --features 'tuner cli backend-ctw' -- \ + tune examples/tuner/strict-smoke-mc-aixi-spec.json \ + --rss-mode hybrid_strict_max \ + --evaluator-cgroup-parent /sys/fs/cgroup/infotheory-tuner/evals \ + --emit-exact-reward-encoding-certificate examples/tuner/strict-smoke-mc-aixi-reward-cert.json +``` + +Strict MC-AIXI advanced TSV cert: + +```bash +cd "$INFOTHEORY_REPO" +sudo --preserve-env=PATH ./scripts/delegate_tuner_cgroup_v2.sh run-in-session theo infotheory-tuner -- \ + env INFOTHEORY_TUNER_EVAL_CGROUP_PARENT=/sys/fs/cgroup/infotheory-tuner/evals \ + cargo run -p infotheory --no-default-features --features 'tuner cli backend-ctw backend-mixture' -- \ + tune examples/tuner/strict-mcaixi-advanced-neural-mixture-tsv-spec.json \ + --rss-mode hybrid_strict_max \ + --evaluator-cgroup-parent /sys/fs/cgroup/infotheory-tuner/evals \ + --emit-exact-reward-encoding-certificate examples/tuner/strict-mcaixi-advanced-neural-mixture-tsv-exact-reward-cert-hybrid-strict-max.json +``` diff --git a/examples/tuner/strict-annealed-advanced-neural-mixture-tsv-spec.json b/examples/tuner/strict-annealed-advanced-neural-mixture-tsv-spec.json new file mode 100644 index 00000000..fb005b34 --- /dev/null +++ b/examples/tuner/strict-annealed-advanced-neural-mixture-tsv-spec.json @@ -0,0 +1,83 @@ +{ + "controller": { + "kind": "annealed_hill_climbing", + "max_mutation_radius": 3 + }, + "input_asset": "dataset", + "baseline_candidate": { + "rate_backend": { + "kind": "mixture", + "spec": { + "schedule": "default", + "alpha": 0.04, + "decay": null, + "experts": [ + { + "encoding_bits": 8, + "base_depth": 12, + "log_prior": 0, + "kind": "fac-ctw", + "num_percept_bits": 8 + }, + { + "encoding_bits": 8, + "kind": "fac-ctw", + "log_prior": 0, + "base_depth": 10, + "num_percept_bits": 8 + }, + { + "encoding_bits": 8, + "base_depth": 8, + "log_prior": 0, + "kind": "fac-ctw", + "num_percept_bits": 8 + } + ], + "kind": "neural" + } + }, + "kind": "rate-ac", + "framing": "framed" + }, + "bounds": { + "allowed_backends": [ + "fac-ctw", + "mixture" + ], + "forbidden_backends": [], + "parameter_ranges": [ + { + "max": 0.2, + "min": 0.01, + "parameter": "rate_backend.spec.alpha" + }, + { + "max": 32, + "min": 4, + "parameter": "rate_backend.spec.experts[0].base_depth" + } + ], + "max_experts": 6, + "max_mixture_nesting_depth": 3, + "min_experts": 2, + "allow_duplicate_experts": false, + "required_experts": [], + "forbidden_expert_pairs": [] + }, + "output_config_path": "strict-annealed-advanced-neural-mixture-tsv-output.json", + "seed": 23, + "report_path": "strict-annealed-advanced-neural-mixture-tsv-report.json", + "max_memory_bytes": 1099511627776, + "min_throughput_bytes_per_second": 1, + "schema_version": 1, + "time_budget_seconds": 300, + "assets": [ + { + "id": "dataset", + "path": "../../benchmarks/6f464811/infotheory-two-json-summary-full.tsv" + } + ], + "kind": "tune", + "eval_time_limit_seconds": 90 +} diff --git a/examples/tuner/strict-annealed-simple-tsv-spec.json b/examples/tuner/strict-annealed-simple-tsv-spec.json new file mode 100644 index 00000000..43c70fcb --- /dev/null +++ b/examples/tuner/strict-annealed-simple-tsv-spec.json @@ -0,0 +1,53 @@ +{ + "kind": "tune", + "controller": { + "max_mutation_radius": 2, + "kind": "annealed_hill_climbing" + }, + "eval_time_limit_seconds": 2, + "time_budget_seconds": 180, + "bounds": { + "min_experts": 1, + "allow_duplicate_experts": false, + "required_experts": [], + "forbidden_expert_pairs": [], + "allowed_backends": [ + "fac-ctw", + "mixture", + "calibrated" + ], + "forbidden_backends": [], + "parameter_ranges": [ + { + "parameter": "rate_backend.base_depth", + "max": 28, + "min": 4 + } + ], + "max_experts": 4, + "max_mixture_nesting_depth": 2 + }, + "schema_version": 1, + "output_config_path": "strict-annealed-simple-tsv-output.json", + "assets": [ + { + "id": "dataset", + "path": "../../benchmarks/6f464811/infotheory-two-json-summary-full.tsv" + } + ], + "seed": 17, + "report_path": "strict-annealed-simple-tsv-report.json", + "max_memory_bytes": 1099511627776, + "input_asset": "dataset", + "baseline_candidate": { + "rate_backend": { + "base_depth": 12, + "kind": "fac-ctw", + "num_percept_bits": 8, + "encoding_bits": 8 + }, + "kind": "rate-ac", + "framing": "framed" + }, + "min_throughput_bytes_per_second": 1 +} diff --git a/examples/tuner/strict-mcaixi-advanced-neural-mixture-tsv-exact-reward-cert-hybrid-strict-max.json b/examples/tuner/strict-mcaixi-advanced-neural-mixture-tsv-exact-reward-cert-hybrid-strict-max.json new file mode 100644 index 00000000..c0e6349e --- /dev/null +++ b/examples/tuner/strict-mcaixi-advanced-neural-mixture-tsv-exact-reward-cert-hybrid-strict-max.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "kind": "exact_reward_encoding", + "dataset_crc32": "850aac41", + "bounds_crc32": "be5166f1", + "evaluator_profile_crc32": "5085d1fd", + "controller_kind": "mc_aixi_fac_ctw", + "action_alphabet_size": 4, + "encoding": "integer_objective_difference", + "scalar_representation": "finite-ieee754-f64-nonfinite-forbidden-v1", + "reward_bits": 32, + "max_reward": 4294967295 +} diff --git a/examples/tuner/strict-mcaixi-advanced-neural-mixture-tsv-exact-reward-cert-process-rss.json b/examples/tuner/strict-mcaixi-advanced-neural-mixture-tsv-exact-reward-cert-process-rss.json new file mode 100644 index 00000000..62d0df46 --- /dev/null +++ b/examples/tuner/strict-mcaixi-advanced-neural-mixture-tsv-exact-reward-cert-process-rss.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "kind": "exact_reward_encoding", + "dataset_crc32": "850aac41", + "bounds_crc32": "be5166f1", + "evaluator_profile_crc32": "bdb72335", + "controller_kind": "mc_aixi_fac_ctw", + "action_alphabet_size": 4, + "encoding": "integer_objective_difference", + "scalar_representation": "finite-ieee754-f64-nonfinite-forbidden-v1", + "reward_bits": 32, + "max_reward": 4294967295 +} diff --git a/examples/tuner/strict-mcaixi-advanced-neural-mixture-tsv-spec.json b/examples/tuner/strict-mcaixi-advanced-neural-mixture-tsv-spec.json new file mode 100644 index 00000000..3dc89c1d --- /dev/null +++ b/examples/tuner/strict-mcaixi-advanced-neural-mixture-tsv-spec.json @@ -0,0 +1,90 @@ +{ + "kind": "tune", + "output_config_path": "strict-mcaixi-advanced-neural-mixture-tsv-output.json", + "eval_time_limit_seconds": 90, + "time_budget_seconds": 300, + "min_throughput_bytes_per_second": 1, + "bounds": { + "min_experts": 2, + "allow_duplicate_experts": false, + "required_experts": [], + "forbidden_expert_pairs": [], + "allowed_backends": [ + "fac-ctw", + "mixture" + ], + "forbidden_backends": [], + "parameter_ranges": [ + { + "min": 0.01, + "max": 0.2, + "parameter": "rate_backend.spec.alpha" + }, + { + "min": 4, + "max": 32, + "parameter": "rate_backend.spec.experts[0].base_depth" + } + ], + "max_experts": 6, + "max_mixture_nesting_depth": 3 + }, + "schema_version": 1, + "seed": 31, + "assets": [ + { + "path": "../../benchmarks/6f464811/infotheory-two-json-summary-full.tsv", + "id": "dataset" + } + ], + "report_path": "strict-mcaixi-advanced-neural-mixture-tsv-report.json", + "max_memory_bytes": 1099511627776, + "controller": { + "kind": "mc_aixi_fac_ctw", + "planner_simulations_per_step": 6, + "interface": { + "observation_bits": 8, + "observation_stream_len": 1, + "observation_key_mode": "full_stream", + "reward_bits": 32, + "agent_actions": 4 + } + }, + "input_asset": "dataset", + "baseline_candidate": { + "kind": "rate-ac", + "rate_backend": { + "kind": "mixture", + "spec": { + "kind": "neural", + "experts": [ + { + "kind": "fac-ctw", + "num_percept_bits": 8, + "log_prior": 0, + "encoding_bits": 8, + "base_depth": 12 + }, + { + "kind": "fac-ctw", + "num_percept_bits": 8, + "encoding_bits": 8, + "log_prior": 0, + "base_depth": 10 + }, + { + "kind": "fac-ctw", + "num_percept_bits": 8, + "log_prior": 0, + "encoding_bits": 8, + "base_depth": 8 + } + ], + "schedule": "default", + "decay": null, + "alpha": 0.05 + } + }, + "framing": "framed" + } +} diff --git a/examples/tuner/strict-mcaixi-simple-tsv-exact-reward-cert-process-rss.json b/examples/tuner/strict-mcaixi-simple-tsv-exact-reward-cert-process-rss.json new file mode 100644 index 00000000..b686e416 --- /dev/null +++ b/examples/tuner/strict-mcaixi-simple-tsv-exact-reward-cert-process-rss.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "kind": "exact_reward_encoding", + "dataset_crc32": "850aac41", + "bounds_crc32": "209df9b3", + "evaluator_profile_crc32": "092d433e", + "controller_kind": "mc_aixi_fac_ctw", + "action_alphabet_size": 2, + "encoding": "integer_objective_difference", + "scalar_representation": "finite-ieee754-f64-nonfinite-forbidden-v1", + "reward_bits": 32, + "max_reward": 4294967295 +} diff --git a/examples/tuner/strict-mcaixi-simple-tsv-spec.json b/examples/tuner/strict-mcaixi-simple-tsv-spec.json new file mode 100644 index 00000000..f277539a --- /dev/null +++ b/examples/tuner/strict-mcaixi-simple-tsv-spec.json @@ -0,0 +1,60 @@ +{ + "kind": "tune", + "assets": [ + { + "path": "../../benchmarks/6f464811/infotheory-two-json-summary-full.tsv", + "id": "dataset" + } + ], + "input_asset": "dataset", + "baseline_candidate": { + "kind": "rate-ac", + "framing": "framed", + "rate_backend": { + "kind": "fac-ctw", + "num_percept_bits": 8, + "encoding_bits": 8, + "base_depth": 10 + } + }, + "eval_time_limit_seconds": 2, + "time_budget_seconds": 180, + "bounds": { + "max_experts": 4, + "max_mixture_nesting_depth": 2, + "min_experts": 1, + "allow_duplicate_experts": false, + "required_experts": [], + "forbidden_expert_pairs": [], + "allowed_backends": [ + "fac-ctw", + "mixture", + "calibrated" + ], + "forbidden_backends": [], + "parameter_ranges": [ + { + "min": 4, + "parameter": "rate_backend.base_depth", + "max": 24 + } + ] + }, + "max_memory_bytes": 1099511627776, + "output_config_path": "strict-mcaixi-simple-tsv-output.json", + "min_throughput_bytes_per_second": 1, + "seed": 29, + "report_path": "strict-mcaixi-simple-tsv-report.json", + "controller": { + "kind": "mc_aixi_fac_ctw", + "interface": { + "observation_bits": 8, + "observation_stream_len": 1, + "observation_key_mode": "full_stream", + "reward_bits": 32, + "agent_actions": 2 + }, + "planner_simulations_per_step": 4 + }, + "schema_version": 1 +} diff --git a/examples/tuner/strict-smoke-dataset.bin b/examples/tuner/strict-smoke-dataset.bin new file mode 100644 index 00000000..5aac44a3 --- /dev/null +++ b/examples/tuner/strict-smoke-dataset.bin @@ -0,0 +1 @@ +planner family passive integration dataset for strict cgroup-v2 validation bla bla bla rrepeatrepeatrepeatrepeatrepeatrepeatrepeatrepeatrepeatrepeatrepeatrepeatrepeatrepeatrepeatrepeatrepeatrepeatrepeatrepeatrepeatrepeatrepeatrepeatrepeatepeat diff --git a/examples/tuner/strict-smoke-mc-aixi-reward-cert.json b/examples/tuner/strict-smoke-mc-aixi-reward-cert.json new file mode 100644 index 00000000..ed9c8e26 --- /dev/null +++ b/examples/tuner/strict-smoke-mc-aixi-reward-cert.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "kind": "exact_reward_encoding", + "dataset_crc32": "228f4bd5", + "bounds_crc32": "5cd0d183", + "evaluator_profile_crc32": "90c32620", + "controller_kind": "mc_aixi_fac_ctw", + "action_alphabet_size": 2, + "encoding": "integer_objective_difference", + "scalar_representation": "finite-ieee754-f64-nonfinite-forbidden-v1", + "reward_bits": 16, + "max_reward": 65535 +} diff --git a/examples/tuner/strict-smoke-mc-aixi-spec.json b/examples/tuner/strict-smoke-mc-aixi-spec.json new file mode 100644 index 00000000..35b1dd77 --- /dev/null +++ b/examples/tuner/strict-smoke-mc-aixi-spec.json @@ -0,0 +1,58 @@ +{ + "seed": 7, + "report_path": "strict-smoke-mc-aixi-report.json", + "controller": { + "planner_simulations_per_step": 2, + "interface": { + "observation_stream_len": 1, + "observation_key_mode": "full_stream", + "reward_bits": 16, + "agent_actions": 2, + "observation_bits": 8 + }, + "kind": "mc_aixi_fac_ctw" + }, + "output_config_path": "strict-smoke-mc-aixi-output.json", + "assets": [ + { + "path": "strict-smoke-dataset.bin", + "id": "dataset" + } + ], + "kind": "tune", + "schema_version": 1, + "input_asset": "dataset", + "eval_time_limit_seconds": 1, + "time_budget_seconds": 5, + "min_throughput_bytes_per_second": 1, + "max_memory_bytes": 1099511627776, + "bounds": { + "forbidden_backends": [], + "parameter_ranges": [ + { + "min": 1, + "parameter": "rate_backend.base_depth", + "max": 16 + } + ], + "max_experts": 2, + "max_mixture_nesting_depth": 1, + "min_experts": 1, + "allow_duplicate_experts": false, + "required_experts": [], + "forbidden_expert_pairs": [], + "allowed_backends": [ + "fac-ctw" + ] + }, + "baseline_candidate": { + "rate_backend": { + "num_percept_bits": 8, + "encoding_bits": 8, + "kind": "fac-ctw", + "base_depth": 8 + }, + "framing": "framed", + "kind": "rate-ac" + } +} diff --git a/examples/tuner/strict-smoke-spec.json b/examples/tuner/strict-smoke-spec.json new file mode 100644 index 00000000..5d912444 --- /dev/null +++ b/examples/tuner/strict-smoke-spec.json @@ -0,0 +1,51 @@ +{ + "eval_time_limit_seconds": 1, + "controller": { + "max_mutation_radius": 1, + "kind": "annealed_hill_climbing" + }, + "min_throughput_bytes_per_second": 1, + "max_memory_bytes": 1099511627776, + "bounds": { + "required_experts": [], + "forbidden_expert_pairs": [], + "allowed_backends": [ + "fac-ctw" + ], + "forbidden_backends": [], + "parameter_ranges": [ + { + "max": 16, + "parameter": "rate_backend.base_depth", + "min": 1 + } + ], + "max_experts": 2, + "max_mixture_nesting_depth": 1, + "min_experts": 1, + "allow_duplicate_experts": false + }, + "schema_version": 1, + "seed": 7, + "assets": [ + { + "id": "dataset", + "path": "strict-smoke-dataset.bin" + } + ], + "report_path": "strict-smoke-report.json", + "output_config_path": "strict-smoke-output.json", + "time_budget_seconds": 5, + "input_asset": "dataset", + "baseline_candidate": { + "rate_backend": { + "num_percept_bits": 8, + "encoding_bits": 8, + "kind": "fac-ctw", + "base_depth": 8 + }, + "framing": "framed", + "kind": "rate-ac" + }, + "kind": "tune" +} diff --git a/examples/tuner/two-json-annealed-spec.json b/examples/tuner/two-json-annealed-spec.json new file mode 100644 index 00000000..c72baff6 --- /dev/null +++ b/examples/tuner/two-json-annealed-spec.json @@ -0,0 +1,162 @@ +{ + "time_budget_seconds": 1800, + "min_throughput_bytes_per_second": 1, + "max_memory_bytes": 1073741824, + "output_config_path": "/tmp/infotheory-tuner-two-json/annealed-output.json", + "report_path": "/tmp/infotheory-tuner-two-json/annealed-report.json", + "controller": { + "kind": "annealed_hill_climbing", + "max_mutation_radius": 4 + }, + "schema_version": 1, + "seed": 1337, + "assets": [ + { + "path": "../../benchmarks/6f464811/infotheory-two-json-summary-full.tsv", + "id": "dataset" + } + ], + "baseline_candidate": { + "kind": "rate-ac", + "rate_backend": { + "kind": "mixture", + "spec": { + "kind": "neural", + "experts": [ + { + "kind": "fac-ctw", + "num_percept_bits": 8, + "log_prior": 0, + "encoding_bits": 8, + "base_depth": 32, + "name": "fac-ctw" + }, + { + "kind": "ppmd", + "log_prior": 0, + "order": 12, + "memory_mb": 256, + "name": "ppmd" + }, + { + "kind": "rosaplus", + "max_order": -1, + "log_prior": 0, + "name": "rosa" + }, + { + "kind": "match", + "hash_bits": 20, + "log_prior": 0, + "max_len": 255, + "base_mix": 0.02, + "confidence_scale": 1, + "name": "match", + "min_len": 4 + }, + { + "kind": "rwkv7", + "name": "rwkv7", + "log_prior": 0, + "method": { + "kind": "online", + "cfg": { + "g_rank": 16, + "seed": 22, + "train_mode": "adam", + "lr": 0.00089999998454005, + "stride": 1, + "hidden": 64, + "layers": 1, + "intermediate": 64, + "decay_rank": 16, + "a_rank": 16, + "v_rank": 16 + }, + "policy": "schedule=0..0.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|0.5%..1%:infer|1%..1.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|1.5%..2%:infer|2%..2.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|2.5%..3%:infer|3%..3.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|3.5%..4%:infer|4%..4.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|4.5%..5%:infer|5%..5.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|5.5%..6%:infer|6%..6.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|6.5%..7%:infer|7%..7.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|7.5%..8%:infer|8%..8.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|8.5%..9%:infer|9%..9.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|9.5%..12%:train(scope=all,opt=adam,lr=0.0001,stride=1,bptt=8,clip=0,momentum=0.9)|12%..17%:infer|17%..18%:train(scope=all,opt=adam,lr=0.0001,stride=1,bptt=8,clip=0,momentum=0.9)|18%..100%:infer" + } + } + ], + "schedule": "default", + "decay": null, + "alpha": 0.03 + } + }, + "framing": "framed" + }, + "bounds": { + "allow_duplicate_experts": false, + "min_experts": 3, + "max_experts": 8, + "max_mixture_nesting_depth": 3, + "allowed_backends": [ + "fac-ctw", + "ppmd", + "rosaplus", + "match", + "rwkv7", + "mixture" + ], + "forbidden_backends": [], + "required_experts": [ + "fac-ctw", + "ppmd" + ], + "forbidden_expert_pairs": [], + "parameter_ranges": [ + { + "max": 0.2, + "parameter": "rate_backend.spec.alpha", + "min": 0.005 + }, + { + "max": 96, + "parameter": "rate_backend.spec.experts[0].base_depth", + "min": 8 + }, + { + "max": 16, + "parameter": "rate_backend.spec.experts[1].order", + "min": 4 + }, + { + "max": 768, + "parameter": "rate_backend.spec.experts[1].memory_mb", + "min": 64 + }, + { + "max": 128, + "parameter": "rate_backend.spec.experts[2].max_order", + "min": -1 + }, + { + "max": 22, + "parameter": "rate_backend.spec.experts[3].hash_bits", + "min": 16 + }, + { + "max": 16, + "parameter": "rate_backend.spec.experts[3].min_len", + "min": 2 + }, + { + "max": 255, + "parameter": "rate_backend.spec.experts[3].max_len", + "min": 32 + }, + { + "max": 0.1, + "parameter": "rate_backend.spec.experts[3].base_mix", + "min": 0.005 + }, + { + "max": 2, + "parameter": "rate_backend.spec.experts[3].confidence_scale", + "min": 0.5 + } + ] + }, + "kind": "tune", + "input_asset": "dataset", + "eval_time_limit_seconds": 2 +} diff --git a/examples/tuner/two-json-mcaixi-spec.json b/examples/tuner/two-json-mcaixi-spec.json new file mode 100644 index 00000000..cba30492 --- /dev/null +++ b/examples/tuner/two-json-mcaixi-spec.json @@ -0,0 +1,169 @@ +{ + "assets": [ + { + "path": "../../benchmarks/6f464811/infotheory-two-json-summary-full.tsv", + "id": "dataset" + } + ], + "kind": "tune", + "output_config_path": "/tmp/infotheory-tuner-two-json/mcaixi-output.json", + "input_asset": "dataset", + "eval_time_limit_seconds": 2, + "time_budget_seconds": 1800, + "min_throughput_bytes_per_second": 1, + "max_memory_bytes": 1073741824, + "controller": { + "kind": "mc_aixi_fac_ctw", + "planner_simulations_per_step": 24, + "interface": { + "reward_bits": 32, + "agent_actions": 20, + "observation_bits": 8, + "observation_stream_len": 1, + "observation_key_mode": "full_stream" + } + }, + "report_path": "/tmp/infotheory-tuner-two-json/mcaixi-report.json", + "seed": 1337, + "bounds": { + "max_mixture_nesting_depth": 3, + "allowed_backends": [ + "fac-ctw", + "ppmd", + "rosaplus", + "match", + "rwkv7", + "mixture" + ], + "forbidden_backends": [], + "parameter_ranges": [ + { + "parameter": "rate_backend.spec.alpha", + "max": 0.2, + "min": 0.005 + }, + { + "parameter": "rate_backend.spec.experts[0].base_depth", + "max": 96, + "min": 8 + }, + { + "parameter": "rate_backend.spec.experts[1].order", + "max": 16, + "min": 4 + }, + { + "parameter": "rate_backend.spec.experts[1].memory_mb", + "max": 768, + "min": 64 + }, + { + "parameter": "rate_backend.spec.experts[2].max_order", + "max": 128, + "min": -1 + }, + { + "parameter": "rate_backend.spec.experts[3].hash_bits", + "max": 22, + "min": 16 + }, + { + "parameter": "rate_backend.spec.experts[3].min_len", + "max": 16, + "min": 2 + }, + { + "parameter": "rate_backend.spec.experts[3].max_len", + "max": 255, + "min": 32 + }, + { + "parameter": "rate_backend.spec.experts[3].base_mix", + "max": 0.1, + "min": 0.005 + }, + { + "parameter": "rate_backend.spec.experts[3].confidence_scale", + "max": 2, + "min": 0.5 + } + ], + "forbidden_expert_pairs": [], + "required_experts": [ + "fac-ctw", + "ppmd" + ], + "min_experts": 3, + "allow_duplicate_experts": false, + "max_experts": 8 + }, + "schema_version": 1, + "baseline_candidate": { + "kind": "rate-ac", + "framing": "framed", + "rate_backend": { + "spec": { + "kind": "neural", + "schedule": "default", + "alpha": 0.03, + "decay": null, + "experts": [ + { + "log_prior": 0, + "kind": "fac-ctw", + "num_percept_bits": 8, + "encoding_bits": 8, + "base_depth": 32, + "name": "fac-ctw" + }, + { + "log_prior": 0, + "kind": "ppmd", + "order": 12, + "memory_mb": 256, + "name": "ppmd" + }, + { + "log_prior": 0, + "kind": "rosaplus", + "max_order": -1, + "name": "rosa" + }, + { + "log_prior": 0, + "kind": "match", + "base_mix": 0.02, + "confidence_scale": 1, + "max_len": 255, + "min_len": 4, + "name": "match", + "hash_bits": 20 + }, + { + "log_prior": 0, + "kind": "rwkv7", + "method": { + "policy": "schedule=0..0.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|0.5%..1%:infer|1%..1.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|1.5%..2%:infer|2%..2.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|2.5%..3%:infer|3%..3.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|3.5%..4%:infer|4%..4.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|4.5%..5%:infer|5%..5.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|5.5%..6%:infer|6%..6.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|6.5%..7%:infer|7%..7.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|7.5%..8%:infer|8%..8.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|8.5%..9%:infer|9%..9.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|9.5%..12%:train(scope=all,opt=adam,lr=0.0001,stride=1,bptt=8,clip=0,momentum=0.9)|12%..17%:infer|17%..18%:train(scope=all,opt=adam,lr=0.0001,stride=1,bptt=8,clip=0,momentum=0.9)|18%..100%:infer", + "kind": "online", + "cfg": { + "hidden": 64, + "layers": 1, + "intermediate": 64, + "decay_rank": 16, + "a_rank": 16, + "v_rank": 16, + "g_rank": 16, + "seed": 22, + "train_mode": "adam", + "lr": 0.00089999998454005, + "stride": 1 + } + }, + "name": "rwkv7" + } + ] + }, + "kind": "mixture" + } + } +} diff --git a/examples/tuner/two-json-warmstart-spec.json b/examples/tuner/two-json-warmstart-spec.json new file mode 100644 index 00000000..96b8f600 --- /dev/null +++ b/examples/tuner/two-json-warmstart-spec.json @@ -0,0 +1,176 @@ +{ + "input_asset": "dataset", + "baseline_candidate": { + "framing": "framed", + "rate_backend": { + "kind": "mixture", + "spec": { + "schedule": "default", + "kind": "neural", + "alpha": 0.03, + "decay": null, + "experts": [ + { + "name": "fac-ctw", + "kind": "fac-ctw", + "num_percept_bits": 8, + "encoding_bits": 8, + "log_prior": 0, + "base_depth": 32 + }, + { + "name": "ppmd", + "memory_mb": 256, + "kind": "ppmd", + "log_prior": 0, + "order": 12 + }, + { + "name": "rosa", + "max_order": -1, + "kind": "rosaplus", + "log_prior": 0 + }, + { + "name": "match", + "confidence_scale": 1, + "kind": "match", + "base_mix": 0.02, + "min_len": 4, + "hash_bits": 20, + "log_prior": 0, + "max_len": 255 + }, + { + "name": "rwkv7", + "log_prior": 0, + "kind": "rwkv7", + "method": { + "cfg": { + "intermediate": 64, + "decay_rank": 16, + "a_rank": 16, + "v_rank": 16, + "g_rank": 16, + "seed": 22, + "train_mode": "adam", + "lr": 0.00089999998454005, + "stride": 1, + "hidden": 64, + "layers": 1 + }, + "kind": "online", + "policy": "schedule=0..0.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|0.5%..1%:infer|1%..1.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|1.5%..2%:infer|2%..2.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|2.5%..3%:infer|3%..3.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|3.5%..4%:infer|4%..4.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|4.5%..5%:infer|5%..5.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|5.5%..6%:infer|6%..6.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|6.5%..7%:infer|7%..7.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|7.5%..8%:infer|8%..8.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|8.5%..9%:infer|9%..9.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|9.5%..12%:train(scope=all,opt=adam,lr=0.0001,stride=1,bptt=8,clip=0,momentum=0.9)|12%..17%:infer|17%..18%:train(scope=all,opt=adam,lr=0.0001,stride=1,bptt=8,clip=0,momentum=0.9)|18%..100%:infer" + } + } + ] + } + }, + "kind": "rate-ac" + }, + "report_path": "/tmp/infotheory-tuner-two-json/warmstart-report.json", + "seed": 1337, + "eval_time_limit_seconds": 2, + "time_budget_seconds": 1800, + "min_throughput_bytes_per_second": 1, + "max_memory_bytes": 1073741824, + "schema_version": 1, + "output_config_path": "/tmp/infotheory-tuner-two-json/warmstart-output.json", + "assets": [ + { + "path": "../../benchmarks/6f464811/infotheory-two-json-summary-full.tsv", + "id": "dataset" + }, + { + "path": "/tmp/infotheory-tuner-two-json/warmstart-teacher.json", + "id": "teacher" + } + ], + "controller": { + "kind": "aiqi_warmstart_exact_jh", + "planner_simulations_per_step": 24, + "interface": { + "reward_bits": 32, + "agent_actions": 20, + "observation_bits": 8, + "observation_stream_len": 1, + "observation_key_mode": "full_stream" + }, + "label_phase_period": 8, + "warmstart_teacher_dataset_asset": "teacher", + "return_horizon": 4 + }, + "bounds": { + "forbidden_backends": [], + "parameter_ranges": [ + { + "min": 0.005, + "parameter": "rate_backend.spec.alpha", + "max": 0.2 + }, + { + "min": 8, + "parameter": "rate_backend.spec.experts[0].base_depth", + "max": 96 + }, + { + "min": 4, + "parameter": "rate_backend.spec.experts[1].order", + "max": 16 + }, + { + "min": 64, + "parameter": "rate_backend.spec.experts[1].memory_mb", + "max": 768 + }, + { + "min": -1, + "parameter": "rate_backend.spec.experts[2].max_order", + "max": 128 + }, + { + "min": 16, + "parameter": "rate_backend.spec.experts[3].hash_bits", + "max": 22 + }, + { + "min": 2, + "parameter": "rate_backend.spec.experts[3].min_len", + "max": 16 + }, + { + "min": 32, + "parameter": "rate_backend.spec.experts[3].max_len", + "max": 255 + }, + { + "min": 0.005, + "parameter": "rate_backend.spec.experts[3].base_mix", + "max": 0.1 + }, + { + "min": 0.5, + "parameter": "rate_backend.spec.experts[3].confidence_scale", + "max": 2 + } + ], + "min_experts": 3, + "allow_duplicate_experts": false, + "required_experts": [ + "fac-ctw", + "ppmd" + ], + "forbidden_expert_pairs": [], + "max_mixture_nesting_depth": 3, + "max_experts": 8, + "allowed_backends": [ + "fac-ctw", + "ppmd", + "rosaplus", + "match", + "rwkv7", + "mixture" + ] + }, + "kind": "tune" +} diff --git a/examples/tuner/walkthrough_improving_config.md b/examples/tuner/walkthrough_improving_config.md new file mode 100644 index 00000000..2e916adc --- /dev/null +++ b/examples/tuner/walkthrough_improving_config.md @@ -0,0 +1,158 @@ +# Walkthrough: Improving `examples/two.json` with Theorem-Facing Tuner Modes + +This walkthrough is a reproducible, strict (theorem-facing) comparison across: + +1. `annealed_hill_climbing` +2. `mc_aixi_fac_ctw` +3. `aiqi_warmstart_exact_jh` + +All three runs use one shared baseline family anchored to `examples/two.json`, one shared bounds space, one shared dataset, one shared strict evaluator profile, and one shared deployability envelope. + +Before copy-pasting commands below, set the repository root once: + +```bash +export INFOTHEORY_REPO=/path/to/infotheory +``` + +## What this compares fairly + +- Same baseline candidate family for all three modes: `rate-ac` + neural mixture derived from `examples/two.json`. +- Same search space (`bounds`) for all three modes. +- Same per-eval deadline and memory envelope. +- Same strict memory mode for all three modes: + - `--rss-mode hybrid_strict_max` + - delegated cgroup-v2 parent via `--evaluator-cgroup-parent` +- Same evaluation cap and time budget. + + +## Included example specs + +These three committed specs demonstrate the shared design: + +- [two-json-annealed-spec.json](two-json-annealed-spec.json) +- [two-json-mcaixi-spec.json](two-json-mcaixi-spec.json) +- [two-json-warmstart-spec.json](two-json-warmstart-spec.json) + +They use this in-tree dataset by default: + +- `$INFOTHEORY_REPO/benchmarks/6f464811/infotheory-two-json-summary-full.tsv` +- Reason: it is in-tree, stable for audit, and size-appropriate for tuner comparisons. + +## Reproducible strict benchmark script + +Use: + +- [benchmark_tuner_two_json_modes.sh](../../scripts/benchmark_tuner_two_json_modes.sh) + +Behavior: + +- Anchors from `examples/two.json` and canonicalizes it into tune-acceptable baseline JSON. +- Runs annealed, MC-AIXI, then warmstart sequentially. +- Emits exact reward certificates under the strict evaluator profile. +- Bootstraps warmstart teacher fingerprint via explicit probe/patch flow. +- Writes machine-readable comparison artifacts. + +Reversibility note (important for audit): + +- This benchmark *does* include `rate_backend.spec.experts[2].max_order` in the search space. +- A prior reversibility failure on this dimension was traced to mutation-kind instability at the + `-1 <-> 0` boundary (JSON numeric typing flipped from signed to unsigned by value). +- The kernel now preserves signed integer mutation semantics whenever the configured range has a + negative lower bound, so `max_order` tuning remains enabled without violating reversible-kernel checks. + +Defaults: + +- Subject data: `git show HEAD:README.md` written into run directory. +- Per-eval limit: `2` seconds. +- Memory cap: `1` GiB. +- Max evaluations per mode: `120`. +- Time budget per mode: `600` seconds. + +## Strict setup and run + +One-time delegated cgroup setup: + +```bash +cd "$INFOTHEORY_REPO" +sudo ./scripts/delegate_tuner_cgroup_v2.sh setup theo infotheory-tuner +``` + +Run full strict comparison (default subject = `README.md` at HEAD): + +```bash +cd "$INFOTHEORY_REPO" +sudo --preserve-env=PATH ./scripts/delegate_tuner_cgroup_v2.sh run-in-session theo infotheory-tuner -- \ + env INFOTHEORY_TUNER_EVAL_CGROUP_PARENT=/sys/fs/cgroup/infotheory-tuner/evals \ + ./scripts/benchmark_tuner_two_json_modes.sh +``` + +Run with explicit subject file + limits: + +```bash +cd "$INFOTHEORY_REPO" +sudo --preserve-env=PATH ./scripts/delegate_tuner_cgroup_v2.sh run-in-session theo infotheory-tuner -- \ + env INFOTHEORY_TUNER_EVAL_CGROUP_PARENT=/sys/fs/cgroup/infotheory-tuner/evals \ + ./scripts/benchmark_tuner_two_json_modes.sh /path/to/input.bin 2 1 +``` + +Optional 30-minute envelope (example): + +```bash +cd "$INFOTHEORY_REPO" +sudo --preserve-env=PATH ./scripts/delegate_tuner_cgroup_v2.sh run-in-session theo infotheory-tuner -- \ + env INFOTHEORY_TUNER_EVAL_CGROUP_PARENT=/sys/fs/cgroup/infotheory-tuner/evals \ + TUNER_MAX_EVALUATIONS=180 TUNER_TIME_BUDGET_SECONDS=1800 \ + ./scripts/benchmark_tuner_two_json_modes.sh /path/to/input.bin 2 1 +``` + +## Output artifacts + +Per run, script prints `run_dir` under: + +- `/tmp/infotheory-tuner-two-json/run-YYYYMMDD-HHMMSS` + +Key outputs: + +- `comparison-summary.json` +- `comparison-summary.tsv` +- `benchmark.log` +- `annealed-report.json` +- `mcaixi-report.json` +- `warmstart-report.json` +- strict exact reward cert files for MC-AIXI/warmstart + +## What to return for verification + +After running, return: + +1. `run_dir` +2. `comparison-summary.tsv` +3. `comparison-summary.json` +4. last ~120 lines of `benchmark.log` +5. confirmation of theorem-facing strict markers in reports: + - `provenance.executor_controls.rss_mode.strict_theorem_memory_certified: true` + - strict memory provenance (`cgroup_v2_peak` / strict hybrid path) + +Copy-paste checks: + +```bash +RUN_DIR="/tmp/infotheory-tuner-two-json/run-YYYYMMDD-HHMMSS" +ls -l "$RUN_DIR"/{comparison-summary.tsv,comparison-summary.json,annealed-report.json,mcaixi-report.json,warmstart-report.json} +cat "$RUN_DIR/comparison-summary.tsv" +python3 - <<'PY' "$RUN_DIR" +import json, pathlib, sys +run = pathlib.Path(sys.argv[1]) +for name in ["annealed-report.json", "mcaixi-report.json", "warmstart-report.json"]: + doc = json.loads((run / name).read_text()) + rss = ( + doc.get("provenance", {}) + .get("executor_controls", {}) + .get("rss_mode", {}) + ) + print(name) + print(" status:", doc.get("status")) + print(" objective_bits:", doc.get("best", {}).get("objective_bits")) + print(" strict_theorem_memory_certified:", rss.get("strict_theorem_memory_certified")) +PY +tail -n 120 "$RUN_DIR/benchmark.log" +``` diff --git a/examples/two.json b/examples/two.json index eb6d6891..56aeb5ed 100644 --- a/examples/two.json +++ b/examples/two.json @@ -1,37 +1,39 @@ { "kind": "neural", - "alpha": 0.1, "experts": [ { - "name": "ctw", - "kind": "ctw", - "depth": 32, - "log_prior": 0.0 + "encoding_bits": 8, + "name": "fac-ctw", + "kind": "fac-ctw", + "num_percept_bits": 8, + "msb_first": true, + "log_prior": 0, + "base_depth": 32 }, { - "name": "ppmd", - "kind": "ppmd", "order": 12, "memory_mb": 256, - "log_prior": 0.0 + "name": "ppmd", + "kind": "ppmd", + "log_prior": 0 }, { - "name": "rosa", "kind": "rosaplus", - "max_order": -1, - "log_prior": 0.0 + "log_prior": 0, + "name": "rosa", + "max_order": -1 }, { - "name": "match", "kind": "match", - "log_prior": 0.0 + "name": "match", + "log_prior": 0 }, { - "name": "rwkv", - "kind": "rwkv", - "method": "cfg:hidden=64,layers=1,intermediate=64,decay_rank=16,a_rank=16,v_rank=16,g_rank=16,seed=22,train=adam,lr=0.0009,stride=1;policy:schedule=0..0.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|0.5%..1%:infer|1%..1.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|1.5%..2%:infer|2%..2.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|2.5%..3%:infer|3%..3.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|3.5%..4%:infer|4%..4.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|4.5%..5%:infer|5%..5.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|5.5%..6%:infer|6%..6.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|6.5%..7%:infer|7%..7.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|7.5%..8%:infer|8%..8.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|8.5%..9%:infer|9%..9.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|9.5%..12%:train(scope=all,opt=adam,lr=0.0001,stride=1,bptt=8,clip=0,momentum=0.9)|12%..17%:infer|17%..18%:train(scope=all,opt=adam,lr=0.0001,stride=1,bptt=8,clip=0,momentum=0.9)|18%..100%:infer", - "log_prior": 0.0 + "kind": "rwkv7", + "log_prior": 0, + "name": "rwkv7", + "method": "cfg:hidden=64,layers=1,intermediate=64,decay_rank=16,a_rank=16,v_rank=16,g_rank=16,seed=22,train=adam,lr=0.0009,stride=1;policy:schedule=0..0.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|0.5%..1%:infer|1%..1.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|1.5%..2%:infer|2%..2.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|2.5%..3%:infer|3%..3.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|3.5%..4%:infer|4%..4.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|4.5%..5%:infer|5%..5.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|5.5%..6%:infer|6%..6.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|6.5%..7%:infer|7%..7.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|7.5%..8%:infer|8%..8.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|8.5%..9%:infer|9%..9.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|9.5%..12%:train(scope=all,opt=adam,lr=0.0001,stride=1,bptt=8,clip=0,momentum=0.9)|12%..17%:infer|17%..18%:train(scope=all,opt=adam,lr=0.0001,stride=1,bptt=8,clip=0,momentum=0.9)|18%..100%:infer" } - - ] + ], + "alpha": 0.03 } diff --git a/examples/two_heavy.json b/examples/two_heavy.json index 48e3ca18..da51b687 100644 --- a/examples/two_heavy.json +++ b/examples/two_heavy.json @@ -1,79 +1,83 @@ { - "kind": "neural", - "alpha": 0.03, "experts": [ { - "name": "rwkv", - "kind": "rwkv", - "method": "cfg:hidden=64,layers=1,intermediate=64,decay_rank=16,a_rank=16,v_rank=16,g_rank=16,seed=22,train=adam,lr=0.0009,stride=1;policy:schedule=0..1%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=1,clip=0,momentum=0.9)|1%..5%:train(scope=head,opt=adam,lr=0.001,stride=1,bptt=1,clip=0,momentum=0.9)|5%..7%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=1,clip=0,momentum=0.9)", - "log_prior": 0.0 + "name": "rwkv7", + "log_prior": 0, + "kind": "rwkv7", + "method": "cfg:hidden=64,layers=1,intermediate=64,decay_rank=16,a_rank=16,v_rank=16,g_rank=16,seed=22,train=adam,lr=0.0009,stride=1;policy:schedule=0..1%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=1,clip=0,momentum=0.9)|1%..5%:train(scope=head,opt=adam,lr=0.001,stride=1,bptt=1,clip=0,momentum=0.9)|5%..7%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=1,clip=0,momentum=0.9)" }, { "name": "classical", + "log_prior": 0, "kind": "mixture", - "log_prior": 0.0, "spec": { - "kind": "neural", - "alpha": 0.03, "experts": [ { "name": "rosa", + "log_prior": 0, "kind": "rosaplus", - "max_order": -1, - "log_prior": 0.0 + "max_order": -1 }, { - "name": "ctw", - "kind": "ctw", - "depth": 24, - "log_prior": 0.0 + "encoding_bits": 8, + "log_prior": 0, + "base_depth": 24, + "name": "fac-ctw", + "kind": "fac-ctw", + "num_percept_bits": 8 }, { + "log_prior": 0, "name": "ppmd", - "kind": "ppmd", - "order": 10, "memory_mb": 64, - "log_prior": 0.0 + "kind": "ppmd", + "order": 10 }, { "name": "match", "kind": "match", - "log_prior": 0.0 + "log_prior": 0 }, { "name": "sparse", "kind": "sparse-match", - "log_prior": 0.0 + "log_prior": 0 }, { - "name": "cal-ctw", - "kind": "calibrated", - "context": "text", "bins": 33, + "log_prior": 0, + "context": "text", + "bias_clip": 4, + "name": "cal-ctw", "learning_rate": 0.02, - "bias_clip": 4.0, - "log_prior": 0.0, + "kind": "calibrated", "base": { - "kind": "ctw", - "depth": 24 + "encoding_bits": 8, + "kind": "fac-ctw", + "base_depth": 24, + "num_percept_bits": 8 } }, { - "name": "cal-ppmd", - "kind": "calibrated", - "context": "text-repeat", "bins": 33, + "log_prior": 0, + "context": "text-repeat", + "bias_clip": 4, + "name": "cal-ppmd", "learning_rate": 0.02, - "bias_clip": 4.0, - "log_prior": 0.0, + "kind": "calibrated", "base": { - "kind": "ppmd", "order": 10, - "memory_mb": 64 + "memory_mb": 64, + "kind": "ppmd" } } - ] + ], + "kind": "neural", + "alpha": 0.03 } } - ] + ], + "kind": "neural", + "alpha": 0.03 } diff --git a/examples/two_pseq.json b/examples/two_pseq.json index fc91606a..79bd0d80 100644 --- a/examples/two_pseq.json +++ b/examples/two_pseq.json @@ -1,43 +1,44 @@ { - "kind": "neural", - "alpha": 0.1, "experts": [ { - "name": "ctw", - "kind": "ctw", - "depth": 32, - "log_prior": 0.0 + "name": "fac-ctw", + "base_depth": 32, + "num_percept_bits": 8, + "encoding_bits": 8, + "log_prior": 0, + "kind": "fac-ctw" }, { "name": "ppmd", - "kind": "ppmd", - "order": 12, "memory_mb": 256, - "log_prior": 0.0 + "log_prior": 0, + "kind": "ppmd", + "order": 12 }, { "name": "sequitur", + "log_prior": 0, "kind": "sequitur", - "context_bytes": 24, - "log_prior": 0.0 + "context_bytes": 24 }, { "name": "rosa", - "kind": "rosaplus", "max_order": -1, - "log_prior": 0.0 + "kind": "rosaplus", + "log_prior": 0 }, { "name": "match", - "kind": "match", - "log_prior": 0.0 + "log_prior": 0, + "kind": "match" }, { - "name": "rwkv", - "kind": "rwkv", - "method": "cfg:hidden=64,layers=1,intermediate=64,decay_rank=16,a_rank=16,v_rank=16,g_rank=16,seed=22,train=adam,lr=0.0009,stride=1;policy:schedule=0..0.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|0.5%..1%:infer|1%..1.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|1.5%..2%:infer|2%..2.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|2.5%..3%:infer|3%..3.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|3.5%..4%:infer|4%..4.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|4.5%..5%:infer|5%..5.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|5.5%..6%:infer|6%..6.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|6.5%..7%:infer|7%..7.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|7.5%..8%:infer|8%..8.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|8.5%..9%:infer|9%..9.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|9.5%..12%:train(scope=all,opt=adam,lr=0.0001,stride=1,bptt=8,clip=0,momentum=0.9)|12%..17%:infer|17%..18%:train(scope=all,opt=adam,lr=0.0001,stride=1,bptt=8,clip=0,momentum=0.9)|18%..100%:infer", - "log_prior": 0.0 + "name": "rwkv7", + "log_prior": 0, + "kind": "rwkv7", + "method": "cfg:hidden=64,layers=1,intermediate=64,decay_rank=16,a_rank=16,v_rank=16,g_rank=16,seed=22,train=adam,lr=0.0009,stride=1;policy:schedule=0..0.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|0.5%..1%:infer|1%..1.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|1.5%..2%:infer|2%..2.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|2.5%..3%:infer|3%..3.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|3.5%..4%:infer|4%..4.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|4.5%..5%:infer|5%..5.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|5.5%..6%:infer|6%..6.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|6.5%..7%:infer|7%..7.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|7.5%..8%:infer|8%..8.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|8.5%..9%:infer|9%..9.5%:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=8,clip=0,momentum=0.9)|9.5%..12%:train(scope=all,opt=adam,lr=0.0001,stride=1,bptt=8,clip=0,momentum=0.9)|12%..17%:infer|17%..18%:train(scope=all,opt=adam,lr=0.0001,stride=1,bptt=8,clip=0,momentum=0.9)|18%..100%:infer" } - ] + ], + "kind": "neural", + "alpha": 0.1 } - diff --git a/infotheory_py/Cargo.toml b/infotheory_py/Cargo.toml deleted file mode 100644 index b46294ea..00000000 --- a/infotheory_py/Cargo.toml +++ /dev/null @@ -1,32 +0,0 @@ -[package] -name = "infotheory_py" -version = "1.1.1" -edition = "2024" -license = "ISC OR Apache-2.0" -description = "PyO3 bindings for infotheory." -homepage = "https://infotheory.tech" -repository = "https://github.com/turtle261/infotheory" -publish = false - -[lib] -name = "_core" -crate-type = ["cdylib"] - -[dependencies] -anyhow = "1.0.100" -infotheory = { path = "..", default-features = false } -pyo3 = { version = "0.28.2", features = ["abi3-py310"] } -rayon = "1.11.0" -serde_json = "1.0.149" - -[features] -default = ["backend-rosa", "backend-mamba", "backend-rwkv", "backend-zpaq"] -python-extension = ["pyo3/extension-module"] -backend-rosa = ["infotheory/backend-rosa"] -backend-mamba = ["infotheory/backend-mamba"] -backend-rwkv = ["infotheory/backend-rwkv"] -backend-zpaq = ["infotheory/backend-zpaq"] -vm = ["infotheory/vm"] - -[lints] -workspace = true diff --git a/ite-bench/RESULTS.md b/ite-bench/RESULTS.md deleted file mode 100644 index 9d3cc4b4..00000000 --- a/ite-bench/RESULTS.md +++ /dev/null @@ -1,108 +0,0 @@ -# ITE Benchmark: Validation Results - -## Executive Summary - -This benchmark framework validates information-theoretic estimators against oracle truths and required mathematical properties. - -## Test Results - -### Rust Estimator (infotheory) -This suite now runs a **strict, broad validation matrix** across multiple regimes, oracles, and quantities. -Failures are expected under strict tolerances and are treated as signals for estimator or model improvements. - -Covered quantities (non-exhaustive): -- Shannon Entropy H(X) -- Mutual Information I(X;Y) -- Conditional Entropy H(X|Y) -- Joint Entropy H(X,Y) -- KL Divergence D_KL -- JS Divergence D_JS -- Cross-Entropy H(P,Q) -- Total Variation Distance (TVD) -- Normalized Entropy Distance (NED) -- Normalized Transform Effort (NTE) -- Entropy Rate H_rate -- NCD metric axioms (Vitányi) - -### External estimator dependencies -None. This benchmark is intentionally self-contained. - -## Key Findings (Current Strict Run) - -### 1. Estimator Methodology Differences - -| Aspect | infotheory | -|--------|-----------| -| Method | Compression/rate based (depending on primitive/backend) | -| Data Type | Designed for arbitrary byte sequences | -| Accuracy | Validated against oracle truths + mathematical properties | - -### 2. Discrete Data Compatibility - -**Test Setup**: Discrete uniform distribution on {0,1,2,3,4,5,6,7} - -- Validation focuses on oracle accuracy and bound/inequality checks for discrete data. - -### 3. Information-Theoretic Properties Verified - -✓ **Non-negativity**: I(X;Y) ≥ 0, D_KL ≥ 0, H(X|Y) ≥ 0 -✓ **Divergence bounds**: 0 ≤ D_JS ≤ log(2) -✓ **Oracle accuracy**: Estimates within strict tolerance (may fail; intended) -✓ **Subadditivity / data processing**: H(X,Y) ≤ H(X)+H(Y), I(X;Z) ≤ I(X;Y) -✓ **Metric axioms**: NCD approximate non-negativity, identity, symmetry, triangle - -## Framework Architecture - -``` -├── src/ITE/ -│ ├── Types.lean # Core types: Quantity, Estimator, SampleBundle -│ ├── Oracles.lean # Ground truth generators -│ ├── Estimators.lean # Rust estimator adapter (infotheory CLI) -│ ├── Verification.lean # Accuracy & inequality verification -│ └── Reporting.lean # Result structures -├── src/Runner.lean # Comprehensive validation runner -└── infotheory/ # Rust CLI (cargo project) - -``` - -## Usage - -Build and run: -```bash -lake build runner -./.lake/build/bin/runner -``` - -## Recommendations - -### For Production Use -- **Discrete/categorical data**: Rust infotheory (marginal measures) -- **Research/experimentation**: Use strict suite to identify estimator weaknesses - -## Future Work - -1. Extend rate-backend coverage to RWKV and fac-CTW -2. Increase regime diversity (mixtures, heavy tails, higher dimensionality) -3. Tighten formal identities (NED equivalences, NTE bounds) with larger samples - -## Technical Implementation - -### Rust Adapter -- Calls `../target/release/infotheory` (relative to `ite-bench/`) -- Writes samples to temporary binary files -- Parses float output from stdout -- Handles all ITE quantities via CLI primitives - -### Validation Logic -- Validates estimates against oracle truths -- Verifies core inequalities and structural identities -- Provides detailed error messages for failures - -## Conclusion - -The benchmark successfully demonstrates: -1. ✅ infotheory works reliably for discrete data -2. ✅ Clean validation against mathematical properties -3. ✅ Comprehensive error handling and reporting - -This provides a solid foundation for information-theoretic estimation research and validates the Rust implementation as robust for discrete/categorical data analysis. diff --git a/ite-bench/src/ITE/Estimators.lean b/ite-bench/src/ITE/Estimators.lean index 66b6f4ef..c4647260 100644 --- a/ite-bench/src/ITE/Estimators.lean +++ b/ite-bench/src/ITE/Estimators.lean @@ -128,38 +128,31 @@ def infotheoryEstimator (binPath : FilePath := FilePath.mk "../target/release/in let params := _params let paramString (k : String) : Option String := params.strings[k]? - let maxOrderStr : Option String := - paramString "max_order" let rateBackendStr := paramString "rate_backend" - let ncdBackendStr := paramString "ncd_backend" + let compressionBackendStr := paramString "compression_backend" let methodStr := paramString "method" let withCommonFlags (args : Array String) : Array String := let args := match rateBackendStr with | some rb => args.push "--rate-backend" |>.push rb | none => args - let args := match ncdBackendStr with - | some nb => args.push "--ncd-backend" |>.push nb + let args := match compressionBackendStr with + | some nb => args.push "--compression-backend" |>.push nb | none => args let args := match methodStr with | some m => args.push "--method" |>.push m | none => args args - let withMaxOrder (args : Array String) : Array String := - match maxOrderStr with - | some mo => args.push mo - | none => args - let runUnary (primName : String) (path : FilePath) : IO (Except String Float) := do - let args := withCommonFlags <| withMaxOrder #[primName, path.toString] + let args := withCommonFlags #[primName, path.toString] let out ← IO.Process.output { cmd := binPath.toString, args := args } if out.exitCode ≠ 0 then return .error s!"infotheory call failed: {out.stderr}" return parseFloatSimple out.stdout let runBinary (primName : String) (p1 p2 : FilePath) : IO (Except String Float) := do - let args := withCommonFlags <| withMaxOrder #[primName, p1.toString, p2.toString] + let args := withCommonFlags #[primName, p1.toString, p2.toString] let out ← IO.Process.output { cmd := binPath.toString, args := args } if out.exitCode ≠ 0 then return .error s!"infotheory call failed: {out.stderr}" diff --git a/ite-bench/src/Runner.lean b/ite-bench/src/Runner.lean index 254ac8b1..2c072470 100644 --- a/ite-bench/src/Runner.lean +++ b/ite-bench/src/Runner.lean @@ -107,26 +107,44 @@ private def oracleGenFromOutcome (key : String) (outcome : OracleOutcome) : IO ( return (outcome.bundle, v) private def mkParams - (maxOrder : Option String := none) (rateBackend : Option String := none) - (ncdBackend : Option String := none) + (compressionBackend : Option String := none) (method : Option String := none) : EstimatorParams := Id.run do let mut strings := HashMap.empty - match maxOrder with - | some v => strings := strings.insert "max_order" v - | none => pure () match rateBackend with | some v => strings := strings.insert "rate_backend" v | none => pure () - match ncdBackend with - | some v => strings := strings.insert "ncd_backend" v + match compressionBackend with + | some v => strings := strings.insert "compression_backend" v | none => pure () match method with | some v => strings := strings.insert "method" v | none => pure () return { scalars := HashMap.empty, strings := strings } +private def checkByteCtwEntropyRateContract + (est : Estimator) + (label : String) + (bundle : SampleBundle) + (sourceRate : Float) + (tolRate : Float) + (paramsRate : EstimatorParams) + (paramsRateCtw : EstimatorParams) : IO Bool := do + let iidRate ← runEstimateIO est .shannonEntropy bundle paramsRate + let ctwRate ← runEstimateIO est .entropyRate bundle paramsRateCtw + let gainFraction := 0.25 + let iidGap := iidRate - sourceRate + let requiredMax := iidRate - gainFraction * iidGap + let lowerOk := ctwRate + tolRate >= sourceRate + let upperOk := ctwRate <= requiredMax + IO.println s!"[CONTRACT] {label} H_rate (CTW byte-stream) source={sourceRate} iid={iidRate} ctw={ctwRate} lowerSlack={tolRate} requiredMax={requiredMax} gainFraction={gainFraction}" + if !lowerOk then + IO.println s!"[FAIL] {label} CTW byte-stream entropy rate fell below source entropy beyond tolerance: ctw={ctwRate}, source={sourceRate}, tol={tolRate}" + if !upperOk then + IO.println s!"[FAIL] {label} CTW byte-stream entropy rate did not materially beat IID baseline: ctw={ctwRate}, requiredMax={requiredMax}, iid={iidRate}" + pure <| lowerOk && upperOk + private def runSuite : IO Bool := do let est := infotheoryEstimator let rustBin := FilePath.mk "../target/release/infotheory" @@ -194,7 +212,7 @@ private def runSuite : IO Bool := do let (bundleH, truthHX) ← oracleGenFromOutcome "H_X" outcomeInd let (bundleMI, truthMI) ← oracleGenFromOutcome "I_XY" outcomeInd - let paramsMarg := mkParams (some "0") + let paramsMarg := mkParams let repHX ← verifyAccuracyWith est (fun _ => pure (bundleH, truthHX)) .shannonEntropy r paramsMarg 30 let repMI ← verifyAccuracyWith est (fun _ => pure (bundleMI, truthMI)) .mutualInformation r paramsMarg 30 @@ -326,7 +344,7 @@ private def runSuite : IO Bool := do -- Entropy rate: binary Markov chain let outcomeMarkov ← (binaryMarkovOracle 0.9 0.8).generate r 60000 let (bundleRate, truthRate) ← oracleGenFromOutcome "H_RATE" outcomeMarkov - let paramsRate := mkParams (some "-1") + let paramsRate := mkParams let repRate ← verifyAccuracyWith est (fun _ => pure (bundleRate, truthRate)) .entropyRate r paramsRate 20 let tolRate := ToleranceDefaults.defaults.quantity .entropyRate r IO.println s!"[ACCURACY] Markov H_rate MAE={repRate.mae} maxAbs={repRate.maxAbsError} (tol={tolRate}, strictScale={strictScale}, allowed={strictScale*tolRate})" @@ -335,12 +353,13 @@ private def runSuite : IO Bool := do IO.println "[FAIL] Entropy rate exceeded tolerance" -- Entropy rate with CTW backend and explicit depth - let paramsRateCtw := mkParams (some "-1") (some "ctw") none (some "16") - let repRateCtw ← verifyAccuracyWith est (fun _ => pure (bundleRate, truthRate)) .entropyRate r paramsRateCtw 10 - IO.println s!"[ACCURACY] Markov H_rate (CTW) MAE={repRateCtw.mae} maxAbs={repRateCtw.maxAbsError} (tol={tolRate}, strictScale={strictScale}, allowed={strictScale*tolRate})" - if repRateCtw.maxAbsError > strictScale * tolRate then + let paramsRateCtw := mkParams (some "ctw") none (some "16") + -- Direct CTW is byte-stream CTW over MSB-expanded bytes. On binary 0/1 + -- symbol data, validate its finite-sample byte contract rather than reuse + -- the direct estimator's tight equality-to-source-entropy gate. + let ctwRateOk ← checkByteCtwEntropyRateContract est "Markov" bundleRate truthRate tolRate paramsRate paramsRateCtw + if !ctwRateOk then ok := false - IO.println "[FAIL] Entropy rate (CTW) exceeded tolerance" -- Entropy rate: binary Markov chain (order 2) let outcomeMarkov2 ← (binaryMarkov2Oracle 0.1 0.7 0.4 0.9).generate r 60000 @@ -351,11 +370,9 @@ private def runSuite : IO Bool := do ok := false IO.println "[FAIL] Entropy rate (Markov2) exceeded tolerance" - let repRate2Ctw ← verifyAccuracyWith est (fun _ => pure (bundleRate2, truthRate2)) .entropyRate r paramsRateCtw 10 - IO.println s!"[ACCURACY] Markov2 H_rate (CTW) MAE={repRate2Ctw.mae} maxAbs={repRate2Ctw.maxAbsError} (tol={tolRate}, strictScale={strictScale}, allowed={strictScale*tolRate})" - if repRate2Ctw.maxAbsError > strictScale * tolRate then + let ctwRate2Ok ← checkByteCtwEntropyRateContract est "Markov2" bundleRate2 truthRate2 tolRate paramsRate paramsRateCtw + if !ctwRate2Ok then ok := false - IO.println "[FAIL] Entropy rate (Markov2, CTW) exceeded tolerance" -- ZPAQ rate backend sanity: copy-like data should compress well let pattern ← randBytes 64 @@ -365,7 +382,7 @@ private def runSuite : IO Bool := do for i in [:pattern.size] do copyData := copyData.push (pattern.get! i) let copyBundle : SampleBundle := { bytesX := some copyData } - let paramsZpaq := mkParams (some "-1") (some "zpaq") none (some "2") + let paramsZpaq := mkParams (some "zpaq") none (some "2") let zpaqRate ← runEstimateIO est .entropyRate copyBundle paramsZpaq IO.println s!"[ACCURACY] ZPAQ H_rate on copy-like data = {zpaqRate}" if zpaqRate > 0.3 then diff --git a/nyx-lite b/nyx-lite deleted file mode 160000 index 399915fe..00000000 --- a/nyx-lite +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 399915fe5d304625614f973014967f130455589d diff --git a/projman.sh b/projman.sh index b143fa68..210561e8 100755 --- a/projman.sh +++ b/projman.sh @@ -6,6 +6,36 @@ ROOT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) say() { printf '%s\n' "$*"; } fail() { printf 'ERROR: %s\n' "$*" >&2; exit 1; } +build_mode() { + printf '%s' "${INFOTHEORY_BUILD_MODE:-native}" +} + +validate_build_mode() { + case "$(build_mode)" in + native|portable) ;; + *) fail "INFOTHEORY_BUILD_MODE must be one of: native, portable" ;; + esac +} + +portable_rustflags() { + case "$(uname -s)" in + Linux|FreeBSD|OpenBSD) printf '%s' "-C target-cpu=generic -C link-arg=-fuse-ld=lld" ;; + *) printf '%s' "-C target-cpu=generic" ;; + esac +} + +run_cargo_mode() { + validate_build_mode + mode=$(build_mode) + if [ "$mode" = "portable" ]; then + CARGO_BUILD_RUSTFLAGS="$(portable_rustflags)" \ + RUSTDOCFLAGS="${RUSTDOCFLAGS:-$(portable_rustflags)}" \ + cargo "$@" + else + cargo "$@" + fi +} + need_cmd() { command -v "$1" >/dev/null 2>&1 || fail "Missing required command: $1" } @@ -15,7 +45,15 @@ has_kvm() { } vm_artifacts_present() { - [ -f "$ROOT_DIR/vmlinux-6.1.58" ] && [ -f "$ROOT_DIR/nyx-lite/vm_image/dockerimage/rootfs.ext4" ] && [ -f "$ROOT_DIR/nyx-lite/guest/aixi_initramfs.cpio" ] + [ -f "$ROOT_DIR/vmlinux-6.1.58" ] && [ -f "$ROOT_DIR/vendor/nyx-lite/vm_image/dockerimage/rootfs.ext4" ] && [ -f "$ROOT_DIR/vendor/nyx-lite/guest/aixi_initramfs.cpio" ] +} + +cmd_check_nyx_lite_standalone() { + say "[check-vm-builder] Checking standalone nyx-lite build_rootfs compile..." + need_cmd cargo + (cd "$ROOT_DIR" && \ + cargo check -q --manifest-path "$ROOT_DIR/vendor/nyx-lite/Cargo.toml" --bin build_rootfs) + say "[check-vm-builder] Done" } cmd_init_vm() { @@ -25,7 +63,7 @@ cmd_init_vm() { # Kernel (cached) if [ ! -f "$ROOT_DIR/vmlinux-6.1.58" ]; then if command -v wget >/dev/null 2>&1; then - (cd "$ROOT_DIR" && sh "$ROOT_DIR/nyx-lite/vm_image/download_kernel.sh") + (cd "$ROOT_DIR" && sh "$ROOT_DIR/vendor/nyx-lite/vm_image/download_kernel.sh") elif command -v curl >/dev/null 2>&1; then (cd "$ROOT_DIR" && curl -L -o vmlinux-6.1.58 'https://s3.amazonaws.com/spec.ccfc.min/firecracker-ci/v1.6/x86_64/vmlinux-6.1.58') else @@ -38,12 +76,12 @@ cmd_init_vm() { # Minimal initramfs (cpio) for nyx-lite guest need_cmd cc need_cmd cpio - say "[init-vm] Building nyx-lite/guest/aixi_initramfs.cpio" + say "[init-vm] Building vendor/nyx-lite/guest/aixi_initramfs.cpio" (cd "$ROOT_DIR" && \ - cc -O2 -static -s nyx-lite/guest/aixi_guest.c -o nyx-lite/guest/aixi_guest && \ - mkdir -p nyx-lite/guest/initramfs && \ - cp -f nyx-lite/guest/aixi_guest nyx-lite/guest/initramfs/init && \ - (cd nyx-lite/guest/initramfs && find . -print | cpio -o -H newc > ../aixi_initramfs.cpio)) + cc -O2 -static -s vendor/nyx-lite/guest/aixi_guest.c -o vendor/nyx-lite/guest/aixi_guest && \ + mkdir -p vendor/nyx-lite/guest/initramfs && \ + cp -f vendor/nyx-lite/guest/aixi_guest vendor/nyx-lite/guest/initramfs/init && \ + (cd vendor/nyx-lite/guest/initramfs && find . -print | cpio -o -H newc > ../aixi_initramfs.cpio)) # Docker rootfs build (ext4) if [ "${SKIP_DOCKER:-}" = "1" ]; then @@ -56,12 +94,12 @@ cmd_init_vm() { need_cmd docker need_cmd tar need_cmd mke2fs - say "[init-vm] Building nyx-lite/vm_image/dockerimage/rootfs.ext4 via RootfsBuilder (no sudo)" + say "[init-vm] Building vendor/nyx-lite/vm_image/dockerimage/rootfs.ext4 via RootfsBuilder (no sudo)" (cd "$ROOT_DIR" && \ - cargo run -q -p nyx-lite --bin build_rootfs -- \ - "$ROOT_DIR/nyx-lite/vm_image/dockerimage/Dockerfile" \ - "$ROOT_DIR/nyx-lite/vm_image/dockerimage" \ - "$ROOT_DIR/nyx-lite/vm_image/dockerimage/rootfs.ext4" \ + cargo run -q --manifest-path "$ROOT_DIR/vendor/nyx-lite/Cargo.toml" --bin build_rootfs -- \ + "$ROOT_DIR/vendor/nyx-lite/vm_image/dockerimage/Dockerfile" \ + "$ROOT_DIR/vendor/nyx-lite/vm_image/dockerimage" \ + "$ROOT_DIR/vendor/nyx-lite/vm_image/dockerimage/rootfs.ext4" \ --size-mib 512 \ --work-dir "$ROOT_DIR/target/tmp/rootfs_work") @@ -71,11 +109,13 @@ cmd_init_vm() { cmd_code_test() { say "[code_test] Building + testing Rust (release)..." need_cmd cargo + validate_build_mode + say "[code_test] Build mode: $(build_mode)" - (cd "$ROOT_DIR" && cargo build --release) + (cd "$ROOT_DIR" && run_cargo_mode build --release) if [ "${BUILD_CLI:-0}" = "1" ]; then say "[code_test] BUILD_CLI=1 set; checking optional CLI binary" - (cd "$ROOT_DIR" && cargo build --release --features cli) + (cd "$ROOT_DIR" && run_cargo_mode build --release --features cli) fi # If docker is available, enable the nyx-lite rootfs builder test. @@ -87,9 +127,9 @@ cmd_code_test() { if vm_artifacts_present && has_kvm; then say "[code_test] VM artifacts present and /dev/kvm accessible; running with --features vm" if [ "$DOCKER_TEST" -eq 1 ]; then - (cd "$ROOT_DIR" && NYX_TEST_DOCKER=1 cargo test --release --features vm) + (cd "$ROOT_DIR" && NYX_TEST_DOCKER=1 run_cargo_mode test --release --features vm) else - (cd "$ROOT_DIR" && cargo test --release --features vm) + (cd "$ROOT_DIR" && run_cargo_mode test --release --features vm) fi else if vm_artifacts_present; then @@ -98,9 +138,9 @@ cmd_code_test() { say "[code_test] VM artifacts not initialized; running without vm feature" fi if [ "$DOCKER_TEST" -eq 1 ]; then - (cd "$ROOT_DIR" && NYX_TEST_DOCKER=1 cargo test --release) + (cd "$ROOT_DIR" && NYX_TEST_DOCKER=1 run_cargo_mode test --release) else - (cd "$ROOT_DIR" && cargo test --release) + (cd "$ROOT_DIR" && run_cargo_mode test --release) fi fi @@ -118,16 +158,36 @@ cmd_lean_test() { } cmd_test_full() { + cmd_check_nyx_lite_standalone cmd_init_vm - cmd_code_test + BUILD_CLI=1 cmd_code_test cmd_lean_test } +cmd_test_ci() { + say "[test_ci] Running local CI preflight script..." + need_cmd sh + (cd "$ROOT_DIR" && sh "$ROOT_DIR/scripts/test_ci_local.sh" "$@") + say "[test_ci] Done" +} + cmd_test_all() { + cmd_test_ci cmd_test_full } cmd_bench() { + if [ "${1:-}" = "cli" ]; then + shift + cmd_bench_cli "$@" + return 0 + fi + if [ "${1:-}" = "mcts" ]; then + shift + cmd_bench_mcts "$@" + return 0 + fi + suite=${INFOTHEORY_BENCH_SUITE:-two-json} case "${1:-}" in two-json|two_json|two|core|full) @@ -140,8 +200,8 @@ cmd_bench() { ;; esac case "${suite}" in - extra) suite_display="examples/extra.json" ;; - *) suite=two-json; suite_display="examples/two.json" ;; + extra) suite_display="configs/bench/extra.json" ;; + *) suite=two-json; suite_display="configs/bench/two.json" ;; esac say "[bench] Running ${suite_display} benchmark suite..." need_cmd sh @@ -149,12 +209,56 @@ cmd_bench() { say "[bench] Done" } -cmd_bench__aixi_competitors() { - say "[bench__aixi_competitors] Running reproducible Guix benchmark (Infotheory Rust/Python vs PyAIXI vs C++ MC-AIXI)..." +cmd_bench_cli() { + [ $# -ge 1 ] || fail "Usage: ./projman.sh bench cli [preset]" + need_cmd bash + validate_build_mode + cli_build_mode=$(build_mode) + case "${1:-}" in + -h|--help) + (cd "$ROOT_DIR" && INFOTHEORY_CLI_BENCH_BUILD_MODE="$cli_build_mode" bash "$ROOT_DIR/scripts/bench_cli_hyperfine.sh" "$@") + return 0 + ;; + --plan) + say "[bench_cli] Rendering hyperfine CLI plan (build mode: ${cli_build_mode})..." + (cd "$ROOT_DIR" && INFOTHEORY_CLI_BENCH_BUILD_MODE="$cli_build_mode" bash "$ROOT_DIR/scripts/bench_cli_hyperfine.sh" "$@") + say "[bench_cli] Done" + return 0 + ;; + esac + say "[bench_cli] Running hyperfine CLI comparison against baseline '$1' (build mode: ${cli_build_mode})..." + (cd "$ROOT_DIR" && INFOTHEORY_CLI_BENCH_BUILD_MODE="$cli_build_mode" bash "$ROOT_DIR/scripts/bench_cli_hyperfine.sh" "$@" && "$ROOT_DIR/scripts/summarize_interpret.sh") + say "[bench_cli] Done" +} + +cmd_bench_mcts() { + [ $# -ge 1 ] || fail "Usage: ./projman.sh bench mcts [--root ]" + baseline_commit="$1" + shift + need_cmd cargo + need_cmd git + need_cmd python3 + validate_build_mode + mcts_build_mode=$(build_mode) + say "[bench_mcts] Running MCTS planner regression benchmark against baseline '${baseline_commit}' (build mode: ${mcts_build_mode})..." + if [ "$mcts_build_mode" = "portable" ]; then + pf=$(portable_rustflags) + (cd "$ROOT_DIR" && \ + CARGO_BUILD_RUSTFLAGS="$pf" \ + RUSTDOCFLAGS="${RUSTDOCFLAGS:-$pf}" \ + python3 "$ROOT_DIR/scripts/bench_mcts_regression.py" --baseline "$baseline_commit" "$@") + else + (cd "$ROOT_DIR" && python3 "$ROOT_DIR/scripts/bench_mcts_regression.py" --baseline "$baseline_commit" "$@") + fi + say "[bench_mcts] Done" +} + +cmd_bench_aixi_competitors() { + say "[bench_aixi_competitors] Running reproducible Guix benchmark (Infotheory Rust/Python vs PyAIXI vs C++ MC-AIXI)..." need_cmd guix need_cmd bash (cd "$ROOT_DIR" && bash "$ROOT_DIR/scripts/bench_aixi_competitors_guix.sh" "$@") - say "[bench__aixi_competitors] Done" + say "[bench_aixi_competitors] Done" } cmd_plot() { @@ -170,8 +274,8 @@ cmd_plot() { ;; esac case "${suite}" in - extra) suite_display="examples/extra.json" ;; - *) suite=two-json; suite_display="examples/two.json" ;; + extra) suite_display="configs/bench/extra.json" ;; + *) suite=two-json; suite_display="configs/bench/two.json" ;; esac say "[plot] Legacy plot generation is superseded by the benchman TUI." say "[plot] Use './projman.sh tui ${suite}' to inspect ${suite_display} benchmarks." @@ -242,56 +346,84 @@ cmd_clean() { rm -rf "$ROOT_DIR/ite-bench/.lake/build" || true # nyx-lite guest artifacts - rm -f "$ROOT_DIR/nyx-lite/guest/aixi_guest" || true - rm -rf "$ROOT_DIR/nyx-lite/guest/initramfs" || true - rm -f "$ROOT_DIR/nyx-lite/guest/aixi_initramfs.cpio" || true + rm -f "$ROOT_DIR/vendor/nyx-lite/guest/aixi_guest" || true + rm -rf "$ROOT_DIR/vendor/nyx-lite/guest/initramfs" || true + rm -f "$ROOT_DIR/vendor/nyx-lite/guest/aixi_initramfs.cpio" || true # docker rootfs artifact (rebuildable) - rm -f "$ROOT_DIR/nyx-lite/vm_image/dockerimage/rootfs.ext4" || true - rm -rf "$ROOT_DIR/nyx-lite/vm_image/dockerimage/mnt" || true + rm -f "$ROOT_DIR/vendor/nyx-lite/vm_image/dockerimage/rootfs.ext4" || true + rm -rf "$ROOT_DIR/vendor/nyx-lite/vm_image/dockerimage/mnt" || true say "[clean] Done" } +cmd_legacy_aixi_convert() { + [ $# -eq 1 ] || fail "Usage: ./projman.sh legacy_aixi_convert " + + lua_cmd="" + if command -v luajit >/dev/null 2>&1; then + lua_cmd="luajit" + elif command -v lua >/dev/null 2>&1; then + lua_cmd="lua" + else + fail "Missing required command: luajit or lua" + fi + + (cd "$ROOT_DIR" && "$lua_cmd" "$ROOT_DIR/scripts/legacy_aixi_convert.lua" "$1") +} + usage() { cat <<'EOF' Usage: ./projman.sh Commands: bench [suite] Run benchmark suite (`two-json` default, or `extra`). Requires /tmp/enwik7 to exist and be exactly 10000000 bytes. Resumes the newest raw TSV for the selected suite by default; set INFOTHEORY_BENCH_FRESH=1 for a new run. Not included in test_all. - bench__aixi_competitors Run reproducible Guix time-machine benchmark for Infotheory MC-AIXI (Rust+Python) vs PyAIXI and C++ MC-AIXI. Fails fast if Guix is unavailable. + bench cli [preset] Build baseline vs dirty current trees and compare CLI workloads with hyperfine. Presets: `default` (signal-focused defaults) and `quick` (same matrix with lighter defaults). Writes artifacts under /var/tmp/infotheory_bench/. + bench mcts [--root

] Run Criterion planner benchmarks (`mcts_planners`) on a baseline worktree and current tree, then enforce Tranche 3.5 Part 1 regression gates (rho_uct >=5%, parallel >=10% fail). + bench_aixi_competitors Run reproducible Guix time-machine benchmark for Infotheory MC-AIXI (Rust+Python) vs PyAIXI and C++ MC-AIXI. Fails fast if Guix is unavailable. plot [suite] Open benchmark results in the benchman TUI for the selected suite (`two-json` default, or `extra`). Not included in test_all. tui [suite] Build and launch the interactive benchmark TUI (`benchman`) for the selected suite (`two-json` default, or `extra`). Supports --summary-tsv/--baseline-summary-tsv/--raw-tsv/--subjects and manages /tmp/plotimgs. tui log-loss Build and launch the log-loss diagnostic TUI for .trace.tsv / .nodes.tsv / .summary.tsv. tui man Open the local benchman manual via nvim man pager (MANPAGER='nvim +Man!'). code_test Build (release) and run Rust tests (release). Uses --features vm iff VM artifacts exist and /dev/kvm is accessible. + test_ci Run fast local CI preflight gates (Rust line coverage, rustdoc coverage, curated feature-gate checks, Python coverage/smoke). Set INFOTHEORY_CI_INCLUDE_VM=1 to include VM slices. init-vm Download/build VM artifacts needed for VM tests (kernel, initramfs, docker rootfs). lean_test Run Lean validation suite (ite-bench). Requires lake. test_full Run init-vm, code_test, and lean_test. - test_all Alias for test_full. + test_all Run test_ci then test_full. clean Clean build artifacts (cargo clean, lake clean, VM images/initramfs). Keeps vmlinux-6.1.58. + legacy_aixi_convert Convert legacy AIXI JSON config to canonical planner_run JSON and write to stdout. External configs print: External configs were deprecated. Environment variables: - INFOTHEORY_BENCH_* Passed through to scripts/bench_two_json.sh for benchmark tuning/output paths, including INFOTHEORY_BENCH_SUBJECTS=rwkv and INFOTHEORY_BENCH_SUITE=extra. - INFOTHEORY_PLOT_* Passed through to scripts/plot_two_json.sh, including INFOTHEORY_PLOT_SUBJECTS=rwkv, INFOTHEORY_PLOT_SUMMARY_TSV=..., and INFOTHEORY_PLOT_SUITE=extra. + INFOTHEORY_BUILD_MODE=native|portable Controls local cargo invocations in projman. `native` uses the repository's default target-cpu=native configuration; `portable` overrides local builds/tests to use generic CPU codegen like CI/release builds. + INFOTHEORY_BENCH_* Passed through to scripts/bench_two_json.sh for benchmark tuning/output paths, including INFOTHEORY_BENCH_SUBJECTS=rwkv7, INFOTHEORY_BENCH_SUITE=extra, and INFOTHEORY_BENCH_BUILD_MODE=native|portable. + INFOTHEORY_CLI_BENCH_* Passed through to scripts/bench_cli_hyperfine.sh for baseline/current CLI benchmark tuning and input selection. For `projman.sh bench cli`, INFOTHEORY_BUILD_MODE is canonical and is forwarded as INFOTHEORY_CLI_BENCH_BUILD_MODE. + INFOTHEORY_PLOT_* Passed through to scripts/plot_two_json.sh, including INFOTHEORY_PLOT_SUBJECTS=rwkv7, INFOTHEORY_PLOT_SUMMARY_TSV=..., and INFOTHEORY_PLOT_SUITE=extra. INFOTHEORY_BASELINE_SUMMARY_TSV / INFOTHEORY_BENCH_RAW_TSV Also read by benchman for baseline overlays and raw inspector detail. SKIP_DOCKER=1 Skip docker rootfs.ext4 build during init-vm. BUILD_CLI=1 Also build optional infotheory CLI binary (feature: cli) during code_test. + INFOTHEORY_CI_INCLUDE_VM=1 Include VM feature compile/Python VM smoke slices in test_ci. + INFOTHEORY_CI_SKIP_RUST_LINE_COVERAGE=1 Skip only the Rust line coverage gate in test_ci. + INFOTHEORY_CI_SKIP_RUSTDOC_COVERAGE=1 Skip only the rustdoc coverage gate in test_ci. + INFOTHEORY_CI_SKIP_FEATURE_GATES=1 Skip only curated Rust feature-gate checks in test_ci. + INFOTHEORY_CI_SKIP_PYTHON=1 Skip only Python coverage/smoke gates in test_ci. EOF } cmd=${1:-} case "$cmd" in bench) shift; cmd_bench "$@" ;; - bench__aixi_competitors) shift; cmd_bench__aixi_competitors "$@" ;; + bench_aixi_competitors) shift; cmd_bench_aixi_competitors "$@" ;; plot) shift; cmd_plot "$@" ;; tui) shift; cmd_tui "$@" ;; code_test) shift; cmd_code_test "$@" ;; + test_ci) shift; cmd_test_ci "$@" ;; init-vm) shift; cmd_init_vm "$@" ;; lean_test) shift; cmd_lean_test "$@" ;; test_full) shift; cmd_test_full "$@" ;; test_all) shift; cmd_test_all "$@" ;; clean) shift; cmd_clean "$@" ;; + legacy_aixi_convert) shift; cmd_legacy_aixi_convert "$@" ;; -h|--help|help|'') usage ;; *) usage; fail "Unknown command: $cmd" ;; esac diff --git a/pyproject.toml b/pyproject.toml index 7228aa1a..40eb3dee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "maturin" [project] name = "infotheory-rs" -version = "1.1.1" +version = "1.2.0" description = "Python bindings for the infotheory Rust library" readme = "README.md" requires-python = ">=3.10" @@ -30,12 +30,12 @@ dev = [ ] [tool.maturin] -manifest-path = "infotheory_py/Cargo.toml" +manifest-path = "crates/infotheory_py/Cargo.toml" module-name = "infotheory_rs._core" python-source = "python" profile = "python-release" editable-profile = "python-release" -features = ["python-extension", "backend-rosa", "backend-mamba", "backend-rwkv", "backend-zpaq"] +features = ["python-extension", "all-backends", "aixi-gameengine"] include = [ { path = "python/infotheory_rs/**/*.py", format = ["sdist", "wheel"] }, ] diff --git a/python/infotheory_rs/__init__.py b/python/infotheory_rs/__init__.py index e5482f33..cec50d45 100644 --- a/python/infotheory_rs/__init__.py +++ b/python/infotheory_rs/__init__.py @@ -4,6 +4,19 @@ - ergonomic wrappers for common entry points (`ncd_paths`, `ncd_bytes`) - abstract base classes for Python-driven AIXI trait adapters +- bit-level predictive primitives (`RateBackendBitSession`, `RateBackendBitSessionCheckpoint`, `BytePrefixMass`, `BinaryPrediction`) +- bit-stream semantics and ordering controls (`BitStreamSemantics`, `BitOrder`) +- context entrypoint for bit sessions (`InfotheoryCtx.rate_backend_bit_session`) + +Python `RateBackendBitSession` exposes `predict_bit`, `predict_one`, `step_bit`, `observe_bit`, `condition_bit`, `reset_frozen`, `begin_bit_stream`, `checkpoint`, `restore_checkpoint`, `clear_checkpoints_if_supported`, and `finish`. The checkpoint object is `RateBackendBitSessionCheckpoint`. See the native docstring on RateBackendBitSession for details. + +Accepted string aliases: + +- `BitOrder`: `"msb"`, `"msbfirst"`, `"msb_first"`, `"lsb"`, `"lsbfirst"`, `"lsb_first"` +- `BitStreamSemantics`: `"bytepacked"`, `"byte_packed"`, `"byte"` map to + `BitStreamSemantics.byte_packed(order=BitOrder.MsbFirst)`; `"binarytokens"`, + `"binary_tokens"`, `"binary"`, `"bit"` map to + `BitStreamSemantics.binary_tokens()` Callback error policy: @@ -16,8 +29,10 @@ from . import _core as _c from abc import ABC, abstractmethod +_REMOVED_PUBLIC_SYMBOLS = frozenset({"SearchNode"}) + for _name in dir(_c): - if not _name.startswith("_"): + if not _name.startswith("_") and _name not in _REMOVED_PUBLIC_SYMBOLS: globals()[_name] = getattr(_c, _name) @@ -188,7 +203,7 @@ def observation_stream_len(self) -> int: return 1 def observation_key_mode(self): - return "fullstream" + return "full_stream" @abstractmethod def get_num_reward_bits(self) -> int: ... diff --git a/python/tests/test_abc_defaults.py b/python/tests/test_abc_defaults.py index 62adfe48..e1117021 100644 --- a/python/tests/test_abc_defaults.py +++ b/python/tests/test_abc_defaults.py @@ -268,7 +268,7 @@ def test_environment_default_drain_observations(): def test_agent_simulator_default_methods(): sim = _Sim() assert sim.observation_stream_len() == 1 - assert sim.observation_key_mode() == "fullstream" + assert sim.observation_key_mode() == "full_stream" assert sim.reward_offset() == 0 assert sim.get_explore_exploit_ratio() == 1.0 assert sim.discount_gamma() == 1.0 diff --git a/python/tests/test_aixi.py b/python/tests/test_aixi.py index 29133273..02c00d0f 100644 --- a/python/tests/test_aixi.py +++ b/python/tests/test_aixi.py @@ -2,8 +2,101 @@ import pytest +class ToyCoinFlipEnv: + def __init__(self, p: float = 0.5, random_seed: int | None = None): + self.p = p + self.state = 0 + self.reward = 0 + self.finished = False + self.rng = random_seed if random_seed is not None else 1 + self._gen_next() + + def _next_u64(self) -> int: + x = self.rng if self.rng != 0 else 0xCAFEBABEDEADBEEF + x ^= (x >> 12) & 0xFFFFFFFFFFFFFFFF + x ^= (x << 25) & 0xFFFFFFFFFFFFFFFF + x ^= (x >> 27) & 0xFFFFFFFFFFFFFFFF + self.rng = x & 0xFFFFFFFFFFFFFFFF + return (self.rng * 0x2545F4914F6CDD1D) & 0xFFFFFFFFFFFFFFFF + + def _gen_bool(self, p: float) -> bool: + return (self._next_u64() >> 11) / float(1 << 53) < p + + def _gen_next(self) -> None: + self.state = 1 if self._gen_bool(self.p) else 0 + + def set_random_seed(self, seed: int) -> None: + self.rng = seed if seed != 0 else 0xCAFEBABEDEADBEEF + self.reward = 0 + self._gen_next() + + def perform_action(self, action: int): + self._gen_next() + self.reward = 1 if action == self.state else 0 + + def get_observation(self) -> int: + return self.state + + def get_reward(self) -> int: + return self.reward + + def is_finished(self) -> bool: + return self.finished + + def drain_observations(self) -> list[int]: + return [self.state] + + def get_observation_bits(self) -> int: + return 1 + + def get_reward_bits(self) -> int: + return 1 + + def get_action_bits(self) -> int: + return 1 + + +class ToyCtwTestEnv: + def __init__(self): + self.cycle = 0 + self.last_action = 0 + self.obs = 0 + self.reward = 0 + self.finished = False + + def perform_action(self, action: int): + if self.cycle == 0: + self.obs = 0 + else: + self.obs = (self.last_action + 1) % 2 + self.reward = 1 if action == self.obs else 0 + self.last_action = action + self.cycle += 1 + + def get_observation(self) -> int: + return self.obs + + def get_reward(self) -> int: + return self.reward + + def is_finished(self) -> bool: + return self.finished + + def drain_observations(self) -> list[int]: + return [self.obs] + + def get_observation_bits(self) -> int: + return 1 + + def get_reward_bits(self) -> int: + return 1 + + def get_action_bits(self) -> int: + return 1 + + def test_aixi_env_smoke(): - env = ait.CoinFlipEnv(0.5) + env = ToyCoinFlipEnv(0.5) env.perform_action(0) assert env.get_observation() in (0, 1) assert isinstance(env.get_reward(), int) @@ -11,8 +104,7 @@ def test_aixi_env_smoke(): def test_agent_config_and_agent_smoke(): cfg = ait.AgentConfig( - algorithm="fac-ctw", - ct_depth=8, + rate_backend=ait.RateBackend.ctw(8), agent_horizon=2, observation_bits=1, observation_stream_len=1, @@ -24,14 +116,37 @@ def test_agent_config_and_agent_smoke(): reward_offset=0, ) agent = ait.Agent(cfg) + assert agent.resolved_random_seed() == 0 + action = agent.get_planned_action([0], 0, 0) + assert action in (0, 1) + + +@pytest.mark.parametrize( + "mcts_strategy", + [ait.MctsStrategy.parallel_uct(2), ait.MctsStrategy.parallel_uct(2, 0.8)], +) +def test_agent_config_and_agent_support_explicit_parallel_mcts(mcts_strategy): + cfg = ait.AgentConfig( + rate_backend=ait.RateBackend.ctw(8), + agent_horizon=2, + observation_bits=1, + observation_stream_len=1, + reward_bits=1, + agent_actions=2, + num_simulations=8, + mcts_strategy=mcts_strategy, + min_reward=0, + max_reward=1, + reward_offset=0, + ) + agent = ait.Agent(cfg) action = agent.get_planned_action([0], 0, 0) assert action in (0, 1) def test_aiqi_config_and_agent_smoke(): cfg = ait.AiqiConfig( - algorithm="ac-ctw", - ct_depth=8, + rate_backend=ait.RateBackend.ctw(8), observation_bits=1, observation_stream_len=1, reward_bits=1, @@ -46,17 +161,69 @@ def test_aiqi_config_and_agent_smoke(): baseline_exploration=0.01, ) agent = ait.AiqiAgent(cfg) + assert agent.resolved_random_seed() == 0 action = agent.get_planned_action() assert action in (0, 1) agent.observe_transition(action, [0], 1) assert agent.steps_observed() == 1 +def test_planner_configs_accept_explicit_bit_stream_semantics(): + agent_cfg = ait.AgentConfig( + rate_backend=ait.RateBackend.ctw(8), + agent_horizon=2, + observation_bits=1, + observation_stream_len=1, + reward_bits=1, + agent_actions=2, + num_simulations=8, + min_reward=0, + max_reward=1, + reward_offset=0, + bit_stream_semantics=ait.BitStreamSemantics.binary_tokens(), + ) + agent = ait.Agent(agent_cfg) + assert agent.get_planned_action([0], 0, 0) in (0, 1) + + aiqi_cfg = ait.AiqiConfig( + rate_backend=ait.RateBackend.ctw(8), + observation_bits=1, + observation_stream_len=1, + reward_bits=1, + agent_actions=2, + min_reward=0, + max_reward=1, + reward_offset=0, + discount_gamma=0.99, + return_horizon=2, + return_bins=8, + augmentation_period=2, + baseline_exploration=0.01, + bit_stream_semantics="binary", + ) + aiqi = ait.AiqiAgent(aiqi_cfg) + assert aiqi.get_planned_action() in (0, 1) + + with pytest.raises(ValueError, match="BytePacked requires action and percept"): + ait.AgentConfig( + rate_backend=ait.RateBackend.ctw(8), + agent_horizon=2, + observation_bits=1, + observation_stream_len=1, + reward_bits=1, + agent_actions=2, + num_simulations=8, + min_reward=0, + max_reward=1, + reward_offset=0, + bit_stream_semantics="byte", + ) + + def test_run_aiqi_with_environment_smoke(): - env = ait.CoinFlipEnv(0.7) + env = ToyCoinFlipEnv(0.7) cfg = ait.AiqiConfig( - algorithm="ac-ctw", - ct_depth=6, + rate_backend=ait.RateBackend.ctw(6), observation_bits=1, observation_stream_len=1, reward_bits=1, @@ -83,13 +250,12 @@ def test_run_aiqi_with_environment_smoke(): assert summary["learn_cycles_completed"] == 6 assert summary["eval_cycles_completed"] == 4 assert isinstance(summary["eval_average_reward"], float) + assert summary["resolved_random_seed"] == 0 def test_run_aiqi_with_generic_rate_backend_smoke(): - env = ait.CoinFlipEnv(0.7) + env = ToyCoinFlipEnv(0.7) cfg = ait.AiqiConfig( - algorithm="ac-ctw", - ct_depth=6, observation_bits=1, observation_stream_len=1, reward_bits=1, @@ -103,7 +269,6 @@ def test_run_aiqi_with_generic_rate_backend_smoke(): augmentation_period=2, baseline_exploration=0.01, rate_backend=ait.RateBackend.ppmd(order=4, memory_mb=8), - rate_backend_max_order=8, ) summary = ait.run_aiqi_with_environment( env, @@ -120,17 +285,14 @@ def test_run_aiqi_with_generic_rate_backend_smoke(): def test_run_mcaixi_with_generic_mixture_rate_backend_smoke(): - env = ait.CoinFlipEnv(0.7) + env = ToyCoinFlipEnv(0.7) mixture = ait.RateBackend.mixture( ait.MixtureSpec( ait.MixtureKind.Convex, [ - ait.MixtureExpertSpec( - ait.RateBackend.ctw(6), max_order=-1, log_prior=0.0, name="ctw" - ), + ait.MixtureExpertSpec(ait.RateBackend.ctw(6), log_prior=0.0, name="ctw"), ait.MixtureExpertSpec( ait.RateBackend.ppmd(order=4, memory_mb=8), - max_order=-1, log_prior=0.0, name="ppmd", ), @@ -139,8 +301,7 @@ def test_run_mcaixi_with_generic_mixture_rate_backend_smoke(): ) ) cfg = ait.AgentConfig( - algorithm="zpaq", - ct_depth=6, + rate_backend=mixture, agent_horizon=2, observation_bits=1, observation_stream_len=1, @@ -153,9 +314,6 @@ def test_run_mcaixi_with_generic_mixture_rate_backend_smoke(): max_reward=1, reward_offset=0, random_seed=77, - rate_backend=mixture, - rate_backend_max_order=8, - zpaq_method="1", ) summary = ait.run_agent_with_environment( env, @@ -170,13 +328,93 @@ def test_run_mcaixi_with_generic_mixture_rate_backend_smoke(): assert summary["learn_cycles_completed"] == 4 assert summary["eval_cycles_completed"] == 2 assert summary["last_action"] in (0, 1) + assert summary["resolved_random_seed"] == 77 + + +def test_run_agent_omitted_seed_matches_explicit_default_seed(): + cfg_omitted = ait.AgentConfig( + rate_backend=ait.RateBackend.ctw(8), + agent_horizon=4, + observation_bits=1, + observation_stream_len=1, + reward_bits=1, + agent_actions=2, + num_simulations=40, + min_reward=0, + max_reward=1, + reward_offset=0, + ) + cfg_explicit = ait.AgentConfig( + rate_backend=ait.RateBackend.ctw(8), + agent_horizon=4, + observation_bits=1, + observation_stream_len=1, + reward_bits=1, + agent_actions=2, + num_simulations=40, + min_reward=0, + max_reward=1, + reward_offset=0, + random_seed=0, + ) + + s1 = ait.run_agent_with_environment(ToyCtwTestEnv(), cfg_omitted, learn_cycles=24, eval_cycles=8) + s2 = ait.run_agent_with_environment(ToyCtwTestEnv(), cfg_explicit, learn_cycles=24, eval_cycles=8) + + assert s1["resolved_random_seed"] == 0 + assert s2["resolved_random_seed"] == 0 + assert s1["learn_total_reward"] == s2["learn_total_reward"] + assert s1["eval_total_reward"] == s2["eval_total_reward"] + assert s1["last_action"] == s2["last_action"] + + +def test_run_aiqi_omitted_seed_matches_explicit_default_seed(): + cfg_omitted = ait.AiqiConfig( + rate_backend=ait.RateBackend.ctw(8), + observation_bits=1, + observation_stream_len=1, + reward_bits=1, + agent_actions=2, + min_reward=0, + max_reward=1, + reward_offset=0, + discount_gamma=0.99, + return_horizon=4, + return_bins=16, + augmentation_period=4, + baseline_exploration=0.2, + ) + cfg_explicit = ait.AiqiConfig( + rate_backend=ait.RateBackend.ctw(8), + observation_bits=1, + observation_stream_len=1, + reward_bits=1, + agent_actions=2, + min_reward=0, + max_reward=1, + reward_offset=0, + discount_gamma=0.99, + return_horizon=4, + return_bins=16, + augmentation_period=4, + baseline_exploration=0.2, + random_seed=0, + ) + + s1 = ait.run_aiqi_with_environment(ToyCtwTestEnv(), cfg_omitted, learn_cycles=24, eval_cycles=8) + s2 = ait.run_aiqi_with_environment(ToyCtwTestEnv(), cfg_explicit, learn_cycles=24, eval_cycles=8) + + assert s1["resolved_random_seed"] == 0 + assert s2["resolved_random_seed"] == 0 + assert s1["learn_total_reward"] == s2["learn_total_reward"] + assert s1["eval_total_reward"] == s2["eval_total_reward"] + assert s1["last_action"] == s2["last_action"] def test_aiqi_rejects_zpaq_algorithm_in_strict_mode(): with pytest.raises(ValueError, match="strict mode"): ait.AiqiConfig( - algorithm="zpaq", - ct_depth=6, + rate_backend=ait.RateBackend.zpaq("1"), observation_bits=1, observation_stream_len=1, reward_bits=1, @@ -192,31 +430,29 @@ def test_aiqi_rejects_zpaq_algorithm_in_strict_mode(): ) -def test_aiqi_rejects_non_power_of_two_return_bins(): - with pytest.raises(ValueError, match="power of two"): - ait.AiqiConfig( - algorithm="ac-ctw", - ct_depth=6, - observation_bits=1, - observation_stream_len=1, - reward_bits=1, - agent_actions=2, - min_reward=0, - max_reward=1, - reward_offset=0, - discount_gamma=0.99, - return_horizon=2, - return_bins=3, - augmentation_period=2, - baseline_exploration=0.01, - ) +def test_aiqi_accepts_non_power_of_two_return_bins(): + cfg = ait.AiqiConfig( + rate_backend=ait.RateBackend.ctw(6), + observation_bits=1, + observation_stream_len=1, + reward_bits=1, + agent_actions=2, + min_reward=0, + max_reward=1, + reward_offset=0, + discount_gamma=0.99, + return_horizon=2, + return_bins=3, + augmentation_period=2, + baseline_exploration=0.01, + ) + assert cfg is not None def test_aiqi_rejects_zpaq_rate_backend_in_strict_mode(): with pytest.raises(ValueError, match="strict frozen conditioning"): ait.AiqiConfig( - algorithm="ac-ctw", - ct_depth=6, + rate_backend=ait.RateBackend.zpaq("1"), observation_bits=1, observation_stream_len=1, reward_bits=1, @@ -229,16 +465,13 @@ def test_aiqi_rejects_zpaq_rate_backend_in_strict_mode(): return_bins=8, augmentation_period=2, baseline_exploration=0.01, - rate_backend=ait.RateBackend.zpaq("1"), - rate_backend_max_order=8, ) def test_mcaixi_rejects_zpaq_rate_backend_in_strict_mode(): with pytest.raises(ValueError, match="A Monte-Carlo AIXI Approximation"): ait.AgentConfig( - algorithm="fac-ctw", - ct_depth=6, + rate_backend=ait.RateBackend.zpaq("1"), agent_horizon=2, observation_bits=1, observation_stream_len=1, @@ -248,15 +481,12 @@ def test_mcaixi_rejects_zpaq_rate_backend_in_strict_mode(): min_reward=0, max_reward=1, reward_offset=0, - rate_backend=ait.RateBackend.zpaq("1"), - rate_backend_max_order=8, ) def test_aiqi_optional_history_pruning_smoke(): cfg = ait.AiqiConfig( - algorithm="ac-ctw", - ct_depth=6, + rate_backend=ait.RateBackend.ctw(6), observation_bits=1, observation_stream_len=1, reward_bits=1, @@ -272,7 +502,7 @@ def test_aiqi_optional_history_pruning_smoke(): baseline_exploration=0.01, ) agent = ait.AiqiAgent(cfg) - env = ait.CoinFlipEnv(0.7) + env = ToyCoinFlipEnv(0.7) for _ in range(64): action = agent.get_planned_action() env.perform_action(action) @@ -282,8 +512,7 @@ def test_aiqi_optional_history_pruning_smoke(): def test_mcaixi_seed_reproducibility_with_deterministic_env(): cfg = ait.AgentConfig( - algorithm="ctw", - ct_depth=8, + rate_backend=ait.RateBackend.ctw(8), agent_horizon=4, observation_bits=1, observation_stream_len=1, @@ -299,7 +528,7 @@ def test_mcaixi_seed_reproducibility_with_deterministic_env(): ) s1 = ait.run_agent_with_environment( - ait.CtwTestEnv(), + ToyCtwTestEnv(), cfg, learn_cycles=40, eval_cycles=20, @@ -309,7 +538,7 @@ def test_mcaixi_seed_reproducibility_with_deterministic_env(): check_finished=False, ) s2 = ait.run_agent_with_environment( - ait.CtwTestEnv(), + ToyCtwTestEnv(), cfg, learn_cycles=40, eval_cycles=20, @@ -327,8 +556,7 @@ def test_mcaixi_seed_reproducibility_with_deterministic_env(): def test_aiqi_seed_reproducibility_with_deterministic_env(): cfg = ait.AiqiConfig( - algorithm="ac-ctw", - ct_depth=8, + rate_backend=ait.RateBackend.ctw(8), observation_bits=1, observation_stream_len=1, reward_bits=1, @@ -345,7 +573,7 @@ def test_aiqi_seed_reproducibility_with_deterministic_env(): ) s1 = ait.run_aiqi_with_environment( - ait.CtwTestEnv(), + ToyCtwTestEnv(), cfg, learn_cycles=40, eval_cycles=20, @@ -355,7 +583,7 @@ def test_aiqi_seed_reproducibility_with_deterministic_env(): check_finished=False, ) s2 = ait.run_aiqi_with_environment( - ait.CtwTestEnv(), + ToyCtwTestEnv(), cfg, learn_cycles=40, eval_cycles=20, diff --git a/python/tests/test_aixi_gameengine.py b/python/tests/test_aixi_gameengine.py new file mode 100644 index 00000000..7fbcdd51 --- /dev/null +++ b/python/tests/test_aixi_gameengine.py @@ -0,0 +1,47 @@ +import pytest + +import infotheory_rs as ait + + +pytestmark = pytest.mark.skipif( + not hasattr(ait, "CoinFlipEnv"), + reason="requires the aixi-gameengine feature", +) + + +def test_coinflip_env_preserves_bias_and_seed_api(): + heads = ait.CoinFlipEnv(1.0, random_seed=7) + heads.perform_action(1) + assert heads.get_observation() == 1 + assert heads.get_reward() == 1 + + heads.set_random_seed(11) + heads.perform_action(0) + assert heads.get_observation() == 1 + assert heads.get_reward() == 0 + + tails = ait.CoinFlipEnv(0.0, random_seed=7) + tails.perform_action(0) + assert tails.get_observation() == 0 + assert tails.get_reward() == 1 + + +def test_coinflip_env_default_seed_matches_explicit_zero_seed(): + default_seed_env = ait.CoinFlipEnv(0.37) + explicit_zero_env = ait.CoinFlipEnv(0.37, random_seed=0) + + trace_default = [] + trace_zero = [] + for action in [0, 1, 1, 0, 1, 0, 0, 1]: + default_seed_env.perform_action(action) + explicit_zero_env.perform_action(action) + trace_default.append((default_seed_env.get_observation(), default_seed_env.get_reward())) + trace_zero.append((explicit_zero_env.get_observation(), explicit_zero_env.get_reward())) + + assert trace_default == trace_zero + + +@pytest.mark.parametrize("p", [-0.1, 1.1, float("nan")]) +def test_coinflip_env_rejects_invalid_probability(p): + with pytest.raises(ValueError, match="finite probability"): + ait.CoinFlipEnv(p) diff --git a/python/tests/test_api_surface.py b/python/tests/test_api_surface.py index 5114e359..e05a0ab2 100644 --- a/python/tests/test_api_surface.py +++ b/python/tests/test_api_surface.py @@ -30,10 +30,16 @@ def test_expected_public_surface_symbols_present(): "RateBackend", "CompressionBackend", "InfotheoryCtx", + "MctsStrategy", "GenerationStrategy", "GenerationUpdateMode", "GenerationConfig", "RateBackendSession", + "BitOrder", + "BitStreamSemantics", + "BinaryPrediction", + "BytePrefixMass", + "RateBackendBitSession", "MixtureKind", "MixtureScheduleMode", "MixtureExpertSpec", @@ -70,33 +76,33 @@ def test_expected_public_surface_symbols_present(): def test_functional_metrics_surface_bytes_and_matrix(): x = b"abracadabra" y = b"alakazam" - assert _is_finite_nonnegative(ait.marginal_entropy_bytes(x)) - assert _is_finite_nonnegative(ait.entropy_rate_bytes(x, 4)) - assert _is_finite_nonnegative(ait.biased_entropy_rate_bytes(x, 4)) - assert _is_finite_nonnegative(ait.joint_marginal_entropy_bytes(x, y)) - assert _is_finite_nonnegative(ait.joint_entropy_rate_bytes(x, y, 4)) - assert _is_finite_nonnegative(ait.conditional_entropy_bytes(x, y, 4)) - assert _is_finite_nonnegative(ait.conditional_entropy_rate_bytes(x, y, 4)) - assert _is_finite_nonnegative(ait.mutual_information_bytes(x, y, 4)) - assert _is_finite_nonnegative(ait.mutual_information_marg_bytes(x, y)) - assert _is_finite_nonnegative(ait.mutual_information_rate_bytes(x, y, 4)) - assert _is_finite_nonnegative(ait.ned_bytes(x, y, 4)) - assert _is_finite_nonnegative(ait.ned_marg_bytes(x, y)) - assert _is_finite_nonnegative(ait.ned_rate_bytes(x, y, 4)) - assert _is_finite_nonnegative(ait.ned_cons_bytes(x, y, 4)) - assert _is_finite_nonnegative(ait.ned_cons_marg_bytes(x, y)) - assert _is_finite_nonnegative(ait.ned_cons_rate_bytes(x, y, 4)) - assert _is_finite_nonnegative(ait.nte_bytes(x, y, 4)) - assert _is_finite_nonnegative(ait.nte_marg_bytes(x, y)) - assert _is_finite_nonnegative(ait.nte_rate_bytes(x, y, 4)) - assert _is_finite_nonnegative(ait.tvd_bytes(x, y, 0)) - assert _is_finite_nonnegative(ait.nhd_bytes(x, y, 0)) - assert _is_finite_nonnegative(ait.cross_entropy_bytes(x, y, 4)) - assert _is_finite_nonnegative(ait.cross_entropy_rate_bytes(x, y, 4)) + assert _is_finite_nonnegative(ait.empirical_entropy_bytes(x)) + assert _is_finite_nonnegative(ait.entropy_rate_bytes(x)) + assert _is_finite_nonnegative(ait.biased_entropy_rate_bytes(x)) + assert _is_finite_nonnegative(ait.empirical_joint_entropy_bytes(x, y)) + assert _is_finite_nonnegative(ait.joint_entropy_rate_bytes(x, y)) + assert _is_finite_nonnegative(ait.conditional_entropy_bytes(x, y)) + assert _is_finite_nonnegative(ait.conditional_entropy_rate_bytes(x, y)) + assert _is_finite_nonnegative(ait.mutual_information_bytes(x, y)) + assert _is_finite_nonnegative(ait.empirical_mutual_information_bytes(x, y)) + assert _is_finite_nonnegative(ait.mutual_information_rate_bytes(x, y)) + assert _is_finite_nonnegative(ait.ned_bytes(x, y)) + assert _is_finite_nonnegative(ait.empirical_ned_bytes(x, y)) + assert _is_finite_nonnegative(ait.ned_rate_bytes(x, y)) + assert _is_finite_nonnegative(ait.ned_cons_bytes(x, y)) + assert _is_finite_nonnegative(ait.empirical_ned_cons_bytes(x, y)) + assert _is_finite_nonnegative(ait.ned_cons_rate_bytes(x, y)) + assert _is_finite_nonnegative(ait.nte_bytes(x, y)) + assert _is_finite_nonnegative(ait.empirical_nte_bytes(x, y)) + assert _is_finite_nonnegative(ait.nte_rate_bytes(x, y)) + assert _is_finite_nonnegative(ait.tvd_bytes(x, y)) + assert _is_finite_nonnegative(ait.nhd_bytes(x, y)) + assert _is_finite_nonnegative(ait.cross_entropy_bytes(x, y)) + assert _is_finite_nonnegative(ait.cross_entropy_rate_bytes(x, y)) assert _is_finite_nonnegative(ait.d_kl_bytes(x, y)) assert _is_finite_nonnegative(ait.js_div_bytes(x, y)) - assert _is_finite_nonnegative(ait.intrinsic_dependence_bytes(x, 4)) - assert _is_finite_nonnegative(ait.resistance_to_transformation_bytes(x, x, 4)) + assert _is_finite_nonnegative(ait.intrinsic_dependence_bytes(x)) + assert _is_finite_nonnegative(ait.resistance_to_transformation_bytes(x, x)) matrix = ait.ncd_matrix_bytes([x, y, b"xyzxyz"], method="5", variant="sym") assert len(matrix) == 9 @@ -114,31 +120,25 @@ def test_backend_objects_context_and_helpers(tmp_path): try: ait.set_default_ctx(ctx) assert isinstance(ait.get_default_ctx(), ait.InfotheoryCtx) - assert _is_finite_nonnegative(ctx.entropy_rate_bytes(b"abcabcabc", 4)) - assert _is_finite_nonnegative(ctx.biased_entropy_rate_bytes(b"abcabcabc", 4)) + assert _is_finite_nonnegative(ctx.entropy_rate_bytes(b"abcabcabc")) + assert _is_finite_nonnegative(ctx.biased_entropy_rate_bytes(b"abcabcabc")) assert _is_finite_nonnegative(ctx.compress_size(b"payload")) assert _is_finite_nonnegative(ctx.compress_size_chain([b"pay", b"load"])) - assert _is_finite_nonnegative( - ctx.cross_entropy_rate_bytes(b"abcabc", b"abcabd", 4) - ) - assert _is_finite_nonnegative(ctx.cross_entropy_bytes(b"abcabc", b"abcabd", 4)) - assert _is_finite_nonnegative(ctx.joint_entropy_rate_bytes(b"abc", b"abd", 4)) - assert _is_finite_nonnegative( - ctx.conditional_entropy_rate_bytes(b"abc", b"abd", 4) - ) + assert _is_finite_nonnegative(ctx.cross_entropy_rate_bytes(b"abcabc", b"abcabd")) + assert _is_finite_nonnegative(ctx.cross_entropy_bytes(b"abcabc", b"abcabd")) + assert _is_finite_nonnegative(ctx.joint_entropy_rate_bytes(b"abc", b"abd")) + assert _is_finite_nonnegative(ctx.conditional_entropy_rate_bytes(b"abc", b"abd")) assert _is_finite_nonnegative( ctx.cross_entropy_conditional_chain([b"ab", b"ca"], b"bc") ) - assert _is_finite_nonnegative(ctx.mutual_information_rate_bytes(b"abc", b"abd", 4)) - assert _is_finite_nonnegative(ctx.mutual_information_bytes(b"abc", b"abd", 4)) - assert _is_finite_nonnegative(ctx.conditional_entropy_bytes(b"abc", b"abd", 4)) - assert _is_finite_nonnegative(ctx.ned_bytes(b"abc", b"abd", 4)) - assert _is_finite_nonnegative(ctx.ned_cons_bytes(b"abc", b"abd", 4)) - assert _is_finite_nonnegative(ctx.nte_bytes(b"abc", b"abd", 4)) - assert _is_finite_nonnegative(ctx.intrinsic_dependence_bytes(b"abcabc", 4)) - assert _is_finite_nonnegative( - ctx.resistance_to_transformation_bytes(b"abc", b"abc", 4) - ) + assert _is_finite_nonnegative(ctx.mutual_information_rate_bytes(b"abc", b"abd")) + assert _is_finite_nonnegative(ctx.mutual_information_bytes(b"abc", b"abd")) + assert _is_finite_nonnegative(ctx.conditional_entropy_bytes(b"abc", b"abd")) + assert _is_finite_nonnegative(ctx.ned_bytes(b"abc", b"abd")) + assert _is_finite_nonnegative(ctx.ned_cons_bytes(b"abc", b"abd")) + assert _is_finite_nonnegative(ctx.nte_bytes(b"abc", b"abd")) + assert _is_finite_nonnegative(ctx.intrinsic_dependence_bytes(b"abcabc")) + assert _is_finite_nonnegative(ctx.resistance_to_transformation_bytes(b"abc", b"abc")) assert _is_finite_nonnegative(ctx.ncd_bytes(b"abc", b"abd", "vitanyi")) assert _is_finite_nonnegative( ait.ncd_paths(str(a), str(b), backend="zpaq", method="5", variant="vitanyi") @@ -167,19 +167,15 @@ def test_backend_objects_context_and_helpers(tmp_path): compressed = ait.compress_bytes_backend(b"payload", "zpaq", "5") assert ait.decompress_bytes_backend(compressed, "zpaq", "5") == b"payload" assert ait.validate_zpaq_rate_method("1") is None - assert _is_finite_nonnegative(ait.entropy_rate_backend(b"abc", 4, backend=rb)) - assert _is_finite_nonnegative(ait.biased_entropy_rate_backend(b"abc", 4, backend=rb)) - assert _is_finite_nonnegative( - ait.joint_entropy_rate_backend(b"abc", b"abd", 4, backend=rb) - ) - assert _is_finite_nonnegative( - ait.mutual_information_rate_backend(b"abc", b"abd", 4, backend=rb) - ) - assert _is_finite_nonnegative(ait.ned_rate_backend(b"abc", b"abd", 4, backend=rb)) - assert _is_finite_nonnegative(ait.nte_rate_backend(b"abc", b"abd", 4, backend=rb)) + assert _is_finite_nonnegative(ait.entropy_rate_backend(b"abc", backend=rb)) + assert _is_finite_nonnegative(ait.biased_entropy_rate_backend(b"abc", backend=rb)) + assert _is_finite_nonnegative(ait.joint_entropy_rate_backend(b"abc", b"abd", backend=rb)) assert _is_finite_nonnegative( - ait.cross_entropy_rate_backend(b"abc", b"abd", 4, backend=rb) + ait.mutual_information_rate_backend(b"abc", b"abd", backend=rb) ) + assert _is_finite_nonnegative(ait.ned_rate_backend(b"abc", b"abd", backend=rb)) + assert _is_finite_nonnegative(ait.nte_rate_backend(b"abc", b"abd", backend=rb)) + assert _is_finite_nonnegative(ait.cross_entropy_rate_backend(b"abc", b"abd", backend=rb)) generated = ctx.generate_bytes( b"abcabcabc", 4, @@ -206,10 +202,12 @@ def test_bit_and_observation_helpers(): assert isinstance( ait.observation_key_from_stream(ait.ObservationKeyMode.StreamHash, stream, 8), int ) - assert isinstance(ait.observation_key_from_stream("stream-hash", stream, 8), int) - assert isinstance(ait.observation_key_from_stream("hash", stream, 8), int) - assert isinstance(ait.observation_repr_from_stream("full", stream, 8), list) - assert isinstance(ait.observation_repr_from_stream("full-stream", stream, 8), list) + assert isinstance(ait.observation_key_from_stream("stream_hash", stream, 8), int) + with pytest.raises(ValueError): + ait.observation_key_from_stream("stream-hash", stream, 8) + assert isinstance(ait.observation_repr_from_stream("full_stream", stream, 8), list) + with pytest.raises(ValueError): + ait.observation_repr_from_stream("full-stream", stream, 8) assert isinstance(ait.observation_repr_from_stream("last", stream, 8), list) @@ -261,11 +259,7 @@ def test_new_rate_backends_parse_and_execute(tmp_path): particle_spec = ait.ParticleSpec(num_particles=4, num_cells=4, cell_dim=8) mixture_spec = ait.MixtureSpec( ait.MixtureKind.Convex, - [ - ait.MixtureExpertSpec( - ait.RateBackend.match(), max_order=-1, log_prior=0.0, name="match" - ) - ], + [ait.MixtureExpertSpec(ait.RateBackend.match(), log_prior=0.0, name="match")], alpha=0.02, schedule=ait.MixtureScheduleMode.Theorem, ) @@ -288,10 +282,8 @@ def test_new_rate_backends_parse_and_execute(tmp_path): payload = b"abracadabra abracadabra" peer = b"alakazam alakazam" for backend in parsed_backends + constructed_backends: - assert _is_finite_nonnegative(ait.entropy_rate_backend(payload, 4, backend=backend)) - assert _is_finite_nonnegative( - ait.cross_entropy_rate_backend(payload, peer, 4, backend=backend) - ) + assert _is_finite_nonnegative(ait.entropy_rate_backend(payload, backend=backend)) + assert _is_finite_nonnegative(ait.cross_entropy_rate_backend(payload, peer, backend=backend)) with pytest.raises(ValueError): ait.rate_backend("unknown-backend") @@ -354,9 +346,9 @@ def test_mamba_rate_backend_parse_construct_metrics_and_roundtrip_parity(): object_backend = ait.RateBackend.mamba(method) for backend in (parsed_backend, object_backend): - assert _is_finite_nonnegative(ait.entropy_rate_backend(payload, 4, backend=backend)) + assert _is_finite_nonnegative(ait.entropy_rate_backend(payload, backend=backend)) assert _is_finite_nonnegative( - ait.cross_entropy_rate_backend(payload, peer, 4, backend=backend) + ait.cross_entropy_rate_backend(payload, peer, backend=backend) ) framed_from_parsed = ait.CompressionBackend.rate_ac(parsed_backend, "framed") diff --git a/python/tests/test_bit_session.py b/python/tests/test_bit_session.py new file mode 100644 index 00000000..5bf7b3fa --- /dev/null +++ b/python/tests/test_bit_session.py @@ -0,0 +1,347 @@ +import math +import pytest + +import infotheory_rs as ait + + +def _normalize_pdf(weights): + total = sum(weights) + return [weight / total for weight in weights] + + +def test_bit_order(): + assert hasattr(ait.BitOrder, "MsbFirst") + assert hasattr(ait.BitOrder, "LsbFirst") + + msb = ait.BitOrder.MsbFirst + lsb = ait.BitOrder.LsbFirst + + assert repr(msb) == "BitOrder.MsbFirst" + assert repr(lsb) == "BitOrder.LsbFirst" + + +def test_bit_stream_semantics(): + msb = ait.BitOrder.MsbFirst + lsb = ait.BitOrder.LsbFirst + + bp_default = ait.BitStreamSemantics.byte_packed() + assert bp_default.kind == "byte_packed" + assert bp_default.order == msb + + bp_lsb = ait.BitStreamSemantics.byte_packed(lsb) + assert bp_lsb.kind == "byte_packed" + assert bp_lsb.order == lsb + + # Check that we can also pass a string alias for the order + bp_lsb_str = ait.BitStreamSemantics.byte_packed("lsb_first") + assert bp_lsb_str.kind == "byte_packed" + assert bp_lsb_str.order == lsb + + bt = ait.BitStreamSemantics.binary_tokens() + assert bt.kind == "binary_tokens" + assert bt.order is None + + assert "byte_packed" in repr(bp_default) + assert "binary_tokens" in repr(bt) + + +def test_binary_prediction(): + pred = ait.BinaryPrediction(0.3, 0.7) + assert pred == ait.BinaryPrediction.from_prob_one_exact(0.7) + assert pred.p0 == 1.0 - pred.p1 + assert pred.p1 == 0.7 + assert pred.prob(False) == pred.p0 + assert pred.prob(True) == pred.p1 + assert "BinaryPrediction" in repr(pred) + + exact = ait.BinaryPrediction.from_prob_one_exact(0.9) + assert exact.p1 == 0.9 + assert abs(exact.p0 - 0.1) < 1e-12 + + floored = ait.BinaryPrediction.from_prob_one(1.2, 0.01) + assert floored.p1 == 0.99 + assert abs(floored.p0 - 0.01) < 1e-12 + + for invalid in (math.nan, math.inf, -math.inf): + with pytest.raises(ValueError, match="must be finite"): + ait.BinaryPrediction.from_prob_one(invalid) + with pytest.raises(ValueError, match="must be finite"): + ait.BinaryPrediction.from_prob_one_exact(invalid) + + # Accept tiny floating-point drift, but canonicalize back to an exact complement. + pred_near_one = ait.BinaryPrediction(math.nextafter(1.0, 0.0), 0.0) + assert pred_near_one.p0 == 1.0 + assert pred_near_one.p1 == 0.0 + + # Even an exact rounded sum of 1.0 must still canonicalize through P(1). + pathological_p0 = 0.9990000000000001 + pathological_p1 = 0.0009999999999999979 + pathological = ait.BinaryPrediction(pathological_p0, pathological_p1) + assert pathological.p1 == pathological_p1 + assert pathological.p0 == 1.0 - pathological_p1 + assert pathological.p0 != pathological_p0 + + ulp = math.ulp(1.0) + pred_boundary = ait.BinaryPrediction(0.5 - 4.0 * ulp, 0.5) + assert pred_boundary.p0 + pred_boundary.p1 == 1.0 + + with pytest.raises(ValueError, match="must sum to 1"): + ait.BinaryPrediction(0.5 - 5.0 * ulp, 0.5) + with pytest.raises(ValueError, match="must sum to 1"): + ait.BinaryPrediction(0.3, 0.4) + with pytest.raises(ValueError, match="must be finite and >= 0"): + ait.BinaryPrediction(2.0, -1.0) + with pytest.raises(ValueError, match="must sum to 1"): + ait.BinaryPrediction(0.99999, 0.0) + + pred_norm = ait.BinaryPrediction(0.33333333, 0.66666667) + assert pred_norm.p0 + pred_norm.p1 == 1.0 + + +def test_byte_prefix_mass_from_pdf_tracks_symbol_probability(): + pdf = _normalize_pdf([idx + 1 for idx in range(256)]) + symbol = 0b1010_0110 + prefix = ait.BytePrefixMass.from_pdf(pdf, ait.BitOrder.MsbFirst) + + assert not prefix.is_complete() + assert not prefix.has_partial_bits() + with pytest.raises(RuntimeError, match="only meaningful after a full byte"): + prefix.symbol() + + product = 1.0 + for bit_idx in range(8): + bit = ((symbol >> (7 - bit_idx)) & 1) == 1 + pred = prefix.prediction() + product *= pred.prob(bit) + prefix.observe(bit) + if bit_idx < 7: + assert prefix.has_partial_bits() + + assert prefix.is_complete() + assert not prefix.has_partial_bits() + assert prefix.symbol() == symbol + assert abs(product - pdf[symbol]) < 1e-12 + assert "complete=True" in repr(prefix) + + +def test_byte_prefix_mass_from_log_probs_matches_backend_row(): + backend = ait.RateBackend.ctw(6) + byte_session = ait.RateBackendSession(backend, total_symbols=16) + + for symbol in b"bit-session": + log_probs = byte_session.fill_log_probs() + prefix = ait.BytePrefixMass.from_log_probs(log_probs, "msb_first") + expected = math.exp(log_probs[symbol]) + + product = 1.0 + for bit_idx in range(8): + bit = ((symbol >> (7 - bit_idx)) & 1) == 1 + pred = prefix.prediction() + product *= pred.prob(bit) + prefix.observe(bit) + + assert prefix.is_complete() + assert prefix.symbol() == symbol + assert abs(product - expected) < 1e-9 + byte_session.observe(bytes([symbol])) + + byte_session.finish() + + +def test_byte_prefix_mass_rejects_wrong_row_lengths(): + with pytest.raises(ValueError, match="expects exactly 256 entries"): + ait.BytePrefixMass.from_pdf([0.5, 0.5]) + with pytest.raises(ValueError, match="expects exactly 256 entries"): + ait.BytePrefixMass.from_log_probs([0.0] * 255) + + +def test_byte_packed_bit_session_matches_byte_prediction_chain(): + backend = ait.RateBackend.ctw(6) + + # 16 symbols * 8 bits = 128 bits + byte_session = ait.RateBackendSession(backend, total_symbols=16) + bit_session = ait.RateBackendBitSession( + backend, + total_bits=128, + semantics=ait.BitStreamSemantics.byte_packed(ait.BitOrder.MsbFirst), + ) + + data = b"bit-session" + for symbol in data: + row = byte_session.fill_log_probs() + expected = math.exp(row[symbol]) + + product = 1.0 + for bit_idx in range(8): + bit = ((symbol >> (7 - bit_idx)) & 1) == 1 + pred = bit_session.step_bit(bit) + product *= pred.prob(bit) + + byte_session.observe(bytes([symbol])) + assert abs(product - expected) < 1e-9 + + byte_session.finish() + bit_session.finish() + + +def test_bit_session_string_aliases(): + # Verify string alias parsing + backend = ait.RateBackend.ctw(6) + + # "byte" or "byte_packed" as string alias + sess_byte = ait.RateBackendBitSession(backend, total_bits=8, semantics="byte") + sess_byte.finish() + + # "binary" or "binary_tokens" as string alias + sess_bit = ait.RateBackendBitSession(backend, total_bits=9, semantics="binary") + sess_bit.finish() + + +def test_byte_packed_bit_session_rejects_non_byte_aligned_lengths(): + backend = ait.RateBackend.ctw(6) + + # Try non-byte aligned total_bits + with pytest.raises(RuntimeError, match="whole number of bytes"): + ait.RateBackendBitSession(backend, total_bits=9, semantics="byte") + + sess = ait.RateBackendBitSession(backend, total_bits=8, semantics="byte") + + with pytest.raises(RuntimeError, match="whole number of bytes"): + sess.reset_frozen(total_bits=9) + + +def test_bit_session_predict_and_condition(): + backend = ait.RateBackend.ctw(6) + sess = ait.RateBackendBitSession(backend, total_bits=8, semantics="binary") + + pred1 = sess.predict_bit() + p1_val = sess.predict_one() + assert abs(pred1.p1 - p1_val) < 1e-12 + + sess.condition_bit(True) + sess.observe_bit(False) + sess.finish() + + +def test_byte_packed_bit_session_rejects_mixed_update_modes_without_panicking_python(): + backend = ait.RateBackend.ctw(6) + sess = ait.RateBackendBitSession(backend, total_bits=8, semantics="byte") + + sess.condition_bit(True) + + with pytest.raises(RuntimeError, match="cannot mix conditioning-only and adaptive updates"): + sess.observe_bit(False) + + +def test_zpaq_bit_session_begin_bit_stream_restarts_without_frozen_reset(): + sess = ait.RateBackendBitSession(ait.RateBackend.zpaq("1"), total_bits=9, semantics="binary") + + with pytest.raises(RuntimeError, match="plugin entropy"): + sess.reset_frozen(total_bits=9) + + sess.begin_bit_stream(total_bits=9) + + for bit in [True, False, True, True, False, False, True, False, True]: + pred = sess.step_bit(bit) + assert abs((pred.p0 + pred.p1) - 1.0) < 1e-12 + + with pytest.raises(RuntimeError, match="semantics are fixed"): + sess.begin_bit_stream(total_bits=9, semantics="byte") + + sess.finish() + + +def test_mixture_with_zpaq_bit_session_begin_bit_stream_restarts_without_frozen_reset(): + mixture_spec = ait.MixtureSpec( + ait.MixtureKind.Bayes, + [ + ait.MixtureExpertSpec(ait.RateBackend.ctw(6)), + ait.MixtureExpertSpec(ait.RateBackend.zpaq("1")), + ], + ) + backend = ait.RateBackend.mixture(mixture_spec) + sess = ait.RateBackendBitSession(backend, total_bits=9, semantics="binary") + + with pytest.raises(RuntimeError, match="plugin entropy"): + sess.reset_frozen(total_bits=9) + + sess.begin_bit_stream(total_bits=9) + + for bit in [True, False, True, False, True, True, False, False, True]: + pred = sess.step_bit(bit) + assert abs((pred.p0 + pred.p1) - 1.0) < 1e-12 + + sess.finish() + + +def test_ctx_rate_backend_bit_session_delegates_to_default_backend(): + rb = ait.RateBackend.ctw(8) + cb = ait.CompressionBackend.zpaq("5") + ctx = ait.InfotheoryCtx(rb, cb) + + sess = ctx.rate_backend_bit_session(total_bits=8, semantics="binary") + + pred = sess.predict_bit() + assert isinstance(pred, ait.BinaryPrediction) + + sess.observe_bit(True) + sess.finish() + + +def test_bit_session_checkpoint_restore_binary_tokens_roundtrip(): + backend = ait.RateBackend.ctw(6) + sess = ait.RateBackendBitSession(backend, total_bits=12, semantics="binary") + + for bit in [True, False, True]: + sess.observe_bit(bit) + + pred_before = sess.predict_bit() + checkpoint = sess.checkpoint() + + for bit in [False, False, True, True]: + sess.observe_bit(bit) + + sess.restore_checkpoint(checkpoint) + pred_after = sess.predict_bit() + assert abs(pred_before.p1 - pred_after.p1) < 1e-12 + + for bit in [True, False, False, True, True, False, True, False, False]: + sess.observe_bit(bit) + sess.finish() + + +def test_bit_session_checkpoint_restore_mid_prefix_byte_packed(): + backend = ait.RateBackend.ctw(6) + sess = ait.RateBackendBitSession( + backend, + total_bits=8, + semantics=ait.BitStreamSemantics.byte_packed(ait.BitOrder.MsbFirst), + ) + + sess.condition_bit(True) + sess.condition_bit(False) + pred_before = sess.predict_one() + checkpoint = sess.checkpoint() + + sess.condition_bit(True) + sess.condition_bit(True) + sess.condition_bit(False) + + sess.restore_checkpoint(checkpoint) + pred_after = sess.predict_one() + assert abs(pred_before - pred_after) < 1e-12 + + sess.clear_checkpoints_if_supported() + for bit in [True, False, True, False, True, False]: + sess.condition_bit(bit) + sess.finish() + + +def test_bit_session_checkpoint_rejects_mismatched_semantics(): + backend = ait.RateBackend.ctw(6) + binary = ait.RateBackendBitSession(backend, total_bits=8, semantics="binary") + byte = ait.RateBackendBitSession(backend, total_bits=8, semantics="byte") + + checkpoint = binary.checkpoint() + with pytest.raises(RuntimeError, match="different backend or bit semantics"): + byte.restore_checkpoint(checkpoint) diff --git a/python/tests/test_cli_benchmark_scripts.py b/python/tests/test_cli_benchmark_scripts.py new file mode 100644 index 00000000..b7301005 --- /dev/null +++ b/python/tests/test_cli_benchmark_scripts.py @@ -0,0 +1,475 @@ +import json +import functools +import os +import pathlib +import shutil +import subprocess + +import pytest + + +def _repo_root() -> pathlib.Path: + return pathlib.Path(__file__).resolve().parents[2] + + +@functools.lru_cache(maxsize=1) +def _resolve_bash_executable() -> str: + if os.name != "nt": + return "bash" + + candidates: list[str] = [] + for env_var in ("ProgramW6432", "ProgramFiles"): + root = os.environ.get(env_var) + if root: + candidates.append(str(pathlib.Path(root) / "Git" / "bin" / "bash.exe")) + candidates.append(str(pathlib.Path(root) / "Git" / "usr" / "bin" / "bash.exe")) + + which_bash = shutil.which("bash") + if which_bash: + candidates.append(which_bash) + + seen: set[str] = set() + for candidate in candidates: + normalized = str(pathlib.Path(candidate)) + key = normalized.lower() + if key in seen: + continue + seen.add(key) + try: + probe = subprocess.run( + [normalized, "--version"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + except OSError: + continue + if probe.returncode == 0 and "GNU bash" in probe.stdout: + return normalized + + pytest.skip("GNU bash executable is required on Windows for benchmark script tests") + + +def _run( + cmd: list[str], + *, + env: dict[str, str] | None = None, +) -> subprocess.CompletedProcess[str]: + if cmd and cmd[0] == "bash": + cmd = [_resolve_bash_executable(), *cmd[1:]] + merged_env = os.environ.copy() + if env: + merged_env.update(env) + return subprocess.run( + cmd, + cwd=_repo_root(), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=merged_env, + ) + + +def _parse_plan(output: str) -> tuple[dict[str, str], list[str]]: + plan: dict[str, str] = {} + cases: list[str] = [] + + for raw_line in output.splitlines(): + line = raw_line.strip() + if not line: + continue + + if line.startswith("PLAN"): + fields = raw_line.split("\t") + if len(fields) >= 3: + _, key, value = fields[:3] + plan[key.strip()] = value.strip() + continue + _, key, value = line.split(None, 2) + plan[key.strip()] = value.strip() + continue + + if line.startswith("CASE"): + fields = raw_line.split("\t") + if len(fields) >= 2: + _, label = fields[:2] + cases.append(label.strip()) + continue + _, label = line.split(None, 1) + cases.append(label.strip()) + + return plan, cases + + +def _write_compare_summary( + path: pathlib.Path, + *, + suite_spec_path: str, + suite_spec_sha256: str, + build_mode: str = "native", + build_features: str = "cli", +) -> None: + path.write_text( + "\n".join( + [ + "\t".join( + [ + "operation", + "subject", + "size_bytes", + "compression_backend", + "suite_spec_path", + "suite_spec_sha256", + "build_mode", + "build_features", + ] + ), + "\t".join( + [ + "h", + "neural_mixture", + "2097152", + "-", + suite_spec_path, + suite_spec_sha256, + build_mode, + build_features, + ] + ), + ] + ) + + "\n", + encoding="utf-8", + ) + + +def test_cli_bench_plan_default_matrix_and_defaults(): + proc = _run(["bash", "scripts/bench_cli_hyperfine.sh", "--plan", "default"]) + assert proc.returncode == 0, proc.stderr + + plan, cases = _parse_plan(proc.stdout) + + expected_rate_backends = { + "rosaplus", + "ctw", + "fac-ctw", + "match", + "sparse-match", + "ppmd", + "sequitur", + "calibrated", + "mixture", + "particle", + "mamba", + "rwkv7", + } + + assert plan["preset"] == "default" + assert plan["runs"] == "10" + assert plan["warmups"] == "3" + assert plan["bytes"] == "32768" + assert set(plan["rate_backends"].split(",")) == expected_rate_backends + + assert int(plan["cases"]) == 40 + assert int(plan["roundtrip_cases"]) == 14 + assert len(cases) == 40 + assert len(set(cases)) == 40 + + assert sum(name.startswith("h_") for name in cases) == 12 + assert sum(name.startswith("compress_rate_ac_") for name in cases) == 12 + assert sum(name.startswith("decompress_rate_ac_") for name in cases) == 12 + assert sum(name.startswith("compress_rate_rans_") for name in cases) == 2 + assert sum(name.startswith("decompress_rate_rans_") for name in cases) == 2 + + +def test_cli_bench_plan_quick_keeps_full_matrix_with_faster_defaults(): + proc = _run(["bash", "scripts/bench_cli_hyperfine.sh", "--plan", "quick"]) + assert proc.returncode == 0, proc.stderr + + plan, cases = _parse_plan(proc.stdout) + + assert plan["preset"] == "quick" + assert plan["runs"] == "5" + assert plan["warmups"] == "1" + assert plan["bytes"] == "16384" + assert int(plan["cases"]) == 40 + assert int(plan["roundtrip_cases"]) == 14 + assert len(cases) == 40 + + +def test_projman_cli_plan_mode_skips_summary_lookup_error(): + proc = _run( + ["bash", "projman.sh", "bench", "cli", "--plan", "quick"], + env={"INFOTHEORY_BUILD_MODE": "portable"}, + ) + assert proc.returncode == 0, proc.stderr + + plan, _ = _parse_plan(proc.stdout) + assert plan["preset"] == "quick" + assert "no summary.tsv found under /var/tmp/infotheory_bench" not in proc.stderr + + +def test_projman_cli_plan_uses_canonical_build_mode_knob(): + proc = _run( + ["bash", "projman.sh", "bench", "cli", "--plan", "quick"], + env={ + "INFOTHEORY_BUILD_MODE": "portable", + "INFOTHEORY_CLI_BENCH_BUILD_MODE": "invalid", + }, + ) + assert proc.returncode == 0, proc.stderr + + plan, _ = _parse_plan(proc.stdout) + assert plan["preset"] == "quick" + + +def test_summarize_interpret_supports_extended_summary_schema(tmp_path: pathlib.Path): + summary = tmp_path / "summary.tsv" + summary.write_text( + "\n".join( + [ + "label\tbaseline_mean_s\tbaseline_stddev_s\tcurrent_mean_s\tcurrent_stddev_s\tratio_current_over_baseline\tbaseline_n\tcurrent_n\tbaseline_sem_s\tcurrent_sem_s\tdelta_s\tse_delta_s\tt_like\tci95_ratio_low\tci95_ratio_high\tpooled_residual_var_s2\tresidual_bits_gaussian", + "h_match\t0.010000000\t0.001000000\t0.011000000\t0.001200000\t1.100000\t20\t20\t0.000223607\t0.000268328\t0.001000000\t0.000349602\t2.860000\t1.030000\t1.170000\t0.000001220\t-5.891000", + "h_ppmd\t0.020000000\t0.002000000\t0.019000000\t0.001700000\t0.950000\t20\t20\t0.000447214\t0.000380789\t-0.001000000\t0.000587724\t1.701000\t0.900000\t1.010000\t0.000003500\t-4.730000", + ] + ) + + "\n", + encoding="utf-8", + ) + + roundtrip = tmp_path / "roundtrip.tsv" + roundtrip.write_text( + "\n".join( + [ + "label\tsubject\tstatus", + "rate_ac_match\tbaseline\tpass", + "rate_ac_match\tcurrent\tpass", + ] + ) + + "\n", + encoding="utf-8", + ) + + proc = _run(["bash", "scripts/summarize_interpret.sh", str(summary)]) + assert proc.returncode == 0, proc.stderr + + out = proc.stdout + assert "Roundtrip checks: 2 passed / 2 total" in out + assert "se_delta(s)" in out + assert "95% CI(change)" in out + assert "h_match" in out + + +def test_summarize_interpret_supports_legacy_summary_schema(tmp_path: pathlib.Path): + summary = tmp_path / "summary.tsv" + summary.write_text( + "\n".join( + [ + "label\tbaseline_mean_s\tbaseline_stddev_s\tcurrent_mean_s\tcurrent_stddev_s\tratio_current_over_baseline", + "h_ctw\t0.050000000\t0.001000000\t0.051000000\t0.001100000\t1.020000", + ] + ) + + "\n", + encoding="utf-8", + ) + + proc = _run(["bash", "scripts/summarize_interpret.sh", str(summary)]) + assert proc.returncode == 0, proc.stderr + + out = proc.stdout + assert "Cases: 1" in out + assert "h_ctw" in out + assert "Legend:" in out + + +def test_benchmark_two_json_specs_are_kept_in_sync(): + repo = _repo_root() + configs_text = (repo / "configs/bench/two.json").read_text(encoding="utf-8") + examples_text = (repo / "examples/two.json").read_text(encoding="utf-8") + assert configs_text == examples_text + assert json.loads(configs_text)["alpha"] == 0.03 + + +def test_compare_bench_two_json_accepts_matching_suite_spec_digest( + tmp_path: pathlib.Path, +): + if shutil.which("luajit") is None: + pytest.skip("luajit not installed") + + baseline = tmp_path / "baseline.tsv" + candidate = tmp_path / "candidate.tsv" + suite_spec_sha256 = "c" * 64 + _write_compare_summary( + baseline, + suite_spec_path="examples/two.json", + suite_spec_sha256=suite_spec_sha256, + ) + _write_compare_summary( + candidate, + suite_spec_path="configs/bench/two.json", + suite_spec_sha256=suite_spec_sha256, + ) + + proc = _run( + [ + "luajit", + "scripts/compare_bench_two_json.lua", + "--baseline", + str(baseline), + str(candidate), + ] + ) + assert proc.returncode == 0, proc.stderr + assert "baseline_suite_spec_sha256" in proc.stdout + assert "candidate_suite_spec_sha256" in proc.stdout + + +def test_compare_bench_two_json_rejects_mismatched_suite_spec_digest( + tmp_path: pathlib.Path, +): + if shutil.which("luajit") is None: + pytest.skip("luajit not installed") + + baseline = tmp_path / "baseline.tsv" + candidate = tmp_path / "candidate.tsv" + _write_compare_summary( + baseline, + suite_spec_path="examples/two.json", + suite_spec_sha256="a" * 64, + ) + _write_compare_summary( + candidate, + suite_spec_path="configs/bench/two.json", + suite_spec_sha256="b" * 64, + ) + + proc = _run( + [ + "luajit", + "scripts/compare_bench_two_json.lua", + "--baseline", + str(baseline), + str(candidate), + ] + ) + assert proc.returncode != 0 + assert "suite spec digest mismatch" in proc.stderr + + +def test_compare_bench_two_json_explains_duplicate_summary_keys( + tmp_path: pathlib.Path, +): + if shutil.which("luajit") is None: + pytest.skip("luajit not installed") + + baseline = tmp_path / "baseline.tsv" + candidate = tmp_path / "candidate.tsv" + baseline.write_text( + "\n".join( + [ + "\t".join( + [ + "operation", + "subject", + "size_bytes", + "cpu", + "compression_backend", + "suite_spec_path", + "suite_spec_sha256", + "build_mode", + "build_features", + ] + ), + "\t".join( + [ + "h", + "ctw", + "4096", + "0", + "-", + "configs/bench/two.json", + "a" * 64, + "native", + "cli", + ] + ), + "\t".join( + [ + "h", + "ctw", + "4096", + "11", + "-", + "configs/bench/two.json", + "a" * 64, + "native", + "cli", + ] + ), + ] + ) + + "\n", + encoding="utf-8", + ) + _write_compare_summary( + candidate, + suite_spec_path="configs/bench/two.json", + suite_spec_sha256="a" * 64, + ) + + proc = _run( + [ + "luajit", + "scripts/compare_bench_two_json.lua", + "--baseline", + str(baseline), + str(candidate), + ] + ) + assert proc.returncode != 0 + assert proc.stdout == "" + assert "duplicate comparison row" in proc.stderr + assert "operation=h, subject=fac-ctw, size_bytes=4096, compression_backend=-" in proc.stderr + assert "differing columns: cpu: 0 != 11" in proc.stderr + assert "mix CPU affinities" in proc.stderr + + +def test_bench_two_json_build_mode_namespace_is_bench_scoped(): + script_text = (_repo_root() / "scripts/bench_two_json.sh").read_text(encoding="utf-8") + + assert "INFOTHEORY_CLI_BENCH_BUILD_MODE" not in script_text + assert "INFOTHEORY_BENCH_BUILD_MODE" in script_text + assert "CARGO_BUILD_RUSTFLAGS" in script_text + + +def test_bench_two_json_compare_hint_uses_current_baseline_resolver(): + script_text = (_repo_root() / "scripts/bench_two_json.sh").read_text(encoding="utf-8") + + assert "current_two_json_baseline_tsv()" in script_text + assert 'benchmarks/current/infotheory-two-json-summary"*.tsv' in script_text + assert "--baseline '${CURRENT_BASELINE_TSV}'" in script_text + assert "infotheory-two-json-summary-20260322-120428.tsv" not in script_text + + +def test_checked_in_benchmark_summary_tsv_uses_lf_line_endings(): + repo = _repo_root() + crlf = [ + path.relative_to(repo) + for path in (repo / "benchmarks").rglob("*summary*.tsv") + if b"\r" in path.read_bytes() + ] + assert crlf == [], ( + "benchmark summary TSV files must use LF line endings only; " + f"found CR in: {', '.join(str(p) for p in crlf)}" + ) + + +def test_bench_two_json_summary_writer_uses_lf_line_terminator(): + script_text = (_repo_root() / "scripts/bench_two_json.sh").read_text(encoding="utf-8") + marker = 'with open(summary_path, "w", newline="") as fh:' + start = script_text.find(marker) + assert start != -1, "summary TSV writer block not found in bench_two_json.sh" + block = script_text[start : start + 400] + assert 'lineterminator="\\n"' in block, ( + "bench_two_json summary csv.DictWriter must set lineterminator='\\n'" + ) diff --git a/python/tests/test_cli_parity_expanded.py b/python/tests/test_cli_parity_expanded.py index 8194626b..8d0629f0 100644 --- a/python/tests/test_cli_parity_expanded.py +++ b/python/tests/test_cli_parity_expanded.py @@ -28,11 +28,11 @@ def _close(a: float, b: float, tol: float = 1e-6) -> None: def test_metrics_and_cross_entropy_parity(): x = "abracadabra" y = "alakazam" - metrics = _batch({"op": "metrics", "text": x, "max_order": 3}) - _close(metrics["h0"], ait.marginal_entropy_bytes(x.encode())) - _close(metrics["h_rate"], ait.entropy_rate_bytes(x.encode(), 3)) - cross = _batch({"op": "cross_entropy", "text_x": x, "text_y": y, "max_order": 3}) - _close(cross["cross_entropy"], ait.cross_entropy_rate_bytes(x.encode(), y.encode(), 3)) + metrics = _batch({"op": "metrics", "text": x}) + _close(metrics["h0"], ait.empirical_entropy_bytes(x.encode())) + _close(metrics["h_rate"], ait.entropy_rate_bytes(x.encode())) + cross = _batch({"op": "cross_entropy", "text_x": x, "text_y": y}) + _close(cross["cross_entropy"], ait.cross_entropy_rate_bytes(x.encode(), y.encode())) def test_ncd_file_and_matrix_parity(tmp_path): diff --git a/python/tests/test_generation.py b/python/tests/test_generation.py index 94acf14b..0465ab6d 100644 --- a/python/tests/test_generation.py +++ b/python/tests/test_generation.py @@ -1,6 +1,7 @@ import math import infotheory_rs as ait +import pytest PROMPT = ( @@ -61,7 +62,6 @@ def test_generation_session_fill_log_probs_and_reset_frozen(): cfg = ait.GenerationConfig.sampled_frozen(7) session = ait.RateBackendSession( ait.RateBackend.ctw(32), - max_order=32, total_symbols=len(PROMPT) + 8, ) @@ -74,7 +74,6 @@ def test_generation_session_fill_log_probs_and_reset_frozen(): twin = ait.RateBackendSession( ait.RateBackend.ctw(32), - max_order=32, total_symbols=len(PROMPT) + 8, ) twin.observe(PROMPT[:64]) @@ -86,28 +85,86 @@ def test_generation_session_fill_log_probs_and_reset_frozen(): assert first == second +def test_generation_session_begin_stream_restarts_zpaq_without_frozen_reset(): + session = ait.RateBackendSession(ait.RateBackend.zpaq("1"), total_symbols=9) + + with pytest.raises(RuntimeError, match="plugin entropy"): + session.reset_frozen(9) + + session.begin_stream(9) + initial = session.fill_log_probs() + assert _finite_log_probs(initial) + + session.observe(bytes([0, 1, 0, 1, 1, 0, 1, 0, 1])) + session.begin_stream(9) + restarted = session.fill_log_probs() + assert _finite_log_probs(restarted) + + fresh = ait.RateBackendSession(ait.RateBackend.zpaq("1"), total_symbols=9) + expected = fresh.fill_log_probs() + assert _finite_log_probs(expected) + for idx in range(256): + assert abs(restarted[idx] - expected[idx]) < 1e-12 + assert abs(initial[idx] - restarted[idx]) < 1e-12 + + session.finish() + fresh.finish() + + +def test_generation_session_begin_stream_restarts_mixture_with_zpaq_without_frozen_reset(): + backend = ait.RateBackend.mixture( + ait.MixtureSpec( + ait.MixtureKind.Bayes, + [ + ait.MixtureExpertSpec(ait.RateBackend.ctw(6)), + ait.MixtureExpertSpec(ait.RateBackend.zpaq("1")), + ], + ) + ) + session = ait.RateBackendSession(backend, total_symbols=9) + + with pytest.raises(RuntimeError, match="plugin entropy"): + session.reset_frozen(9) + + session.begin_stream(9) + initial = session.fill_log_probs() + assert _finite_log_probs(initial) + + session.observe(bytes([1, 0, 1, 0, 1, 1, 0, 0, 1])) + session.begin_stream(9) + restarted = session.fill_log_probs() + assert _finite_log_probs(restarted) + + fresh = ait.RateBackendSession(backend, total_symbols=9) + expected = fresh.fill_log_probs() + assert _finite_log_probs(expected) + any_changed = any(abs(restarted[idx] - expected[idx]) > 1e-12 for idx in range(256)) + assert any_changed, "mixture+zpaq restart should preserve fitted state from resettable experts" + + session.finish() + fresh.finish() + + def test_generation_is_deterministic_across_core_backends(): cfg = ait.GenerationConfig.sampled_frozen(42) cases = [ - ("ctw", ait.RateBackend.ctw(32), 32), - ("rosaplus", ait.RateBackend.rosaplus(), -1), - ("match", ait.RateBackend.match(), -1), - ("ppmd", ait.RateBackend.ppmd(order=10, memory_mb=8), -1), - ("rwkv7", ait.RateBackend.rwkv7(_rwkv7_cfg_method()), -1), + ("ctw", ait.RateBackend.ctw(32)), + ("rosaplus", ait.RateBackend.rosaplus()), + ("match", ait.RateBackend.match()), + ("ppmd", ait.RateBackend.ppmd(order=10, memory_mb=8)), + ("rwkv7", ait.RateBackend.rwkv7(_rwkv7_cfg_method())), ] - for name, backend, max_order in cases: + for name, backend in cases: first = ait.generate_bytes( PROMPT, 8, - max_order=max_order, backend=backend, config=cfg, ) second = ait.generate_bytes( PROMPT, 8, - max_order=max_order, backend=backend, config=cfg, ) diff --git a/python/tests/test_golden_hashes.py b/python/tests/test_golden_hashes.py index e92276c5..80ce5ed6 100644 --- a/python/tests/test_golden_hashes.py +++ b/python/tests/test_golden_hashes.py @@ -48,7 +48,7 @@ def test_zpaq_roundtrip_and_hash_fixture_b(): def test_batch_metrics_output_hash_stability(): - line = _batch_line({"op": "metrics", "text": "abracadabra", "max_order": 3}) + line = _batch_line({"op": "metrics", "text": "abracadabra"}) assert line == '{"h0":2.040373,"h_rate":1.763318,"id":0.135787,"len":11}' assert ( hashlib.sha256(line.encode("utf-8")).hexdigest() diff --git a/python/tests/test_packaging_features.py b/python/tests/test_packaging_features.py index 3f4d2d31..94b96bb5 100644 --- a/python/tests/test_packaging_features.py +++ b/python/tests/test_packaging_features.py @@ -15,26 +15,24 @@ def test_pyproject_maturin_features_include_mamba(): match = re.search(r"(?m)^\s*features\s*=\s*\[(?P[^\]]+)\]", pyproject) assert match is not None, "missing [tool.maturin].features in pyproject.toml" features = _quoted_tokens(match.group("body")) - assert "backend-mamba" in features + assert "all-backends" in features def test_python_release_wheel_build_features_include_mamba(): workflow = (_repo_root() / ".github/workflows/python-release.yml").read_text() - assert "backend-mamba" in workflow - assert "backend-rwkv" in workflow - assert "backend-zpaq" in workflow + assert "all-backends" in workflow def test_python_ci_explicit_feature_builds_include_mamba(): workflow = (_repo_root() / ".github/workflows/python.yml").read_text() feature_args = re.findall( - r"maturin develop --profile python-release --manifest-path infotheory_py/Cargo.toml --features ([^\n]+)", + r"maturin develop --profile python-release --manifest-path crates/infotheory_py/Cargo.toml(?:\s+--target-dir\s+\S+)? --features ([^\n]+)", workflow, ) assert feature_args, "no explicit maturin develop feature commands found in python.yml" for args in feature_args: features = [part.strip() for part in args.strip().split(",")] - assert "backend-mamba" in features + assert "all-backends" in features def test_python_ci_linux_uses_clang_and_lld_for_python_release_builds(): @@ -44,7 +42,7 @@ def test_python_ci_linux_uses_clang_and_lld_for_python_release_builds(): assert "RUSTFLAGS: -C link-arg=-fuse-ld=lld -C target-cpu=x86-64" in workflow assert 'uv pip install --python "$VENV_PY"' in workflow assert 'VIRTUAL_ENV: .venv' in workflow - assert '"$VENV_PY" -m maturin develop --profile python-release --manifest-path infotheory_py/Cargo.toml' in workflow + assert '"$VENV_PY" -m maturin develop --profile python-release --manifest-path crates/infotheory_py/Cargo.toml' in workflow assert '"$VENV_PY" -m pytest' in workflow @@ -63,7 +61,7 @@ def test_python_release_linux_build_targets_manylinux2014(): def test_python_release_linux_build_overrides_local_linker_and_uses_py310_abi3_base(): workflow = (_repo_root() / ".github/workflows/python-release.yml").read_text() - assert "cargo check --release --manifest-path infotheory_py/Cargo.toml" in workflow + assert "cargo check --release --manifest-path crates/infotheory_py/Cargo.toml" in workflow assert "RUSTFLAGS: -C target-cpu=x86-64" in workflow assert "CC: clang" in workflow assert "CXX: clang++" in workflow @@ -84,7 +82,7 @@ def test_python_release_workflow_avoids_uv_run_project_sync(): def test_infotheory_py_does_not_enable_pyo3_auto_initialize_for_extension_builds(): - cargo_toml = (_repo_root() / "infotheory_py/Cargo.toml").read_text() + cargo_toml = (_repo_root() / "crates/infotheory_py/Cargo.toml").read_text() assert 'features = ["abi3-py310"]' in cargo_toml assert "auto-initialize" not in cargo_toml @@ -96,7 +94,7 @@ def test_pyproject_uses_python_release_profile_for_wheel_and_editable_builds(): def test_zpaq_build_disables_cpp_lto_for_python_extension_builds(): - build_rs = (_repo_root() / "zpaq_rs" / "build.rs").read_text() + build_rs = (_repo_root() / "vendor" / "zpaq_rs" / "build.rs").read_text() assert "fn building_python_extension()" in build_rs assert 'env::var_os("PYO3_BUILD_EXTENSION_MODULE").is_some()' in build_rs assert "!building_python_extension()" in build_rs diff --git a/python/tests/test_predictor_mcts.py b/python/tests/test_predictor_mcts.py index 2bfabf5c..e43371c3 100644 --- a/python/tests/test_predictor_mcts.py +++ b/python/tests/test_predictor_mcts.py @@ -1,6 +1,26 @@ +import pytest + import infotheory_rs as ait +def _agent_config(mcts_strategy=None): + kwargs = { + "rate_backend": ait.RateBackend.ctw(8), + "agent_horizon": 2, + "observation_bits": 1, + "observation_stream_len": 1, + "reward_bits": 1, + "agent_actions": 2, + "num_simulations": 4, + "min_reward": 0, + "max_reward": 1, + "reward_offset": 0, + } + if mcts_strategy is not None: + kwargs["mcts_strategy"] = mcts_strategy + return ait.AgentConfig(**kwargs) + + def test_predictor_wrappers_smoke(): p = ait.CtwPredictor(8) q = p.predict_one() @@ -10,21 +30,52 @@ def test_predictor_wrappers_smoke(): assert isinstance(p.model_name(), str) -def test_search_tree_smoke(): - cfg = ait.AgentConfig( - algorithm="fac-ctw", - ct_depth=8, - agent_horizon=2, - observation_bits=1, - observation_stream_len=1, - reward_bits=1, - agent_actions=2, - num_simulations=4, - min_reward=0, - max_reward=1, - reward_offset=0, - ) - agent = ait.Agent(cfg) +def test_mcts_strategy_python_surface(): + rho_uct = ait.MctsStrategy.rho_uct() + assert rho_uct.kind == "rho_uct" + assert rho_uct.workers is None + assert rho_uct.bu_uct_m_max is None + + parallel_wu = ait.MctsStrategy.parallel_uct(4) + assert parallel_wu.kind == "parallel_uct" + assert parallel_wu.workers == 4 + assert parallel_wu.bu_uct_m_max is None + + parallel_bu = ait.MctsStrategy.parallel_uct(4, 0.8) + assert parallel_bu.kind == "parallel_uct" + assert parallel_bu.workers == 4 + assert parallel_bu.bu_uct_m_max == pytest.approx(0.8) + + +def test_mcts_strategy_parallel_uct_rejects_invalid_parameters(): + with pytest.raises(ValueError, match="workers"): + ait.MctsStrategy.parallel_uct(0) + with pytest.raises(ValueError, match="bu_uct_m_max"): + ait.MctsStrategy.parallel_uct(2, 1.0) + + +@pytest.mark.parametrize( + "mcts_strategy", + [ + None, + ait.MctsStrategy.rho_uct(), + ait.MctsStrategy.parallel_uct(2), + ait.MctsStrategy.parallel_uct(2, 0.8), + ], +) +def test_search_tree_smoke(mcts_strategy): + agent = ait.Agent(_agent_config(mcts_strategy)) tree = ait.SearchTree() + if mcts_strategy is not None: + tree = ait.SearchTree(mcts_strategy) + assert tree.mcts_strategy.kind == mcts_strategy.kind + else: + assert tree.mcts_strategy.kind == "rho_uct" a = tree.search(agent, [0], 0, 0, 4) assert a in (0, 1) + + +def test_search_node_is_not_exported(): + assert not hasattr(ait, "SearchNode") + with pytest.raises(AttributeError, match="SearchNode"): + getattr(ait, "SearchNode") diff --git a/python/tests/test_smoke.py b/python/tests/test_smoke.py index 00118a5f..3ea8e2d6 100644 --- a/python/tests/test_smoke.py +++ b/python/tests/test_smoke.py @@ -1,12 +1,12 @@ import infotheory_rs as ait -def test_import_and_core_metrics(): +def test_smoke_import_and_basic_calls(): assert isinstance(ait.vm_enabled(), bool) x = b"abracadabra" y = b"alakazam" - assert ait.marginal_entropy_bytes(x) >= 0.0 - assert ait.mutual_information_bytes(x, y, 0) >= 0.0 + assert ait.empirical_entropy_bytes(x) >= 0.0 + assert ait.mutual_information_bytes(x, y) >= 0.0 def test_backend_and_ctx_usage(): diff --git a/python/tests/test_trait_adapters.py b/python/tests/test_trait_adapters.py index f809a63e..f46dbab8 100644 --- a/python/tests/test_trait_adapters.py +++ b/python/tests/test_trait_adapters.py @@ -1,6 +1,7 @@ import infotheory_rs as ait import os import pathlib +import pytest import subprocess import sys @@ -104,6 +105,11 @@ def observation_key_mode(self): return self._mode +class ZeroHorizonSim(DummySim): + def horizon(self) -> int: + return 0 + + class RunnerTupleEnv(ait.EnvironmentABC): def __init__(self): self.obs = 0 @@ -142,8 +148,7 @@ def get_action_bits(self) -> int: def _test_agent_config() -> ait.AgentConfig: return ait.AgentConfig( - algorithm="ac-ctw", - ct_depth=8, + rate_backend=ait.RateBackend.ctw(8), agent_horizon=2, observation_bits=1, observation_stream_len=1, @@ -205,8 +210,8 @@ def test_search_with_simulator_adapter(): assert action in (0, 1) -def test_search_with_simulator_accepts_cli_observation_key_aliases(): - for mode in ("full", "full-stream", "stream-hash"): +def test_search_with_simulator_accepts_canonical_observation_key_names(): + for mode in ("full_stream", "stream_hash"): action = ait.search_with_simulator(DummySimWithKeyMode(mode), [0], 0, 0, 4) assert action in (0, 1) @@ -215,7 +220,7 @@ def _repo_root() -> pathlib.Path: return pathlib.Path(__file__).resolve().parents[2] -def test_search_with_simulator_respects_rayon_num_threads_env(): +def test_search_with_simulator_defaults_to_sequential_rho_uct(): code = """ import infotheory_rs as ait class ProbeSim(ait.AgentSimulatorABC): @@ -255,9 +260,68 @@ def boxed_clone_with_seed(self, seed: int): text=True, ) assert proc.returncode == 0, proc.stderr + assert int(proc.stdout.strip()) == 0 + + +def test_search_with_simulator_parallel_uct_uses_seeded_clones(): + code = """ +import infotheory_rs as ait +class ProbeSim(ait.AgentSimulatorABC): + def __init__(self): + self._rng = ait.RandomGenerator() + self._obs = 0 + self.seed_clone_calls = 0 + def get_num_actions(self) -> int: return 2 + def get_num_observation_bits(self) -> int: return 1 + def get_num_reward_bits(self) -> int: return 1 + def horizon(self) -> int: return 2 + def max_reward(self) -> int: return 1 + def min_reward(self) -> int: return 0 + def model_update_action(self, action: int): self._obs = action & 1 + def gen_percept_and_update(self, bits: int) -> int: return self._obs if bits == 1 else 0 + def model_revert(self, steps: int): return None + def gen_range(self, end: int) -> int: return self._rng.gen_range(end) + def gen_f64(self) -> float: return self._rng.gen_f64() + def boxed_clone_with_seed(self, seed: int): + self.seed_clone_calls += 1 + c = ProbeSim() + c._rng = self._rng.fork_with(seed) + c._obs = self._obs + return c +sim = ProbeSim() +ait.search_with_simulator( + sim, + [0], + 0, + 0, + 8, + mcts_strategy=ait.MctsStrategy.parallel_uct(2), +) +print(sim.seed_clone_calls) +""" + proc = subprocess.run( + [sys.executable, "-c", code], + cwd=_repo_root(), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert proc.returncode == 0, proc.stderr assert int(proc.stdout.strip()) >= 1 +def test_search_with_simulator_parallel_uct_rejects_zero_horizon(): + with pytest.raises(ValueError, match=r"parallel_uct requires agent.horizon\(\) >= 1"): + ait.search_with_simulator( + ZeroHorizonSim(), + [0], + 0, + 0, + 1, + mcts_strategy=ait.MctsStrategy.parallel_uct(2), + ) + + def test_predictor_callback_exception_is_fatal(): code = """ import infotheory_rs as ait diff --git a/scripts/bench_aiqi_vs_aixi.sh b/scripts/bench_aiqi_vs_aixi.sh index 15e63ee5..20b938e1 100755 --- a/scripts/bench_aiqi_vs_aixi.sh +++ b/scripts/bench_aiqi_vs_aixi.sh @@ -164,8 +164,7 @@ def run_mcaixi(args): min_reward = 0 max_reward = 1 cfg = ait.AgentConfig( - algorithm="ac-ctw", - ct_depth=4, + rate_backend=ait.RateBackend.ctw(4), agent_horizon=6, observation_bits=observation_bits, observation_stream_len=1, @@ -185,8 +184,7 @@ def run_mcaixi(args): min_reward = -2 max_reward = 4 cfg = ait.AgentConfig( - algorithm="ac-ctw", - ct_depth=42, + rate_backend=ait.RateBackend.ctw(42), agent_horizon=2, observation_bits=observation_bits, observation_stream_len=1, diff --git a/scripts/bench_aixi_competitors_guix.sh b/scripts/bench_aixi_competitors_guix.sh index 87cb6d20..3f6a9f60 100755 --- a/scripts/bench_aixi_competitors_guix.sh +++ b/scripts/bench_aixi_competitors_guix.sh @@ -212,7 +212,7 @@ export PATH="$bench_venv/bin:$PATH" export PYO3_PYTHON="$bench_python" ( - cd "$repo_root/infotheory_py" + cd "$repo_root/crates/infotheory_py" "$bench_python" -m maturin develop --release --no-default-features --features "python-extension" --pip-path "$bench_venv/bin/pip" ) @@ -235,7 +235,7 @@ INFOTHEORY_BIN="$CARGO_TARGET_DIR/release/infotheory" INNER chmod +x "$inner_script" -echo "[bench__aixi_competitors] Running inside Guix time-machine..." +echo "[bench_aixi_competitors] Running inside Guix time-machine..." "$guix_bin" time-machine -C "$guix_channels" -- shell --pure --container --network --no-cwd \ --share="$repo_root=$repo_root" \ bash coreutils findutils grep sed gawk git make gcc-toolchain \ diff --git a/scripts/bench_aixi_competitors_runner.py b/scripts/bench_aixi_competitors_runner.py index ac74bf40..d12d82f8 100755 --- a/scripts/bench_aixi_competitors_runner.py +++ b/scripts/bench_aixi_competitors_runner.py @@ -242,11 +242,18 @@ def main() -> None: p.add_argument("--exploration-exploitation-ratio", type=float, required=True) args = p.parse_args() + if args.algorithm == "ac-ctw": + rate_backend = ait.RateBackend.ctw(args.ct_depth) + elif args.algorithm == "fac-ctw": + num_percept_bits = 2 if args.workload == "coinflip" else 6 + rate_backend = ait.RateBackend.fac_ctw(args.ct_depth, num_percept_bits, 8) + else: + raise ValueError(f"unsupported algorithm {args.algorithm}") + if args.workload == "coinflip": env = ait.CoinFlipEnv(args.coin_flip_p, args.seed) cfg = ait.AgentConfig( - algorithm=args.algorithm, - ct_depth=args.ct_depth, + rate_backend=rate_backend, agent_horizon=args.horizon, observation_bits=1, observation_stream_len=1, @@ -263,8 +270,7 @@ def main() -> None: else: env = ait.KuhnPokerEnv(args.seed) cfg = ait.AgentConfig( - algorithm=args.algorithm, - ct_depth=args.ct_depth, + rate_backend=rate_backend, agent_horizon=args.horizon, observation_bits=3, observation_stream_len=1, @@ -649,7 +655,12 @@ def main() -> None: "cycles_per_sec", ] with raw_tsv.open("w", encoding="utf-8", newline="") as f: - writer = csv.DictWriter(f, fieldnames=raw_fields, delimiter="\t") + writer = csv.DictWriter( + f, + fieldnames=raw_fields, + delimiter="\t", + lineterminator="\n", + ) writer.writeheader() writer.writerows(rows) @@ -722,7 +733,12 @@ def mean_std(name: str) -> Tuple[float, float]: "cycles_per_sec_std", ] with summary_tsv.open("w", encoding="utf-8", newline="") as f: - writer = csv.DictWriter(f, fieldnames=summary_fields, delimiter="\t") + writer = csv.DictWriter( + f, + fieldnames=summary_fields, + delimiter="\t", + lineterminator="\n", + ) writer.writeheader() writer.writerows(summary_rows) diff --git a/scripts/bench_aixi_vs_pyaixi.sh b/scripts/bench_aixi_vs_pyaixi.sh index 54a84fd5..c0574f38 100755 --- a/scripts/bench_aixi_vs_pyaixi.sh +++ b/scripts/bench_aixi_vs_pyaixi.sh @@ -249,9 +249,16 @@ def main(): max_reward = max(raw_rewards) reward_offset = max(0, -min_reward) + if args.algorithm == "ac-ctw": + rate_backend = ait.RateBackend.ctw(args.ct_depth) + elif args.algorithm == "fac-ctw": + percept_bits = adapter.get_observation_bits() + adapter.get_reward_bits() + rate_backend = ait.RateBackend.fac_ctw(args.ct_depth, percept_bits, 8) + else: + raise ValueError(f"unsupported algorithm {args.algorithm}") + cfg = ait.AgentConfig( - algorithm=args.algorithm, - ct_depth=args.ct_depth, + rate_backend=rate_backend, agent_horizon=args.horizon, observation_bits=adapter.get_observation_bits(), observation_stream_len=1, diff --git a/scripts/bench_cli_hyperfine.sh b/scripts/bench_cli_hyperfine.sh new file mode 100644 index 00000000..82cf4735 --- /dev/null +++ b/scripts/bench_cli_hyperfine.sh @@ -0,0 +1,818 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat <<'EOF' +Usage: ./scripts/bench_cli_hyperfine.sh [preset] + ./scripts/bench_cli_hyperfine.sh --plan [preset] + +Builds a detached baseline tree and a snapshot of the current dirty working tree, +then compares CLI workloads with hyperfine. + +The benchmark runs two full passes to reduce order effects: + 1) baseline then current + 2) current then baseline + +Environment: + INFOTHEORY_CLI_BENCH_ROOT=/var/tmp/infotheory_bench + INFOTHEORY_CLI_BENCH_PROFILE=release + INFOTHEORY_CLI_BENCH_RUNS= + INFOTHEORY_CLI_BENCH_WARMUPS= + INFOTHEORY_CLI_BENCH_BYTES= + INFOTHEORY_CLI_BENCH_BUILD_MODE=native|portable + INFOTHEORY_CLI_BENCH_CARGO_FEATURES=cli + INFOTHEORY_CLI_BENCH_BASELINE_FEATURES=... + INFOTHEORY_CLI_BENCH_CURRENT_FEATURES=... + INFOTHEORY_CLI_BENCH_NO_DEFAULT_FEATURES=0|1 + +Presets: + default Full matrix, tuned for signal quality (runs=10, warmups=3, bytes=32768) + quick Full matrix with faster defaults (runs=5, warmups=1, bytes=16384) + +Modes: + --plan Print the expanded workload plan and effective tuning without running builds +EOF +} + +need_cmd() { + command -v "$1" >/dev/null 2>&1 || { + echo "missing required command: $1" >&2 + exit 1 + } +} + +fail() { + echo "error: $*" >&2 + exit 1 +} + +sanitize_name() { + printf '%s' "$1" | tr -c 'A-Za-z0-9_.-' '_' +} + +join_by() { + local sep="$1" + shift || true + local out="" + local item + for item in "$@"; do + if [[ -n "${out}" ]]; then + out+="${sep}" + fi + out+="${item}" + done + printf '%s' "${out}" +} + +require_non_negative_int() { + local name="$1" + local value="$2" + if [[ ! "${value}" =~ ^[0-9]+$ ]]; then + fail "${name} must be a non-negative integer (got '${value}')" + fi +} + +require_positive_int() { + local name="$1" + local value="$2" + require_non_negative_int "${name}" "${value}" + if [[ "${value}" -le 0 ]]; then + fail "${name} must be greater than zero (got '${value}')" + fi +} + +build_rustflags() { + local cpu_flag + case "${INFOTHEORY_CLI_BENCH_BUILD_MODE:-native}" in + native) cpu_flag="-C target-cpu=native" ;; + portable) cpu_flag="-C target-cpu=generic" ;; + *) + echo "invalid INFOTHEORY_CLI_BENCH_BUILD_MODE: ${INFOTHEORY_CLI_BENCH_BUILD_MODE}" >&2 + exit 1 + ;; + esac + + case "$(uname -s)" in + Linux|FreeBSD|OpenBSD) printf '%s\n' "${cpu_flag} -C link-arg=-fuse-ld=lld" ;; + *) printf '%s\n' "${cpu_flag}" ;; + esac +} + +resolve_bench_tuning() { + local preset="$1" + local preset_runs + local preset_warmups + local preset_bytes + + case "${preset}" in + default) + preset_runs=10 + preset_warmups=3 + preset_bytes=32768 + ;; + quick) + preset_runs=5 + preset_warmups=1 + preset_bytes=16384 + ;; + *) + fail "unknown preset '${preset}' (expected default or quick)" + ;; + esac + + BENCH_RUNS="${INFOTHEORY_CLI_BENCH_RUNS:-${preset_runs}}" + BENCH_WARMUPS="${INFOTHEORY_CLI_BENCH_WARMUPS:-${preset_warmups}}" + BENCH_BYTES="${INFOTHEORY_CLI_BENCH_BYTES:-${preset_bytes}}" + + require_positive_int "INFOTHEORY_CLI_BENCH_RUNS" "${BENCH_RUNS}" + require_non_negative_int "INFOTHEORY_CLI_BENCH_WARMUPS" "${BENCH_WARMUPS}" + require_positive_int "INFOTHEORY_CLI_BENCH_BYTES" "${BENCH_BYTES}" +} + +prepare_input() { + local repo_root="$1" + local bytes="$2" + local out_dir="$3" + local out_path="${out_dir}/portable_${bytes}.bin" + local bench_two + bench_two="$(bench_config_path "${repo_root}" "two.json")" + mkdir -p "${out_dir}" + : > "${out_path}" + while [[ "$(wc -c < "${out_path}")" -lt "${bytes}" ]]; do + cat \ + "${repo_root}/LICENSE-APACHE" \ + "${repo_root}/benchmarks/baseline/infotheory-two-json-summary-20260310-212017.tsv" \ + "${bench_two}" >> "${out_path}" + done + truncate -s "${bytes}" "${out_path}" + printf '%s\n' "${out_path}" +} + +write_calibrated_spec() { + local out_path="$1" + mkdir -p "$(dirname "${out_path}")" + cat >"${out_path}" <<'EOF' +{ + "kind": "calibrated", + "context": "text", + "bins": 33, + "learning_rate": 0.02, + "bias_clip": 4.0, + "base": { + "kind": "match" + } +} +EOF +} + +bench_config_path() { + local repo_root="$1" + local name="$2" + if [[ -f "${repo_root}/configs/bench/${name}" ]]; then + printf '%s\n' "${repo_root}/configs/bench/${name}" + return 0 + fi + if [[ -f "${repo_root}/examples/${name}" ]]; then + printf '%s\n' "${repo_root}/examples/${name}" + return 0 + fi + printf '%s\n' "${SCRIPT_REPO_ROOT}/configs/bench/${name}" +} + +copy_dirty_tree() { + local src_root="$1" + local dst_root="$2" + local bench_root="$3" + mkdir -p "${dst_root}" + local exclude_args=( + --exclude=.git + --exclude=target + --exclude=.cargo-target-* + --exclude=.tmp + --exclude=.bench-runs + ) + if [[ "${bench_root}" == "${src_root}"/* ]]; then + exclude_args+=(--exclude="${bench_root#${src_root}/}") + fi + tar -C "${src_root}" -cf - "${exclude_args[@]}" . | tar -C "${dst_root}" -xf - +} + +build_infotheory_bin() { + local src_root="$1" + local build_root="$2" + local rustflags="$3" + local features="$4" + local profile="${INFOTHEORY_CLI_BENCH_PROFILE:-release}" + local no_default=() + if [[ "${INFOTHEORY_CLI_BENCH_NO_DEFAULT_FEATURES:-0}" == "1" ]]; then + no_default+=(--no-default-features) + fi + + mkdir -p "${build_root}" + ( + cd "${src_root}" + export TMPDIR="${build_root}/tmp" + mkdir -p "${TMPDIR}" + export CARGO_TARGET_DIR="${build_root}/target" + export RUSTFLAGS="${rustflags}" + cargo build --locked -p infotheory --bin infotheory --profile "${profile}" "${no_default[@]}" --features "${features}" + ) + printf '%s\n' "${build_root}/target/${profile}/infotheory" +} + +format_command() { + local quoted=() + local token + for token in "$@"; do + printf -v token '%q' "${token}" + quoted+=("${token}") + done + (IFS=' '; printf '%s' "${quoted[*]}") +} + +run_command_checked() { + local label="$1" + local stage="$2" + local command_line="$3" + local log_dir="$4" + local log_path + + log_path="${log_dir}/$(sanitize_name "${label}_${stage}").log" + if ! bash -c "${command_line}" >"${log_path}" 2>&1; then + echo "error: ${stage} failed for case '${label}'" >&2 + echo "command: ${command_line}" >&2 + if [[ -s "${log_path}" ]]; then + sed -n '1,120p' "${log_path}" >&2 + fi + exit 1 + fi +} + +run_hyperfine_case() { + local out_dir="$1" + local label="$2" + local order="$3" + local first_name="$4" + local second_name="$5" + local first_cmd="$6" + local second_cmd="$7" + local runs="$8" + local warmups="$9" + local log_dir="${10}" + local json_path="${out_dir}/${label}__${order}.json" + local log_path + + log_path="${log_dir}/$(sanitize_name "hyperfine_${label}_${order}").log" + if ! hyperfine \ + --shell=none \ + --style none \ + --warmup "${warmups}" \ + --runs "${runs}" \ + --export-json "${json_path}" \ + --command-name "${first_name}" \ + --command-name "${second_name}" \ + "${first_cmd}" \ + "${second_cmd}" >"${log_path}" 2>&1; then + echo "error: hyperfine failed for case '${label}' (order: ${order})" >&2 + echo "first (${first_name}): ${first_cmd}" >&2 + echo "second (${second_name}): ${second_cmd}" >&2 + if [[ -s "${log_path}" ]]; then + sed -n '1,120p' "${log_path}" >&2 + fi + exit 1 + fi + + [[ -s "${json_path}" ]] || fail "missing hyperfine JSON output for case '${label}' (${order})" +} + +append_summary_rows() { + local cases_dir="$1" + local summary_tsv="$2" + python3 - "$cases_dir" "$summary_tsv" <<'PY' +import json +import math +import pathlib +import sys + +cases_dir = pathlib.Path(sys.argv[1]) +summary_tsv = pathlib.Path(sys.argv[2]) + + +def stats(values: list[float]) -> tuple[int, float, float, float, float]: + n = len(values) + if n == 0: + return 0, math.nan, math.nan, math.nan, math.nan + mean = sum(values) / n + if n >= 2: + var = sum((x - mean) ** 2 for x in values) / (n - 1) + else: + var = 0.0 + std = math.sqrt(var) + sem = std / math.sqrt(n) + return n, mean, std, sem, var + + +def fmt(value: float, digits: int = 9) -> str: + if math.isfinite(value): + return f"{value:.{digits}f}" + if value > 0: + return "inf" + if value < 0: + return "-inf" + return "nan" + + +samples: dict[str, dict[str, list[float]]] = {} + +for path in sorted(cases_dir.glob("*.json")): + data = json.loads(path.read_text(encoding="utf-8")) + label = path.stem.split("__", 1)[0] + bucket = samples.setdefault(label, {"baseline": [], "current": []}) + for result in data.get("results", []): + command_name = str(result.get("command", "")).strip().lower() + if command_name not in bucket: + continue + times = result.get("times") + if isinstance(times, list) and times: + for value in times: + try: + bucket[command_name].append(float(value)) + except (TypeError, ValueError): + continue + continue + try: + bucket[command_name].append(float(result["mean"])) + except (KeyError, TypeError, ValueError): + continue + +if not samples: + raise SystemExit("no hyperfine case JSON files found") + +with summary_tsv.open("w", encoding="utf-8") as f: + f.write( + "label\tbaseline_mean_s\tbaseline_stddev_s\tcurrent_mean_s\tcurrent_stddev_s" + "\tratio_current_over_baseline\tbaseline_n\tcurrent_n\tbaseline_sem_s" + "\tcurrent_sem_s\tdelta_s\tse_delta_s\tt_like\tci95_ratio_low" + "\tci95_ratio_high\tpooled_residual_var_s2\tresidual_bits_gaussian\n" + ) + + for label in sorted(samples): + base_vals = samples[label]["baseline"] + current_vals = samples[label]["current"] + if not base_vals or not current_vals: + raise SystemExit(f"missing baseline/current timing samples for case '{label}'") + + b_n, b_mean, b_std, b_sem, _ = stats(base_vals) + c_n, c_mean, c_std, c_sem, _ = stats(current_vals) + + ratio = math.inf if b_mean == 0 else c_mean / b_mean + delta = c_mean - b_mean + se_delta = math.sqrt((b_sem * b_sem) + (c_sem * c_sem)) + t_like = math.inf if se_delta == 0 else abs(delta) / se_delta + + if ( + b_mean > 0 + and c_mean > 0 + and math.isfinite(b_sem) + and math.isfinite(c_sem) + ): + log_ratio = math.log(c_mean / b_mean) + se_log_ratio = math.sqrt((b_sem / b_mean) ** 2 + (c_sem / c_mean) ** 2) + ci95_low = math.exp(log_ratio - 1.96 * se_log_ratio) + ci95_high = math.exp(log_ratio + 1.96 * se_log_ratio) + else: + ci95_low = math.nan + ci95_high = math.nan + + pooled_denom = (b_n - 1) + (c_n - 1) + pooled_var = ( + (sum((x - b_mean) ** 2 for x in base_vals) + sum((x - c_mean) ** 2 for x in current_vals)) + / pooled_denom + if pooled_denom > 0 + else 0.0 + ) + if pooled_var > 0: + residual_bits = 0.5 * math.log2(2.0 * math.pi * math.e * pooled_var) + else: + residual_bits = float("-inf") + + f.write( + "\t".join( + [ + label, + fmt(b_mean), + fmt(b_std), + fmt(c_mean), + fmt(c_std), + fmt(ratio, 6), + str(b_n), + str(c_n), + fmt(b_sem), + fmt(c_sem), + fmt(delta), + fmt(se_delta), + fmt(t_like, 6), + fmt(ci95_low, 6), + fmt(ci95_high, 6), + fmt(pooled_var), + fmt(residual_bits, 6), + ] + ) + + "\n" + ) +PY +} + +add_case() { + CASE_LABELS+=("$1") + CASE_BASELINE_COMMANDS+=("$2") + CASE_CURRENT_COMMANDS+=("$3") +} + +add_roundtrip_case() { + ROUNDTRIP_LABELS+=("$1") + ROUNDTRIP_INPUT_PATHS+=("$2") + ROUNDTRIP_BASELINE_COMPRESS_COMMANDS+=("$3") + ROUNDTRIP_CURRENT_COMPRESS_COMMANDS+=("$4") + ROUNDTRIP_BASELINE_DECOMPRESS_COMMANDS+=("$5") + ROUNDTRIP_CURRENT_DECOMPRESS_COMMANDS+=("$6") + ROUNDTRIP_BASELINE_VERIFY_OUTPUTS+=("$7") + ROUNDTRIP_CURRENT_VERIFY_OUTPUTS+=("$8") +} + +add_rate_spec() { + RATE_SPEC_LABELS+=("$1") + RATE_SPEC_BACKENDS+=("$2") + RATE_SPEC_METHODS+=("$3") +} + +build_rate_backend_specs() { + local run_root="$1" + local rwkv_cfg="cfg:hidden=64,layers=1,intermediate=64,decay_rank=16,a_rank=16,v_rank=16,g_rank=16,seed=22,train=adam,lr=0.0009,stride=1;policy:schedule=0..100:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=1,clip=1.0,momentum=0.9)" + local mamba_cfg="cfg:hidden=64,layers=1,intermediate=128,state=16,conv=4,dt_rank=16,seed=26,train=adam,lr=0.001,stride=1;policy:schedule=0..100:train(scope=all,opt=adam,lr=0.001,stride=1,bptt=1,clip=0,momentum=0.9)" + local two_json + local particle_fast_json + local calibrated_json + + two_json="$(bench_config_path "${SCRIPT_REPO_ROOT}" "two.json")" + particle_fast_json="$(bench_config_path "${SCRIPT_REPO_ROOT}" "particle_fast.json")" + calibrated_json="${run_root}/specs/calibrated_match.json" + write_calibrated_spec "${calibrated_json}" + + RATE_SPEC_LABELS=() + RATE_SPEC_BACKENDS=() + RATE_SPEC_METHODS=() + + add_rate_spec "rosaplus" "rosaplus" "" + add_rate_spec "ctw" "ctw" "32" + add_rate_spec "fac_ctw" "fac-ctw" "16" + add_rate_spec "match" "match" "" + add_rate_spec "sparse_match" "sparse-match" "" + add_rate_spec "ppmd" "ppmd" "10" + add_rate_spec "sequitur" "sequitur" "64" + add_rate_spec "calibrated" "calibrated" "${calibrated_json}" + add_rate_spec "mixture" "mixture" "${two_json}" + add_rate_spec "particle" "particle" "${particle_fast_json}" + add_rate_spec "mamba_cfg" "mamba" "${mamba_cfg}" + add_rate_spec "rwkv7_cfg" "rwkv7" "${rwkv_cfg}" +} + +build_rate_backend_flags() { + local idx="$1" + RATE_BACKEND_FLAGS=(--rate-backend "${RATE_SPEC_BACKENDS[$idx]}") + if [[ -n "${RATE_SPEC_METHODS[$idx]}" ]]; then + RATE_BACKEND_FLAGS+=(--method "${RATE_SPEC_METHODS[$idx]}") + fi +} + +find_rate_spec_index() { + local label="$1" + local i + for ((i = 0; i < ${#RATE_SPEC_LABELS[@]}; i++)); do + if [[ "${RATE_SPEC_LABELS[$i]}" == "${label}" ]]; then + printf '%s\n' "${i}" + return 0 + fi + done + return 1 +} + +build_cases() { + local baseline_bin="$1" + local current_bin="$2" + local input_path="$3" + local run_root="$4" + local i + + mkdir -p \ + "${run_root}/prepared/baseline" \ + "${run_root}/prepared/current" \ + "${run_root}/bench-outputs/baseline" \ + "${run_root}/bench-outputs/current" + + build_rate_backend_specs "${run_root}" + + CASE_LABELS=() + CASE_BASELINE_COMMANDS=() + CASE_CURRENT_COMMANDS=() + ROUNDTRIP_LABELS=() + ROUNDTRIP_INPUT_PATHS=() + ROUNDTRIP_BASELINE_COMPRESS_COMMANDS=() + ROUNDTRIP_CURRENT_COMPRESS_COMMANDS=() + ROUNDTRIP_BASELINE_DECOMPRESS_COMMANDS=() + ROUNDTRIP_CURRENT_DECOMPRESS_COMMANDS=() + ROUNDTRIP_BASELINE_VERIFY_OUTPUTS=() + ROUNDTRIP_CURRENT_VERIFY_OUTPUTS=() + + for ((i = 0; i < ${#RATE_SPEC_LABELS[@]}; i++)); do + local rate_label="${RATE_SPEC_LABELS[$i]}" + build_rate_backend_flags "${i}" + + local baseline_h_cmd + local current_h_cmd + baseline_h_cmd="$(format_command "${baseline_bin}" h "${input_path}" --compression-backend rate-ac "${RATE_BACKEND_FLAGS[@]}")" + current_h_cmd="$(format_command "${current_bin}" h "${input_path}" --compression-backend rate-ac "${RATE_BACKEND_FLAGS[@]}")" + add_case "h_${rate_label}" "${baseline_h_cmd}" "${current_h_cmd}" + done + + for ((i = 0; i < ${#RATE_SPEC_LABELS[@]}; i++)); do + local rate_label="${RATE_SPEC_LABELS[$i]}" + build_rate_backend_flags "${i}" + + local prepared_baseline_comp="${run_root}/prepared/baseline/${rate_label}_ac.itc" + local prepared_current_comp="${run_root}/prepared/current/${rate_label}_ac.itc" + local verify_baseline_out="${run_root}/prepared/baseline/${rate_label}_ac.roundtrip.bin" + local verify_current_out="${run_root}/prepared/current/${rate_label}_ac.roundtrip.bin" + + local prep_baseline_compress + local prep_current_compress + local prep_baseline_decompress + local prep_current_decompress + + prep_baseline_compress="$(format_command "${baseline_bin}" compress "${input_path}" "${prepared_baseline_comp}" --compression-backend rate-ac "${RATE_BACKEND_FLAGS[@]}")" + prep_current_compress="$(format_command "${current_bin}" compress "${input_path}" "${prepared_current_comp}" --compression-backend rate-ac "${RATE_BACKEND_FLAGS[@]}")" + prep_baseline_decompress="$(format_command "${baseline_bin}" decompress "${prepared_baseline_comp}" "${verify_baseline_out}" --compression-backend rate-ac "${RATE_BACKEND_FLAGS[@]}")" + prep_current_decompress="$(format_command "${current_bin}" decompress "${prepared_current_comp}" "${verify_current_out}" --compression-backend rate-ac "${RATE_BACKEND_FLAGS[@]}")" + + add_roundtrip_case \ + "rate_ac_${rate_label}" \ + "${input_path}" \ + "${prep_baseline_compress}" \ + "${prep_current_compress}" \ + "${prep_baseline_decompress}" \ + "${prep_current_decompress}" \ + "${verify_baseline_out}" \ + "${verify_current_out}" + + local bench_baseline_comp="${run_root}/bench-outputs/baseline/${rate_label}_ac.itc" + local bench_current_comp="${run_root}/bench-outputs/current/${rate_label}_ac.itc" + local bench_baseline_dec="${run_root}/bench-outputs/baseline/${rate_label}_ac.decompressed.bin" + local bench_current_dec="${run_root}/bench-outputs/current/${rate_label}_ac.decompressed.bin" + + add_case \ + "compress_rate_ac_${rate_label}" \ + "$(format_command "${baseline_bin}" compress "${input_path}" "${bench_baseline_comp}" --compression-backend rate-ac "${RATE_BACKEND_FLAGS[@]}")" \ + "$(format_command "${current_bin}" compress "${input_path}" "${bench_current_comp}" --compression-backend rate-ac "${RATE_BACKEND_FLAGS[@]}")" + + add_case \ + "decompress_rate_ac_${rate_label}" \ + "$(format_command "${baseline_bin}" decompress "${prepared_baseline_comp}" "${bench_baseline_dec}" --compression-backend rate-ac "${RATE_BACKEND_FLAGS[@]}")" \ + "$(format_command "${current_bin}" decompress "${prepared_current_comp}" "${bench_current_dec}" --compression-backend rate-ac "${RATE_BACKEND_FLAGS[@]}")" + done + + local rans_label + for rans_label in ctw ppmd; do + local idx + idx="$(find_rate_spec_index "${rans_label}")" || fail "missing rate spec label '${rans_label}'" + build_rate_backend_flags "${idx}" + + local prepared_baseline_comp="${run_root}/prepared/baseline/${rans_label}_rans.itc" + local prepared_current_comp="${run_root}/prepared/current/${rans_label}_rans.itc" + local verify_baseline_out="${run_root}/prepared/baseline/${rans_label}_rans.roundtrip.bin" + local verify_current_out="${run_root}/prepared/current/${rans_label}_rans.roundtrip.bin" + + local prep_baseline_compress + local prep_current_compress + local prep_baseline_decompress + local prep_current_decompress + + prep_baseline_compress="$(format_command "${baseline_bin}" compress "${input_path}" "${prepared_baseline_comp}" --compression-backend rate-rans "${RATE_BACKEND_FLAGS[@]}")" + prep_current_compress="$(format_command "${current_bin}" compress "${input_path}" "${prepared_current_comp}" --compression-backend rate-rans "${RATE_BACKEND_FLAGS[@]}")" + prep_baseline_decompress="$(format_command "${baseline_bin}" decompress "${prepared_baseline_comp}" "${verify_baseline_out}" --compression-backend rate-rans "${RATE_BACKEND_FLAGS[@]}")" + prep_current_decompress="$(format_command "${current_bin}" decompress "${prepared_current_comp}" "${verify_current_out}" --compression-backend rate-rans "${RATE_BACKEND_FLAGS[@]}")" + + add_roundtrip_case \ + "rate_rans_${rans_label}" \ + "${input_path}" \ + "${prep_baseline_compress}" \ + "${prep_current_compress}" \ + "${prep_baseline_decompress}" \ + "${prep_current_decompress}" \ + "${verify_baseline_out}" \ + "${verify_current_out}" + + local bench_baseline_comp="${run_root}/bench-outputs/baseline/${rans_label}_rans.itc" + local bench_current_comp="${run_root}/bench-outputs/current/${rans_label}_rans.itc" + local bench_baseline_dec="${run_root}/bench-outputs/baseline/${rans_label}_rans.decompressed.bin" + local bench_current_dec="${run_root}/bench-outputs/current/${rans_label}_rans.decompressed.bin" + + add_case \ + "compress_rate_rans_${rans_label}" \ + "$(format_command "${baseline_bin}" compress "${input_path}" "${bench_baseline_comp}" --compression-backend rate-rans "${RATE_BACKEND_FLAGS[@]}")" \ + "$(format_command "${current_bin}" compress "${input_path}" "${bench_current_comp}" --compression-backend rate-rans "${RATE_BACKEND_FLAGS[@]}")" + + add_case \ + "decompress_rate_rans_${rans_label}" \ + "$(format_command "${baseline_bin}" decompress "${prepared_baseline_comp}" "${bench_baseline_dec}" --compression-backend rate-rans "${RATE_BACKEND_FLAGS[@]}")" \ + "$(format_command "${current_bin}" decompress "${prepared_current_comp}" "${bench_current_dec}" --compression-backend rate-rans "${RATE_BACKEND_FLAGS[@]}")" + done +} + +prepare_roundtrip_artifacts() { + local roundtrip_tsv="$1" + local log_dir="$2" + local i + + printf 'label\tsubject\tstatus\n' >"${roundtrip_tsv}" + + for ((i = 0; i < ${#ROUNDTRIP_LABELS[@]}; i++)); do + local label="${ROUNDTRIP_LABELS[$i]}" + local input_path="${ROUNDTRIP_INPUT_PATHS[$i]}" + + run_command_checked "${label}" "baseline_compress_preflight" "${ROUNDTRIP_BASELINE_COMPRESS_COMMANDS[$i]}" "${log_dir}" + run_command_checked "${label}" "baseline_decompress_preflight" "${ROUNDTRIP_BASELINE_DECOMPRESS_COMMANDS[$i]}" "${log_dir}" + if ! cmp -s "${input_path}" "${ROUNDTRIP_BASELINE_VERIFY_OUTPUTS[$i]}"; then + fail "roundtrip verification failed for baseline case '${label}'" + fi + printf '%s\tbaseline\tpass\n' "${label}" >>"${roundtrip_tsv}" + + run_command_checked "${label}" "current_compress_preflight" "${ROUNDTRIP_CURRENT_COMPRESS_COMMANDS[$i]}" "${log_dir}" + run_command_checked "${label}" "current_decompress_preflight" "${ROUNDTRIP_CURRENT_DECOMPRESS_COMMANDS[$i]}" "${log_dir}" + if ! cmp -s "${input_path}" "${ROUNDTRIP_CURRENT_VERIFY_OUTPUTS[$i]}"; then + fail "roundtrip verification failed for current case '${label}'" + fi + printf '%s\tcurrent\tpass\n' "${label}" >>"${roundtrip_tsv}" + done +} + +print_plan() { + local preset="$1" + local plan_root + plan_root="$(mktemp -d)" + trap 'rm -rf "${plan_root}"' RETURN + + build_cases "/tmp/infotheory-bench-baseline" "/tmp/infotheory-bench-current" "/tmp/infotheory-bench-input.bin" "${plan_root}" + + printf 'PLAN\tpreset\t%s\n' "${preset}" + printf 'PLAN\truns\t%s\n' "${BENCH_RUNS}" + printf 'PLAN\twarmups\t%s\n' "${BENCH_WARMUPS}" + printf 'PLAN\tbytes\t%s\n' "${BENCH_BYTES}" + printf 'PLAN\trate_backends\t%s\n' "$(join_by , "${RATE_SPEC_BACKENDS[@]}")" + printf 'PLAN\tcases\t%s\n' "${#CASE_LABELS[@]}" + printf 'PLAN\troundtrip_cases\t%s\n' "${#ROUNDTRIP_LABELS[@]}" + + local label + for label in "${CASE_LABELS[@]}"; do + printf 'CASE\t%s\n' "${label}" + done +} + +main() { + if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then + usage + exit 0 + fi + + local mode="run" + local baseline_commit="" + local preset="default" + + if [[ "${1:-}" == "--plan" ]]; then + mode="plan" + preset="${2:-default}" + else + [[ $# -ge 1 ]] || { + usage >&2 + exit 1 + } + baseline_commit="$1" + preset="${2:-default}" + fi + + repo_root="$(cd "$(dirname "$0")/.." && pwd)" + SCRIPT_REPO_ROOT="${repo_root}" + resolve_bench_tuning "${preset}" + + if [[ "${mode}" == "plan" ]]; then + print_plan "${preset}" + exit 0 + fi + + need_cmd bash + need_cmd cargo + need_cmd git + need_cmd tar + need_cmd python3 + need_cmd cmp + + if ! command -v hyperfine >/dev/null 2>&1; then + echo "hyperfine is required for CLI benchmarking." >&2 + echo "Install it with: cargo install --locked hyperfine" >&2 + exit 1 + fi + + local bench_root="${INFOTHEORY_CLI_BENCH_ROOT:-/var/tmp/infotheory_bench}" + local stamp + stamp="$(date +%Y%m%d-%H%M%S)" + local run_root="${bench_root}/${stamp}" + local baseline_root="${run_root}/baseline-src" + local current_root="${run_root}/current-src" + local baseline_build="${run_root}/baseline-build" + local current_build="${run_root}/current-build" + local cases_dir="${run_root}/cases" + local log_dir="${run_root}/logs" + mkdir -p "${cases_dir}" "${log_dir}" + + local rustflags + rustflags="$(build_rustflags)" + local shared_features="${INFOTHEORY_CLI_BENCH_CARGO_FEATURES:-cli}" + local baseline_features="${INFOTHEORY_CLI_BENCH_BASELINE_FEATURES:-${shared_features}}" + local current_features="${INFOTHEORY_CLI_BENCH_CURRENT_FEATURES:-${shared_features}}" + + git worktree add --detach "${baseline_root}" "${baseline_commit}" >/dev/null + trap 'git worktree remove --force "${baseline_root}" >/dev/null 2>&1 || true' EXIT + git -C "${baseline_root}" submodule update --init --recursive >/dev/null + copy_dirty_tree "${repo_root}" "${current_root}" "${bench_root}" + + local baseline_bin + local current_bin + baseline_bin="$(build_infotheory_bin "${baseline_root}" "${baseline_build}" "${rustflags}" "${baseline_features}")" + current_bin="$(build_infotheory_bin "${current_root}" "${current_build}" "${rustflags}" "${current_features}")" + + # Allow clocks and thermal state to settle immediately after compilation. + sleep 5 + + local input_path + input_path="$(prepare_input "${SCRIPT_REPO_ROOT}" "${BENCH_BYTES}" "${run_root}/inputs")" + + build_cases "${baseline_bin}" "${current_bin}" "${input_path}" "${run_root}" + + local roundtrip_tsv="${run_root}/roundtrip.tsv" + prepare_roundtrip_artifacts "${roundtrip_tsv}" "${log_dir}" + + echo "Benchmark preset: ${preset}" + echo "Benchmark tuning: runs=${BENCH_RUNS}, warmups=${BENCH_WARMUPS}, bytes=${BENCH_BYTES}" + echo "Cases: ${#CASE_LABELS[@]}" + echo "Roundtrip preflight cases: ${#ROUNDTRIP_LABELS[@]}" + + local order + local i + for order in baseline-current current-baseline; do + echo "Running order: ${order}" + for ((i = 0; i < ${#CASE_LABELS[@]}; i++)); do + local first_name + local second_name + local first_cmd + local second_cmd + + if [[ "${order}" == "baseline-current" ]]; then + first_name="baseline" + second_name="current" + first_cmd="${CASE_BASELINE_COMMANDS[$i]}" + second_cmd="${CASE_CURRENT_COMMANDS[$i]}" + else + first_name="current" + second_name="baseline" + first_cmd="${CASE_CURRENT_COMMANDS[$i]}" + second_cmd="${CASE_BASELINE_COMMANDS[$i]}" + fi + + run_hyperfine_case \ + "${cases_dir}" \ + "${CASE_LABELS[$i]}" \ + "${order}" \ + "${first_name}" \ + "${second_name}" \ + "${first_cmd}" \ + "${second_cmd}" \ + "${BENCH_RUNS}" \ + "${BENCH_WARMUPS}" \ + "${log_dir}" + done + + if [[ "${order}" == "baseline-current" ]]; then + # Reduce drift from immediate back-to-back pass ordering. + sleep 5 + fi + done + + local summary_tsv="${run_root}/summary.tsv" + append_summary_rows "${cases_dir}" "${summary_tsv}" + + echo "CLI benchmark comparison complete." + echo "Run root: ${run_root}" + echo "Summary TSV: ${summary_tsv}" + echo "Roundtrip TSV: ${roundtrip_tsv}" +} + +main "$@" diff --git a/scripts/bench_mcts_regression.py b/scripts/bench_mcts_regression.py new file mode 100644 index 00000000..41ba88cd --- /dev/null +++ b/scripts/bench_mcts_regression.py @@ -0,0 +1,346 @@ +#!/usr/bin/env python3 +"""Baseline-vs-current regression gate for Tranche 3.5 Part 1 MCTS benches.""" + +from __future__ import annotations + +import argparse +import datetime as dt +import os +import pathlib +import re +import shutil +import subprocess +import sys +from dataclasses import dataclass + + +EXPECTED_BENCHMARKS = ( + "mcts_planner_throughput/rho_uct/256", + "mcts_planner_throughput/parallel_uct_wu_workers1/256", + "mcts_planner_throughput/parallel_uct_wu_workers4/256", + "mcts_planner_throughput/parallel_uct_bu_core_workers4/256", + "mcts_tuner_shaped/rho_uct_actions64_h4/192", + "mcts_tuner_shaped/parallel_uct_wu_actions64_h4/192", + "mcts_tuner_shaped/parallel_uct_bu_core_actions64_h4/192", +) + +TIME_LINE_RE = re.compile( + r"time:\s*\[\s*([0-9]+(?:\.[0-9]+)?)\s*(ns|us|ms|s)\s+" + r"([0-9]+(?:\.[0-9]+)?)\s*(ns|us|ms|s)\s+" + r"([0-9]+(?:\.[0-9]+)?)\s*(ns|us|ms|s)\s*\]" +) +UNIT_TO_SECONDS = { + "ns": 1e-9, + "us": 1e-6, + "ms": 1e-3, + "s": 1.0, +} + + +@dataclass(frozen=True) +class BenchResult: + name: str + baseline_s: float + current_s: float + + @property + def delta_pct(self) -> float: + return ((self.current_s / self.baseline_s) - 1.0) * 100.0 + + @property + def threshold_pct(self) -> float: + if "/rho_uct" in self.name: + return 5.0 + if "/parallel_uct_" in self.name: + return 10.0 + raise ValueError(f"unknown benchmark family: {self.name}") + + @property + def is_regression(self) -> bool: + return self.delta_pct >= self.threshold_pct + + +def parse_bench_times(output: str) -> dict[str, float]: + parsed: dict[str, float] = {} + active_name: str | None = None + + for raw_line in output.splitlines(): + line = raw_line.strip().replace("µs", "us") + if not line: + continue + if line.startswith("mcts_"): + active_name = line + continue + if active_name is None: + continue + match = TIME_LINE_RE.search(line) + if match is None: + continue + median_value = float(match.group(3)) + median_unit = match.group(4) + parsed[active_name] = median_value * UNIT_TO_SECONDS[median_unit] + active_name = None + + return parsed + + +def run_command(cmd: list[str], cwd: pathlib.Path, env: dict[str, str]) -> str: + proc = subprocess.run( + cmd, + cwd=str(cwd), + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + check=False, + ) + if proc.returncode != 0: + raise RuntimeError( + f"command failed ({proc.returncode}): {' '.join(cmd)}\n\n{proc.stdout}" + ) + return proc.stdout + + +def ensure_benchmark_target_exists(repo_root: pathlib.Path) -> None: + bench_file = repo_root / "crates" / "infotheory" / "benches" / "mcts_planners.rs" + if not bench_file.is_file(): + raise RuntimeError( + "required benchmark target is unavailable in this checkout: " + f"missing {bench_file}" + ) + + +def ensure_baseline_ref_exists(repo_root: pathlib.Path, baseline_ref: str) -> None: + proc = subprocess.run( + ["git", "rev-parse", "--verify", "--quiet", f"{baseline_ref}^{{commit}}"], + cwd=str(repo_root), + env=os.environ.copy(), + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) + if proc.returncode != 0: + raise RuntimeError(f"unknown baseline ref: {baseline_ref}") + + +def baseline_contains_mcts_harness(repo_root: pathlib.Path, baseline_ref: str) -> bool: + proc = subprocess.run( + [ + "git", + "cat-file", + "-e", + f"{baseline_ref}:crates/infotheory/benches/mcts_planners.rs", + ], + cwd=str(repo_root), + env=os.environ.copy(), + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) + return proc.returncode == 0 + + +def run_benchmark(repo_root: pathlib.Path, target_dir: pathlib.Path, tmp_dir: pathlib.Path) -> str: + ensure_benchmark_target_exists(repo_root) + env = os.environ.copy() + env["CARGO_TARGET_DIR"] = str(target_dir) + env["TMPDIR"] = str(tmp_dir) + env["CARGO_TERM_COLOR"] = "never" + cmd = [ + "cargo", + "bench", + "-p", + "infotheory", + "--bench", + "mcts_planners", + "--", + "--noplot", + ] + return run_command(cmd, cwd=repo_root, env=env) + + +def git_worktree_add(repo_root: pathlib.Path, path: pathlib.Path, commit: str) -> None: + cmd = ["git", "worktree", "add", "--detach", str(path), commit] + run_command(cmd, cwd=repo_root, env=os.environ.copy()) + + +def git_worktree_remove(repo_root: pathlib.Path, path: pathlib.Path) -> None: + cmd = ["git", "worktree", "remove", "--force", str(path)] + run_command(cmd, cwd=repo_root, env=os.environ.copy()) + + +def sync_submodules(repo_root: pathlib.Path) -> None: + env = os.environ.copy() + run_command(["git", "submodule", "sync", "--recursive"], cwd=repo_root, env=env) + run_command( + ["git", "submodule", "update", "--init", "--recursive"], + cwd=repo_root, + env=env, + ) + + +def prepare_baseline_source( + repo_root: pathlib.Path, baseline_dir: pathlib.Path, baseline_ref: str +) -> str: + try: + git_worktree_add(repo_root, baseline_dir, baseline_ref) + sync_submodules(baseline_dir) + return "worktree" + except RuntimeError as exc: + print(f"[mcts-bench] worktree setup failed, falling back to copied checkout: {exc}") + + ignore = shutil.ignore_patterns( + ".bench-runs", + ".codex-test-target", + ".codex-tmp", + "target", + "__pycache__", + ) + shutil.copytree(repo_root, baseline_dir, ignore=ignore) + run_command( + ["git", "checkout", "--detach", baseline_ref], + cwd=baseline_dir, + env=os.environ.copy(), + ) + sync_submodules(baseline_dir) + return "copy" + + +def format_seconds(value: float) -> str: + if value < 1e-6: + return f"{value * 1e9:.2f} ns" + if value < 1e-3: + return f"{value * 1e6:.2f} us" + if value < 1.0: + return f"{value * 1e3:.2f} ms" + return f"{value:.4f} s" + + +def write_summary(run_dir: pathlib.Path, results: list[BenchResult]) -> pathlib.Path: + out = run_dir / "summary.tsv" + with out.open("w", encoding="utf-8") as fh: + fh.write("benchmark\tbaseline_s\tcurrent_s\tdelta_pct\tthreshold_pct\tstatus\n") + for row in results: + status = "FAIL" if row.is_regression else "OK" + fh.write( + f"{row.name}\t{row.baseline_s:.12f}\t{row.current_s:.12f}\t" + f"{row.delta_pct:.6f}\t{row.threshold_pct:.1f}\t{status}\n" + ) + return out + + +def main() -> int: + parser = argparse.ArgumentParser( + description=( + "Run baseline-vs-current MCTS planner benchmarks and enforce " + "Tranche 3.5 Part 1 regression gates." + ) + ) + parser.add_argument("--baseline", required=True, help="Baseline git commit/branch/tag") + parser.add_argument( + "--root", + default="", + help="Directory for benchmark artifacts (default: .bench-runs/mcts-regression)", + ) + args = parser.parse_args() + + repo_root = pathlib.Path(__file__).resolve().parents[1] + root = ( + pathlib.Path(args.root).resolve() + if args.root + else (repo_root / ".bench-runs" / "mcts-regression") + ) + stamp = dt.datetime.now().strftime("%Y%m%d-%H%M%S") + run_dir = root / stamp + baseline_worktree = run_dir / "baseline-src" + baseline_target = run_dir / "baseline-target" + current_target = run_dir / "current-target" + tmp_dir = run_dir / "tmp" + run_dir.mkdir(parents=True, exist_ok=True) + tmp_dir.mkdir(parents=True, exist_ok=True) + + print(f"[mcts-bench] repo: {repo_root}") + print(f"[mcts-bench] baseline: {args.baseline}") + print(f"[mcts-bench] artifacts: {run_dir}") + + ensure_baseline_ref_exists(repo_root, args.baseline) + if not baseline_contains_mcts_harness(repo_root, args.baseline): + raise RuntimeError( + "baseline predates the MCTS benchmark harness; use the first " + "harness-bearing anchor commit or a later baseline" + ) + + baseline_output = "" + current_output = "" + baseline_mode = "" + try: + baseline_mode = prepare_baseline_source(repo_root, baseline_worktree, args.baseline) + print(f"[mcts-bench] baseline source mode: {baseline_mode}") + print("[mcts-bench] running baseline benchmark...") + baseline_output = run_benchmark(baseline_worktree, baseline_target, tmp_dir) + (run_dir / "baseline.log").write_text(baseline_output, encoding="utf-8") + + print("[mcts-bench] running current benchmark...") + current_output = run_benchmark(repo_root, current_target, tmp_dir) + (run_dir / "current.log").write_text(current_output, encoding="utf-8") + finally: + if baseline_mode == "worktree": + try: + git_worktree_remove(repo_root, baseline_worktree) + except Exception as exc: # pragma: no cover + print(f"[mcts-bench] warning: failed to remove worktree: {exc}", file=sys.stderr) + + baseline_times = parse_bench_times(baseline_output) + current_times = parse_bench_times(current_output) + + missing = [ + name + for name in EXPECTED_BENCHMARKS + if name not in baseline_times or name not in current_times + ] + if missing: + print("[mcts-bench] ERROR: missing benchmark rows:") + for name in missing: + print(f" - {name}") + print(f"[mcts-bench] see logs in: {run_dir}") + return 2 + + results: list[BenchResult] = [ + BenchResult( + name=name, + baseline_s=baseline_times[name], + current_s=current_times[name], + ) + for name in EXPECTED_BENCHMARKS + ] + summary_path = write_summary(run_dir, results) + + print("") + print( + f"{'benchmark':72} {'baseline':>12} {'current':>12} " + f"{'change':>10} {'gate':>7} {'status':>6}" + ) + print("-" * 126) + failures = [] + for row in results: + status = "FAIL" if row.is_regression else "OK" + gate = f"{row.threshold_pct:.1f}%" + change = f"{row.delta_pct:+.2f}%" + print( + f"{row.name:72} {format_seconds(row.baseline_s):>12} {format_seconds(row.current_s):>12} " + f"{change:>10} {gate:>7} {status:>6}" + ) + if row.is_regression: + failures.append(row) + + print("") + print(f"[mcts-bench] summary: {summary_path}") + if failures: + print("[mcts-bench] RESULT: FAIL (regression gate exceeded)") + return 1 + print("[mcts-bench] RESULT: PASS (all regression gates satisfied)") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/bench_two_json.sh b/scripts/bench_two_json.sh index 7f89b230..f6be0b7a 100644 --- a/scripts/bench_two_json.sh +++ b/scripts/bench_two_json.sh @@ -9,6 +9,7 @@ TIME_CMD=/usr/bin/time BENCH_SUITE=${INFOTHEORY_BENCH_SUITE:-two-json} COMP_BACKEND=${INFOTHEORY_BENCH_COMPRESSION_BACKEND:-rate-ac} BENCH_FEATURES=${INFOTHEORY_BENCH_FEATURES:-cli} +BENCH_BUILD_MODE=${INFOTHEORY_BENCH_BUILD_MODE:-${INFOTHEORY_BUILD_MODE:-native}} REPEATS=${INFOTHEORY_BENCH_REPEATS:-3} WARMUPS=${INFOTHEORY_BENCH_WARMUPS:-1} SIZES=${INFOTHEORY_BENCH_SIZES:-"4096 16384 65536 262144 1048576 2097152 4194304 10000000"} @@ -19,7 +20,7 @@ RAW_TSV= SUMMARY_TSV= WORK_DIR= OUTPUT_MODE= -RAW_HEADER="operation subject subject_kind expert_kind series size_bytes repetition cpu compression_backend input_sha256 archive_bytes entropy_bpb real_seconds user_seconds sys_seconds rss_kib verified" +RAW_HEADER="operation subject subject_kind expert_kind series size_bytes repetition cpu compression_backend input_sha256 suite_spec_path suite_spec_sha256 build_mode build_features archive_bytes entropy_bpb real_seconds user_seconds sys_seconds rss_kib verified" say() { printf '%s\n' "$*"; } fail() { printf 'ERROR: %s\n' "$*" >&2; exit 1; } @@ -28,14 +29,14 @@ need_cmd() { command -v "$1" >/dev/null 2>&1 || fail "Missing required command: case "${BENCH_SUITE}" in two-json|two_json|two|core|full) BENCH_SUITE=two-json - SUITE_SPEC_PATH="${ROOT_DIR}/examples/two.json" - SUITE_DISPLAY="examples/two.json" + SUITE_SPEC_PATH="${ROOT_DIR}/configs/bench/two.json" + SUITE_DISPLAY="configs/bench/two.json" SUITE_PATH_PREFIX="infotheory-two-json" ;; extra) BENCH_SUITE=extra - SUITE_SPEC_PATH="${ROOT_DIR}/examples/extra.json" - SUITE_DISPLAY="examples/extra.json" + SUITE_SPEC_PATH="${ROOT_DIR}/configs/bench/extra.json" + SUITE_DISPLAY="configs/bench/extra.json" SUITE_PATH_PREFIX="infotheory-extra" ;; *) @@ -44,6 +45,14 @@ case "${BENCH_SUITE}" in esac [ -f "${SUITE_SPEC_PATH}" ] || fail "Benchmark spec not found: ${SUITE_SPEC_PATH}" +SUITE_SPEC_RESOLVED=$(python3 - "${SUITE_SPEC_PATH}" <<'PY' +from pathlib import Path +import sys + +print(Path(sys.argv[1]).resolve()) +PY +) +SUITE_SPEC_SHA256=$(sha256sum "${SUITE_SPEC_PATH}" | awk 'NR==1 { print $1 }') cleanup() { if [ "${INFOTHEORY_BENCH_KEEP_WORKDIR:-0}" = "1" ]; then @@ -76,8 +85,9 @@ Environment: INFOTHEORY_BENCH_REPEATS=3 INFOTHEORY_BENCH_WARMUPS=1 INFOTHEORY_BENCH_SIZES="4096 16384 ... 10000000" - INFOTHEORY_BENCH_SUBJECTS=rwkv + INFOTHEORY_BENCH_SUBJECTS=rwkv7 INFOTHEORY_BENCH_FEATURES="cli backend-rwkv" + INFOTHEORY_BENCH_BUILD_MODE=native|portable INFOTHEORY_BENCH_CPU=11 INFOTHEORY_BENCH_COMPRESSION_BACKEND=rate-ac INFOTHEORY_BENCH_WORKDIR_ROOT=/var/tmp @@ -122,6 +132,36 @@ case " ${BENCH_FEATURES} " in *) BENCH_FEATURES="cli ${BENCH_FEATURES}" ;; esac +case "${BENCH_BUILD_MODE}" in + native|portable) ;; + *) + fail "INFOTHEORY_BENCH_BUILD_MODE (or INFOTHEORY_BUILD_MODE fallback) must be 'native' or 'portable'" + ;; +esac + +portable_rustflags() { + case "$(uname -s)" in + Linux|FreeBSD|OpenBSD) printf '%s' "-C target-cpu=generic -C link-arg=-fuse-ld=lld" ;; + *) printf '%s' "-C target-cpu=generic" ;; + esac +} + +build_infotheory_cli() { + if [ "${BENCH_BUILD_MODE}" = "portable" ]; then + bench_rustflags=$(portable_rustflags) + ( + cd "${ROOT_DIR}" && \ + CARGO_INCREMENTAL=0 \ + CARGO_BUILD_RUSTFLAGS="${bench_rustflags}" \ + RUSTDOCFLAGS="${RUSTDOCFLAGS:-${bench_rustflags}}" \ + cargo build --release --features "${BENCH_FEATURES}" --bin infotheory --locked + ) + return 0 + fi + + (cd "${ROOT_DIR}" && CARGO_INCREMENTAL=0 cargo build --release --features "${BENCH_FEATURES}" --bin infotheory --locked) +} + case "${REPEATS}" in ''|*[!0-9]*) fail "INFOTHEORY_BENCH_REPEATS must be a non-negative integer" @@ -153,6 +193,10 @@ latest_existing_raw_tsv() { ls -1t "/tmp/${SUITE_PATH_PREFIX}-raw-"*.tsv 2>/dev/null | head -n 1 || true } +current_two_json_baseline_tsv() { + ls -1t "${ROOT_DIR}/benchmarks/current/infotheory-two-json-summary"*.tsv 2>/dev/null | head -n 1 || true +} + resolve_output_paths() { latest_raw= if [ -n "${INFOTHEORY_BENCH_RAW_TSV:-}" ]; then @@ -196,6 +240,7 @@ with open(path, newline="") as fh: raise SystemExit(f"{path}: empty file") from exc if header != expected: raise SystemExit(f"{path}: unexpected header") + index = {name: idx for idx, name in enumerate(header)} for lineno, row in enumerate(reader, start=2): if not row: continue @@ -203,9 +248,20 @@ with open(path, newline="") as fh: raise SystemExit( f"{path}: line {lineno}: expected {len(expected)} fields, found {len(row)}" ) - if row[16] != "1": + if row[index["verified"]] != "1": continue - key = (row[0], row[1], row[5], row[6], row[7], row[8], row[9]) + key = ( + row[index["operation"]], + row[index["subject"]], + row[index["size_bytes"]], + row[index["repetition"]], + row[index["cpu"]], + row[index["compression_backend"]], + row[index["input_sha256"]], + row[index["suite_spec_sha256"]], + row[index["build_mode"]], + row[index["build_features"]], + ) if key in seen: raise SystemExit(f"{path}: duplicate verified row at line {lineno}: {key!r}") seen.add(key) @@ -263,7 +319,7 @@ PY CPU=$(choose_cpu) say "[bench] Building release CLI binary with features: ${BENCH_FEATURES}" -(cd "${ROOT_DIR}" && CARGO_INCREMENTAL=0 cargo build --release --features "${BENCH_FEATURES}" --bin infotheory --locked) +build_infotheory_cli [ -x "${BIN_PATH}" ] || fail "Expected built binary at ${BIN_PATH}" python3 - "${SUITE_SPEC_PATH}" "${ROOT_DIR}" "${WORK_DIR}" "${SUITE_DISPLAY}" > "${SUBJECTS_TSV}" <<'PY' @@ -324,6 +380,16 @@ def slug(text: str) -> str: text = re.sub(r"[^A-Za-z0-9._-]+", "-", text.strip()) return text.strip("-").lower() or "expert" +def canonical_subject_name(expert): + kind = str(expert.get("kind") or "") + name = str(expert.get("name") or kind or "expert") + # The canonical two-json CTW slot is now the factorized byte/MSB model. + # Preserve old suite files that still named this subject "ctw" while keeping + # deliberately custom names untouched. + if kind == "fac-ctw" and slug(name) == "ctw": + return "fac-ctw" + return name + print("subject\tsubject_kind\texpert_kind\tspec_path\th_order") print( "\t".join( @@ -338,7 +404,7 @@ print( ) for expert in experts: expert_resolved = canonicalize_relative_paths(expert) - name = str(expert_resolved.get("name") or expert_resolved.get("kind") or "expert") + name = canonical_subject_name(expert_resolved) subject = slug(name) out_path = subject_dir / f"{subject}.json" out_path.write_text(json.dumps(expert_resolved, indent=2, sort_keys=True) + "\n") @@ -520,9 +586,12 @@ PY } append_row() { - printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \ - "$1" "$2" "$3" "$4" "$5" "$6" "$7" "$8" "$9" "${10}" "${11}" "${12}" "${13}" "${14}" "${15}" "${16}" "${17}" \ - >> "${RAW_TSV}" + sep= + for field in "$@"; do + printf '%s%s' "${sep}" "${field}" >> "${RAW_TSV}" + sep="$(printf '\t')" + done + printf '\n' >> "${RAW_TSV}" } row_exists() { @@ -533,6 +602,9 @@ row_exists() { row_cpu=$5 row_compression_backend=$6 row_input_sha256=$7 + row_suite_spec_sha256=$8 + row_build_mode=$9 + row_build_features=${10} awk -F '\t' \ -v row_operation="${row_operation}" \ @@ -542,6 +614,9 @@ row_exists() { -v row_cpu="${row_cpu}" \ -v row_compression_backend="${row_compression_backend}" \ -v row_input_sha256="${row_input_sha256}" \ + -v row_suite_spec_sha256="${row_suite_spec_sha256}" \ + -v row_build_mode="${row_build_mode}" \ + -v row_build_features="${row_build_features}" \ ' NR > 1 && $1 == row_operation && @@ -551,7 +626,10 @@ row_exists() { $8 == row_cpu && $9 == row_compression_backend && $10 == row_input_sha256 && - $17 == "1" { + $12 == row_suite_spec_sha256 && + $13 == row_build_mode && + $14 == row_build_features && + $21 == "1" { found = 1 } END { @@ -562,8 +640,12 @@ row_exists() { say "[bench] Source: ${SOURCE_FILE}" say "[bench] Suite: ${BENCH_SUITE} (${SUITE_DISPLAY})" +say "[bench] Suite spec: ${SUITE_SPEC_RESOLVED}" +say "[bench] Suite spec digest: ${SUITE_SPEC_SHA256}" say "[bench] CPU affinity: ${CPU}" say "[bench] Compression backend: ${COMP_BACKEND}" +say "[bench] Build mode: ${BENCH_BUILD_MODE}" +say "[bench] Build features: ${BENCH_FEATURES}" say "[bench] Repeats: ${REPEATS}" say "[bench] Warmups: ${WARMUPS}" say "[bench] Sizes: ${SIZES}" @@ -605,15 +687,15 @@ for size_bytes in ${SIZES}; do need_subject_work=0 rep=1 while [ "${rep}" -le "${REPEATS}" ]; do - if ! row_exists "h" "${subject}" "${size_bytes}" "${rep}" "${CPU}" "-" "${input_sha256}"; then + if ! row_exists "h" "${subject}" "${size_bytes}" "${rep}" "${CPU}" "-" "${input_sha256}" "${SUITE_SPEC_SHA256}" "${BENCH_BUILD_MODE}" "${BENCH_FEATURES}"; then need_subject_work=1 break fi - if ! row_exists "compress" "${subject}" "${size_bytes}" "${rep}" "${CPU}" "${COMP_BACKEND}" "${input_sha256}"; then + if ! row_exists "compress" "${subject}" "${size_bytes}" "${rep}" "${CPU}" "${COMP_BACKEND}" "${input_sha256}" "${SUITE_SPEC_SHA256}" "${BENCH_BUILD_MODE}" "${BENCH_FEATURES}"; then need_subject_work=1 break fi - if ! row_exists "decompress" "${subject}" "${size_bytes}" "${rep}" "${CPU}" "${COMP_BACKEND}" "${input_sha256}"; then + if ! row_exists "decompress" "${subject}" "${size_bytes}" "${rep}" "${CPU}" "${COMP_BACKEND}" "${input_sha256}" "${SUITE_SPEC_SHA256}" "${BENCH_BUILD_MODE}" "${BENCH_FEATURES}"; then need_subject_work=1 break fi @@ -652,13 +734,13 @@ for size_bytes in ${SIZES}; do need_decompress=0 archive_ready=0 - if ! row_exists "h" "${subject}" "${size_bytes}" "${rep}" "${CPU}" "-" "${input_sha256}"; then + if ! row_exists "h" "${subject}" "${size_bytes}" "${rep}" "${CPU}" "-" "${input_sha256}" "${SUITE_SPEC_SHA256}" "${BENCH_BUILD_MODE}" "${BENCH_FEATURES}"; then need_h=1 fi - if ! row_exists "compress" "${subject}" "${size_bytes}" "${rep}" "${CPU}" "${COMP_BACKEND}" "${input_sha256}"; then + if ! row_exists "compress" "${subject}" "${size_bytes}" "${rep}" "${CPU}" "${COMP_BACKEND}" "${input_sha256}" "${SUITE_SPEC_SHA256}" "${BENCH_BUILD_MODE}" "${BENCH_FEATURES}"; then need_compress=1 fi - if ! row_exists "decompress" "${subject}" "${size_bytes}" "${rep}" "${CPU}" "${COMP_BACKEND}" "${input_sha256}"; then + if ! row_exists "decompress" "${subject}" "${size_bytes}" "${rep}" "${CPU}" "${COMP_BACKEND}" "${input_sha256}" "${SUITE_SPEC_SHA256}" "${BENCH_BUILD_MODE}" "${BENCH_FEATURES}"; then need_decompress=1 fi @@ -672,7 +754,8 @@ $(parse_time_file "${time_path}") EOF append_row \ "h" "${subject}" "${subject_kind}" "${expert_kind}" "h:${subject}" "${size_bytes}" "${rep}" "${CPU}" "-" \ - "${input_sha256}" "" "${entropy_bpb}" "${real_seconds}" "${user_seconds}" "${sys_seconds}" "${rss_kib}" "1" + "${input_sha256}" "${SUITE_SPEC_RESOLVED}" "${SUITE_SPEC_SHA256}" "${BENCH_BUILD_MODE}" "${BENCH_FEATURES}" \ + "" "${entropy_bpb}" "${real_seconds}" "${user_seconds}" "${sys_seconds}" "${rss_kib}" "1" fi if [ "${need_compress}" -eq 1 ]; then @@ -697,11 +780,13 @@ EOF rm -f "${restored_path}" append_row \ "compress" "${subject}" "${subject_kind}" "${expert_kind}" "compress:${subject}" "${size_bytes}" "${rep}" "${CPU}" "${COMP_BACKEND}" \ - "${input_sha256}" "${archive_bytes}" "" "${compress_real_seconds}" "${compress_user_seconds}" "${compress_sys_seconds}" "${compress_rss_kib}" "1" + "${input_sha256}" "${SUITE_SPEC_RESOLVED}" "${SUITE_SPEC_SHA256}" "${BENCH_BUILD_MODE}" "${BENCH_FEATURES}" \ + "${archive_bytes}" "" "${compress_real_seconds}" "${compress_user_seconds}" "${compress_sys_seconds}" "${compress_rss_kib}" "1" if [ "${need_decompress}" -eq 1 ]; then append_row \ "decompress" "${subject}" "${subject_kind}" "${expert_kind}" "decompress:${subject}" "${size_bytes}" "${rep}" "${CPU}" "${COMP_BACKEND}" \ - "${input_sha256}" "${archive_bytes}" "" "${decompress_real_seconds}" "${decompress_user_seconds}" "${decompress_sys_seconds}" "${decompress_rss_kib}" "1" + "${input_sha256}" "${SUITE_SPEC_RESOLVED}" "${SUITE_SPEC_SHA256}" "${BENCH_BUILD_MODE}" "${BENCH_FEATURES}" \ + "${archive_bytes}" "" "${decompress_real_seconds}" "${decompress_user_seconds}" "${decompress_sys_seconds}" "${decompress_rss_kib}" "1" need_decompress=0 fi fi @@ -720,7 +805,8 @@ $(parse_time_file "${time_path}") EOF append_row \ "decompress" "${subject}" "${subject_kind}" "${expert_kind}" "decompress:${subject}" "${size_bytes}" "${rep}" "${CPU}" "${COMP_BACKEND}" \ - "${input_sha256}" "${archive_bytes}" "" "${real_seconds}" "${user_seconds}" "${sys_seconds}" "${rss_kib}" "1" + "${input_sha256}" "${SUITE_SPEC_RESOLVED}" "${SUITE_SPEC_SHA256}" "${BENCH_BUILD_MODE}" "${BENCH_FEATURES}" \ + "${archive_bytes}" "" "${real_seconds}" "${user_seconds}" "${sys_seconds}" "${rss_kib}" "1" fi rm -f "${archive_path}" "${restored_path}" @@ -797,6 +883,10 @@ with open(raw_path, newline="") as fh: row["cpu"], row["compression_backend"], row["input_sha256"], + row["suite_spec_path"], + row["suite_spec_sha256"], + row["build_mode"], + row["build_features"], ) ].append(row) @@ -811,6 +901,10 @@ fieldnames = [ "cpu", "compression_backend", "input_sha256", + "suite_spec_path", + "suite_spec_sha256", + "build_mode", + "build_features", "real_seconds_mean", "real_seconds_stdev", "real_seconds_median", @@ -845,11 +939,16 @@ sorted_keys = sorted( ) with open(summary_path, "w", newline="") as fh: - writer = csv.DictWriter(fh, fieldnames=fieldnames, delimiter="\t") + writer = csv.DictWriter( + fh, + fieldnames=fieldnames, + delimiter="\t", + lineterminator="\n", + ) writer.writeheader() for key in sorted_keys: rows = groups[key] - operation, subject, subject_kind, expert_kind, series, size_bytes, cpu, compression_backend, input_sha256 = key + operation, subject, subject_kind, expert_kind, series, size_bytes, cpu, compression_backend, input_sha256, suite_spec_path, suite_spec_sha256, build_mode, build_features = key real = [row["real_seconds"] for row in rows] user = [row["user_seconds"] for row in rows] sysc = [row["sys_seconds"] for row in rows] @@ -872,6 +971,10 @@ with open(summary_path, "w", newline="") as fh: "cpu": cpu, "compression_backend": compression_backend, "input_sha256": input_sha256, + "suite_spec_path": suite_spec_path, + "suite_spec_sha256": suite_spec_sha256, + "build_mode": build_mode, + "build_features": build_features, "real_seconds_mean": fmt(mean(real)), "real_seconds_stdev": fmt(stdev(real)), "real_seconds_median": fmt(median(real)), @@ -900,8 +1003,13 @@ PY say "[bench] Raw TSV: ${RAW_TSV}" say "[bench] Summary TSV: ${SUMMARY_TSV}" if [ "${BENCH_SUITE}" = "two-json" ]; then - say "[bench] Compare against the checked-in baseline:" - say " python3 '${ROOT_DIR}/scripts/compare_bench_two_json.py' '${SUMMARY_TSV}'" + CURRENT_BASELINE_TSV=$(current_two_json_baseline_tsv) + if [ -n "${CURRENT_BASELINE_TSV}" ]; then + say "[bench] Compare against the checked-in baseline:" + say " '${ROOT_DIR}/scripts/compare_bench_two_json.lua' --baseline '${CURRENT_BASELINE_TSV}' '${SUMMARY_TSV}'" + else + say "[bench] No checked-in two-json baseline summary found under benchmarks/current." + fi else say "[bench] No checked-in baseline comparator is configured for suite '${BENCH_SUITE}'." fi diff --git a/scripts/benchmark_tuner_two_json_modes.sh b/scripts/benchmark_tuner_two_json_modes.sh new file mode 100755 index 00000000..4bee4f2c --- /dev/null +++ b/scripts/benchmark_tuner_two_json_modes.sh @@ -0,0 +1,660 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Reproducible three-mode tuner comparison anchored to examples/two.json. +# Modes: annealed_hill_climbing, mc_aixi_fac_ctw, aiqi_warmstart_exact_jh. +# +# Positional arguments (all optional): +# 1) input file path +# 2) per-evaluation time limit (seconds, > 0) +# 3) memory cap (GB, >= 1) +# +# Defaults: +# - input: `git show HEAD:README.md` written to /tmp run directory +# - eval_time_limit_seconds: 2 +# - max_memory_gb: 1 +# - strict RSS/accounting mode: hybrid_strict_max (Linux + delegated cgroup-v2 required) +# +# Environment overrides: +# TUNER_MAX_EVALUATIONS (default: 120) +# TUNER_TIME_BUDGET_SECONDS (default: 600) +# TUNER_OUTPUT_ROOT (default: /tmp/infotheory-tuner-two-json) + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" + +INPUT_FILE="${1:-}" +EVAL_TIME_LIMIT_SECONDS="${2:-2}" +MAX_MEMORY_GB="${3:-1}" +MAX_EVALUATIONS="${TUNER_MAX_EVALUATIONS:-120}" +TIME_BUDGET_SECONDS="${TUNER_TIME_BUDGET_SECONDS:-600}" +OUTPUT_ROOT="${TUNER_OUTPUT_ROOT:-/tmp/infotheory-tuner-two-json}" + +# Keep this list explicit so runs are auditable and reproducible. +FEATURES="tuner cli backend-ctw backend-mixture backend-ppmd backend-rosa backend-match backend-rwkv" +SCALAR_REP="finite-ieee754-f64-nonfinite-forbidden-v1" +OBS_ADAPTER_REF="single-channel-conditional-byte-adapter-v1" +TWO_JSON_PATH="$REPO_ROOT/examples/two.json" +STRICT_RSS_MODE="hybrid_strict_max" +EVAL_CGROUP_PARENT="${INFOTHEORY_TUNER_EVAL_CGROUP_PARENT:-}" + +die() { + echo "Error: $*" >&2 + exit 1 +} + +need_cmd() { + command -v "$1" >/dev/null 2>&1 || die "required command '$1' is not available" +} + +need_cmd cargo +need_cmd git +need_cmd python3 + +if [ "$(uname -s)" != "Linux" ]; then + die "strict theorem-facing benchmark requires Linux (requested rss mode: $STRICT_RSS_MODE)" +fi +if [ -z "$EVAL_CGROUP_PARENT" ]; then + die "INFOTHEORY_TUNER_EVAL_CGROUP_PARENT is required for strict theorem-facing runs" +fi +if [ ! -d "$EVAL_CGROUP_PARENT" ]; then + die "delegated cgroup parent does not exist: $EVAL_CGROUP_PARENT" +fi +if [ -f "$EVAL_CGROUP_PARENT/cgroup.subtree_control" ] && ! grep -Eq '(^|[[:space:]])memory($|[[:space:]])' "$EVAL_CGROUP_PARENT/cgroup.subtree_control"; then + die "delegated cgroup parent must have memory enabled in cgroup.subtree_control: $EVAL_CGROUP_PARENT" +fi + +python3 - "$EVAL_TIME_LIMIT_SECONDS" "$MAX_MEMORY_GB" "$MAX_EVALUATIONS" "$TIME_BUDGET_SECONDS" <<'PY' +import sys + +eval_limit = float(sys.argv[1]) +mem_gb = float(sys.argv[2]) +max_evals = int(sys.argv[3]) +time_budget = float(sys.argv[4]) +if not (eval_limit > 0.0): + raise SystemExit("per-evaluation time limit must be > 0") +if not (mem_gb >= 1.0): + raise SystemExit("memory cap (GB) must be >= 1") +if max_evals <= 0: + raise SystemExit("TUNER_MAX_EVALUATIONS must be > 0") +if not (time_budget > 0.0): + raise SystemExit("TUNER_TIME_BUDGET_SECONDS must be > 0") +PY + +RUN_DIR="$OUTPUT_ROOT/run-$(date +%Y%m%d-%H%M%S)" +mkdir -p "$RUN_DIR" + +SUBJECT_PATH="$RUN_DIR/subject.bin" +if [ -n "$INPUT_FILE" ]; then + [ -f "$INPUT_FILE" ] || die "input file '$INPUT_FILE' does not exist" + cp "$INPUT_FILE" "$SUBJECT_PATH" + SUBJECT_SOURCE="$INPUT_FILE" +else + git -C "$REPO_ROOT" show HEAD:README.md > "$SUBJECT_PATH" + SUBJECT_SOURCE="git show HEAD:README.md" +fi + +SUBJECT_BYTES="$(wc -c < "$SUBJECT_PATH" | tr -d '[:space:]')" +[ "$SUBJECT_BYTES" -gt 0 ] || die "subject file is empty: $SUBJECT_PATH" + +MIN_THROUGHPUT="$(python3 - "$SUBJECT_BYTES" "$EVAL_TIME_LIMIT_SECONDS" <<'PY' +import sys +b = int(sys.argv[1]) +t = float(sys.argv[2]) +print(f"{b / t:.6f}") +PY +)" + +MAX_MEMORY_BYTES="$(python3 - "$MAX_MEMORY_GB" <<'PY' +import sys +gb = float(sys.argv[1]) +print(int(gb * (1024 ** 3))) +PY +)" + +CANONICAL_BASELINE_RATE="$RUN_DIR/two-json-rate-backend-canonical.json" +ANNEALED_SPEC="$RUN_DIR/two-json-annealed-spec.json" +MCAIXI_SPEC="$RUN_DIR/two-json-mcaixi-spec.json" +WARMSTART_SPEC="$RUN_DIR/two-json-warmstart-spec.json" +WARMSTART_TEACHER="$RUN_DIR/warmstart-teacher.json" +MCAIXI_CERT="$RUN_DIR/mcaixi-exact-reward-cert.json" +WARMSTART_CERT="$RUN_DIR/warmstart-exact-reward-cert.json" + +ANNEALED_OUTPUT="$RUN_DIR/annealed-output.json" +ANNEALED_REPORT="$RUN_DIR/annealed-report.json" +MCAIXI_OUTPUT="$RUN_DIR/mcaixi-output.json" +MCAIXI_REPORT="$RUN_DIR/mcaixi-report.json" +WARMSTART_OUTPUT="$RUN_DIR/warmstart-output.json" +WARMSTART_REPORT="$RUN_DIR/warmstart-report.json" +SUMMARY_JSON="$RUN_DIR/comparison-summary.json" +SUMMARY_TSV="$RUN_DIR/comparison-summary.tsv" +RUN_LOG="$RUN_DIR/benchmark.log" + +canonicalize_two_json_rate_backend() { + local input_path="$1" + local output_path="$2" + python3 - "$input_path" "$output_path" <<'PY' +import json +import pathlib +import struct +import sys + +input_path = pathlib.Path(sys.argv[1]) +output_path = pathlib.Path(sys.argv[2]) +doc = json.loads(input_path.read_text()) + +if not isinstance(doc, dict): + raise SystemExit("examples/two.json must be a JSON object") +if doc.get("kind") != "neural": + raise SystemExit("examples/two.json kind must be 'neural'") +experts = doc.get("experts") +if not isinstance(experts, list) or len(experts) == 0: + raise SystemExit("examples/two.json experts must be a non-empty array") + + +def parse_rwkv_cfg_string(raw: str): + if not raw.startswith("cfg:"): + raise SystemExit("rwkv7 method must start with 'cfg:' in examples/two.json") + policy = None + cfg_part = raw + if ";policy:" in raw: + cfg_part, policy = raw.split(";policy:", 1) + cfg_fields = cfg_part[len("cfg:"):].split(",") + parsed = {} + for field in cfg_fields: + key, sep, value = field.partition("=") + if sep != "=": + raise SystemExit(f"invalid rwkv7 cfg field '{field}'") + parsed[key.strip()] = value.strip() + + def parse_int(key: str) -> int: + if key not in parsed: + raise SystemExit(f"rwkv7 cfg missing '{key}'") + return int(parsed[key], 10) + + def parse_float(key: str) -> float: + if key not in parsed: + raise SystemExit(f"rwkv7 cfg missing '{key}'") + value64 = float(parsed[key]) + packed = struct.pack("!f", value64) + return struct.unpack("!f", packed)[0] + + train_raw = parsed.get("train", "none") + train_mode = { + "none": "none", + "sgd": "sgd", + "adam": "adam", + }.get(train_raw) + if train_mode is None: + raise SystemExit(f"unsupported rwkv7 cfg train mode '{train_raw}'") + + cfg = { + "hidden": parse_int("hidden"), + "layers": parse_int("layers"), + "intermediate": parse_int("intermediate"), + "decay_rank": parse_int("decay_rank"), + "a_rank": parse_int("a_rank"), + "v_rank": parse_int("v_rank"), + "g_rank": parse_int("g_rank"), + "seed": parse_int("seed"), + "train_mode": train_mode, + "lr": parse_float("lr"), + "stride": parse_int("stride"), + } + + method = { + "kind": "online", + "cfg": cfg, + "policy": policy if policy is not None and len(policy) > 0 else None, + } + return method + +mixture_spec = { + "kind": "neural", + "schedule": "default", + "alpha": float(doc.get("alpha", 0.01)), + "decay": None, + "experts": [], +} + +for expert in experts: + if not isinstance(expert, dict): + raise SystemExit("each expert in examples/two.json must be an object") + kind = expert.get("kind") + if not isinstance(kind, str): + raise SystemExit("each expert in examples/two.json must include string kind") + + out = { + "kind": kind, + "log_prior": float(expert.get("log_prior", expert.get("prior", 0.0))), + } + if isinstance(expert.get("name"), str): + out["name"] = expert["name"] + + if kind == "ctw": + out["depth"] = int(expert.get("depth", 16)) + elif kind == "fac-ctw": + out["base_depth"] = int(expert.get("base_depth", expert.get("depth", 16))) + out["num_percept_bits"] = int(expert.get("num_percept_bits", 8)) + out["encoding_bits"] = int(expert.get("encoding_bits", 8)) + elif kind == "ppmd": + out["order"] = int(expert.get("order", 10)) + out["memory_mb"] = int(expert.get("memory_mb", 64)) + elif kind == "rosaplus": + out["max_order"] = int(expert.get("max_order", -1)) + elif kind == "match": + out["hash_bits"] = int(expert.get("hash_bits", 20)) + out["min_len"] = int(expert.get("min_len", 4)) + out["max_len"] = int(expert.get("max_len", 255)) + out["base_mix"] = float(expert.get("base_mix", 0.02)) + out["confidence_scale"] = float(expert.get("confidence_scale", 1.0)) + elif kind == "rwkv7": + method = expert.get("method") + if isinstance(method, str): + out["method"] = parse_rwkv_cfg_string(method) + elif isinstance(method, dict): + out["method"] = method + else: + raise SystemExit("rwkv7 expert in examples/two.json must include method") + else: + raise SystemExit(f"unsupported expert kind '{kind}' in examples/two.json") + + mixture_spec["experts"].append(out) + +canonical_rate_backend = { + "kind": "mixture", + "spec": mixture_spec, +} +output_path.write_text(json.dumps(canonical_rate_backend, indent=2) + "\n") +PY +} + +build_specs() { + canonicalize_two_json_rate_backend "$TWO_JSON_PATH" "$CANONICAL_BASELINE_RATE" + python3 - "$CANONICAL_BASELINE_RATE" "$SUBJECT_PATH" "$EVAL_TIME_LIMIT_SECONDS" "$TIME_BUDGET_SECONDS" "$MIN_THROUGHPUT" "$MAX_MEMORY_BYTES" "$ANNEALED_OUTPUT" "$ANNEALED_REPORT" "$MCAIXI_OUTPUT" "$MCAIXI_REPORT" "$WARMSTART_OUTPUT" "$WARMSTART_REPORT" "$WARMSTART_TEACHER" "$ANNEALED_SPEC" "$MCAIXI_SPEC" "$WARMSTART_SPEC" <<'PY' +import copy +import json +import pathlib +import sys + +( + baseline_rate_path, + subject_path, + eval_time_limit_seconds, + time_budget_seconds, + min_throughput, + max_memory_bytes, + annealed_output, + annealed_report, + mcaixi_output, + mcaixi_report, + warmstart_output, + warmstart_report, + warmstart_teacher_path, + annealed_spec_path, + mcaixi_spec_path, + warmstart_spec_path, +) = sys.argv[1:] + +baseline_rate = json.loads(pathlib.Path(baseline_rate_path).read_text()) +baseline_candidate = { + "kind": "rate-ac", + "rate_backend": baseline_rate, + "framing": "framed", +} + +bounds = { + "allowed_backends": ["fac-ctw", "ppmd", "rosaplus", "match", "rwkv7", "mixture"], + "forbidden_backends": [], + "parameter_ranges": [ + {"parameter": "rate_backend.spec.alpha", "min": 0.005, "max": 0.20}, + {"parameter": "rate_backend.spec.experts[0].base_depth", "min": 8.0, "max": 96.0}, + {"parameter": "rate_backend.spec.experts[1].order", "min": 4.0, "max": 16.0}, + {"parameter": "rate_backend.spec.experts[1].memory_mb", "min": 64.0, "max": 768.0}, + {"parameter": "rate_backend.spec.experts[2].max_order", "min": -1.0, "max": 128.0}, + {"parameter": "rate_backend.spec.experts[3].hash_bits", "min": 16.0, "max": 22.0}, + {"parameter": "rate_backend.spec.experts[3].min_len", "min": 2.0, "max": 16.0}, + {"parameter": "rate_backend.spec.experts[3].max_len", "min": 32.0, "max": 255.0}, + {"parameter": "rate_backend.spec.experts[3].base_mix", "min": 0.005, "max": 0.10}, + {"parameter": "rate_backend.spec.experts[3].confidence_scale", "min": 0.5, "max": 2.0}, + ], + "max_experts": 8, + "max_mixture_nesting_depth": 3, + "min_experts": 3, + "allow_duplicate_experts": False, + "required_experts": ["fac-ctw", "ppmd"], + "forbidden_expert_pairs": [], +} + +common = { + "schema_version": 1, + "kind": "tune", + "assets": [{"id": "dataset", "path": subject_path}], + "input_asset": "dataset", + "baseline_candidate": baseline_candidate, + "bounds": bounds, + "eval_time_limit_seconds": float(eval_time_limit_seconds), + "time_budget_seconds": float(time_budget_seconds), + "min_throughput_bytes_per_second": float(min_throughput), + "max_memory_bytes": int(max_memory_bytes), + "seed": 1337, +} + +annealed = copy.deepcopy(common) +annealed["controller"] = { + "kind": "annealed_hill_climbing", + "max_mutation_radius": 4, +} +annealed["output_config_path"] = annealed_output +annealed["report_path"] = annealed_report + +planner_interface = { + "observation_bits": 8, + "observation_stream_len": 1, + "observation_key_mode": "full_stream", + "reward_bits": 32, + "agent_actions": 20, +} + +mcaixi = copy.deepcopy(common) +mcaixi["controller"] = { + "kind": "mc_aixi_fac_ctw", + "interface": planner_interface, + "planner_simulations_per_step": 24, +} +mcaixi["output_config_path"] = mcaixi_output +mcaixi["report_path"] = mcaixi_report + +warmstart = copy.deepcopy(common) +warmstart["assets"] = [ + {"id": "dataset", "path": subject_path}, + {"id": "teacher", "path": warmstart_teacher_path}, +] +warmstart["controller"] = { + "kind": "aiqi_warmstart_exact_jh", + "interface": planner_interface, + "planner_simulations_per_step": 24, + "return_horizon": 4, + "label_phase_period": 8, + "warmstart_teacher_dataset_asset": "teacher", +} +warmstart["output_config_path"] = warmstart_output +warmstart["report_path"] = warmstart_report + +pathlib.Path(annealed_spec_path).write_text(json.dumps(annealed, indent=2) + "\n") +pathlib.Path(mcaixi_spec_path).write_text(json.dumps(mcaixi, indent=2) + "\n") +pathlib.Path(warmstart_spec_path).write_text(json.dumps(warmstart, indent=2) + "\n") +PY +} + +run_tune() { + local mode="$1" + shift + echo "[$(date +%H:%M:%S)] running $mode" | tee -a "$RUN_LOG" + ( + cd "$REPO_ROOT" + cargo run --release -p infotheory --no-default-features --features "$FEATURES" -- "$@" + ) 2>&1 | tee -a "$RUN_LOG" +} + +emit_exact_cert() { + local spec_path="$1" + local cert_path="$2" + run_tune "emit-cert:$cert_path" \ + tune "$spec_path" \ + --rss-mode "$STRICT_RSS_MODE" \ + --evaluator-cgroup-parent "$EVAL_CGROUP_PARENT" \ + --emit-exact-reward-encoding-certificate "$cert_path" +} + +compute_crc32_hex_of_file_bytes() { + local path="$1" + python3 - "$path" <<'PY' +import pathlib +import sys +import zlib +p = pathlib.Path(sys.argv[1]) +raw = p.read_bytes() +print(f"{zlib.crc32(raw) & 0xffffffff:08x}") +PY +} + +extract_observation_adapter_crc() { + local report_path="$1" + python3 - "$report_path" <<'PY' +import json +import pathlib +import sys +report = json.loads(pathlib.Path(sys.argv[1]).read_text()) +value = report["provenance"]["observation_adapter_content_crc32"] +print(value) +PY +} + +write_provisional_warmstart_teacher() { + local teacher_path="$1" + local adapter_crc="$2" + local reward_cert_crc="$3" + cat > "$teacher_path" < "$RUN_LOG" + { + echo "run_dir=$RUN_DIR" + echo "subject_source=$SUBJECT_SOURCE" + echo "subject_path=$SUBJECT_PATH" + echo "subject_bytes=$SUBJECT_BYTES" + echo "eval_time_limit_seconds=$EVAL_TIME_LIMIT_SECONDS" + echo "max_memory_gb=$MAX_MEMORY_GB" + echo "max_memory_bytes=$MAX_MEMORY_BYTES" + echo "min_throughput_bytes_per_second=$MIN_THROUGHPUT" + echo "max_evaluations=$MAX_EVALUATIONS" + echo "time_budget_seconds=$TIME_BUDGET_SECONDS" + echo "features=$FEATURES" + echo "two_json_source=$TWO_JSON_PATH" + echo "rss_mode=$STRICT_RSS_MODE" + echo "evaluator_cgroup_parent=$EVAL_CGROUP_PARENT" + } | tee -a "$RUN_LOG" + + [ -f "$TWO_JSON_PATH" ] || die "missing baseline source: $TWO_JSON_PATH" + + build_specs + + run_tune "annealed" \ + tune "$ANNEALED_SPEC" \ + --rss-mode "$STRICT_RSS_MODE" \ + --evaluator-cgroup-parent "$EVAL_CGROUP_PARENT" \ + --max-evaluations "$MAX_EVALUATIONS" + + emit_exact_cert "$MCAIXI_SPEC" "$MCAIXI_CERT" + run_tune "mcaixi" \ + tune "$MCAIXI_SPEC" \ + --rss-mode "$STRICT_RSS_MODE" \ + --evaluator-cgroup-parent "$EVAL_CGROUP_PARENT" \ + --exact-reward-encoding-certificate "$MCAIXI_CERT" \ + --max-evaluations "$MAX_EVALUATIONS" + + emit_exact_cert "$WARMSTART_SPEC" "$WARMSTART_CERT" + warm_reward_crc="$(compute_crc32_hex_of_file_bytes "$WARMSTART_CERT")" + adapter_crc="$(extract_observation_adapter_crc "$ANNEALED_REPORT")" + write_provisional_warmstart_teacher "$WARMSTART_TEACHER" "$adapter_crc" "$warm_reward_crc" + + warm_probe_err="$RUN_DIR/warmstart-probe-error.log" + set +e + ( + cd "$REPO_ROOT" + cargo run --release -p infotheory --no-default-features --features "$FEATURES" -- \ + tune "$WARMSTART_SPEC" \ + --rss-mode "$STRICT_RSS_MODE" \ + --evaluator-cgroup-parent "$EVAL_CGROUP_PARENT" \ + --exact-reward-encoding-certificate "$WARMSTART_CERT" \ + --max-evaluations 1 + ) >"$RUN_DIR/warmstart-probe-stdout.log" 2>"$warm_probe_err" + probe_status=$? + set -e + if [ "$probe_status" -ne 0 ]; then + expected_fp="$(extract_expected_fingerprint_from_error_log "$warm_probe_err")" + if [ -z "$expected_fp" ]; then + cat "$warm_probe_err" >&2 + die "warmstart probe failed, and expected task_fingerprint could not be extracted" + fi + patch_warmstart_teacher_task_fingerprint "$WARMSTART_TEACHER" "$expected_fp" + fi + + run_tune "warmstart" \ + tune "$WARMSTART_SPEC" \ + --rss-mode "$STRICT_RSS_MODE" \ + --evaluator-cgroup-parent "$EVAL_CGROUP_PARENT" \ + --exact-reward-encoding-certificate "$WARMSTART_CERT" \ + --max-evaluations "$MAX_EVALUATIONS" + + write_summary + + { + echo + echo "Completed three-mode comparison anchored to examples/two.json." + echo "Run directory: $RUN_DIR" + echo "Summary JSON: $SUMMARY_JSON" + echo "Summary TSV: $SUMMARY_TSV" + echo "Reports:" + echo " annealed: $ANNEALED_REPORT" + echo " mcaixi: $MCAIXI_REPORT" + echo " warmstart: $WARMSTART_REPORT" + } | tee -a "$RUN_LOG" +} + +main diff --git a/scripts/check_tuner_traceability.py b/scripts/check_tuner_traceability.py new file mode 100644 index 00000000..3bf27858 --- /dev/null +++ b/scripts/check_tuner_traceability.py @@ -0,0 +1,422 @@ +#!/usr/bin/env python3 +"""Check that the Tuner V1 traceability checklist points at live symbols. + +The checklist is intentionally human-readable, so this script keeps the gate +lightweight: it verifies the specific code symbols and regression names that +anchor each normative section still exist as definitions in the implementation. +""" + +from __future__ import annotations + +from pathlib import Path +import sys + +try: + import tree_sitter + import tree_sitter_rust +except ImportError as _exc: + sys.exit( + f"check_tuner_traceability: missing dependency: {_exc}\n" + "Run with: uv run --no-project --with tree-sitter --with tree-sitter-rust " + "python scripts/check_tuner_traceability.py" + ) + + +ROOT = Path(__file__).resolve().parents[1] +TRACEABILITY = ROOT / "docs" / "tuner-v1-traceability.md" +TUNER = ROOT / "crates" / "infotheory" / "src" / "tuner.rs" +TUNER_MODULE_DIR = ROOT / "crates" / "infotheory" / "src" / "tuner" +TUNER_SOURCES = ( + TUNER, + TUNER_MODULE_DIR / "annealer.rs", + TUNER_MODULE_DIR / "causal_dataset.rs", + TUNER_MODULE_DIR / "certificates.rs", + TUNER_MODULE_DIR / "config.rs", + TUNER_MODULE_DIR / "eval.rs", + TUNER_MODULE_DIR / "planner_bridge.rs", + TUNER_MODULE_DIR / "report.rs", + TUNER_MODULE_DIR / "tests.rs", +) +WARMSTART = ROOT / "crates" / "infotheory" / "src" / "aixi" / "warmstart.rs" +WARMSTART_CONTRACT = ( + ROOT / "crates" / "infotheory" / "src" / "aixi" / "warmstart_contract.rs" +) +PLANNER_RUNTIME = ROOT / "crates" / "infotheory" / "src" / "aixi" / "planner_runtime.rs" +TUNER_TESTS = ROOT / "crates" / "infotheory" / "tests" / "tuner_integration.rs" +SPEC_TESTS = ROOT / "crates" / "infotheory" / "src" / "spec" / "document" / "tests.rs" +SPEC_PARSER = ROOT / "crates" / "infotheory" / "src" / "spec" / "document" / "parser.rs" + + +REQUIRED_REFS: tuple[tuple[str, Path], ...] = ( + ("TuneExecutionConfig::from_json_value", TUNER), + ("TuneExecutionConfig::apply_theorem_object", TUNER), + ("parse_tune_command_args", TUNER), + ("VerifiedDeterministicEvaluatorTable", TUNER), + ("parse_causal_header_profile", TUNER), + ("validate_header_profile_consistency", TUNER), + ("load_dataset", TUNER), + ("parse_lowered_event", TUNER), + ("structured_causal_dataset_rejects_missing_header_and_charged_history", TUNER), + ("causal_dataset_header_and_event_grammar_are_strict", TUNER), + ("ObjectiveTarget::PlannerDeployableModel", TUNER), + ("planner_deployability_report", TUNER), + ("theorem_timing_basis", TUNER), + ("theorem_claims_report", TUNER), + ("PlannerControllerContract::reward_encoder", TUNER), + ("TunerRewardEncoder::exact_integer_objective_difference", TUNER), + ("parse_finite_reward_map", TUNER), + ("finite_reward_map_rejects_reachable_rewards_alias", TUNER), + ("complete_nonnegative_interval_max", TUNER), + ( + "CompiledTuneController::exact_objective_difference_controller", + TUNER, + ), + ("TunerRawObservation::from_runtime_step", TUNER), + ("observation_adapter_spec_value", TUNER), + ("load_exact_state_observation_certificate", TUNER), + ("validate_exact_state_observation_artifact", TUNER), + ("project_observation_output", TUNER), + ("compile_tuner_planner_run_spec", TUNER), + ("validate_theorem_planner_mutation_domain", TUNER), + ("warmstart_exact_jh_planner_task_fingerprint", WARMSTART_CONTRACT), + ("WarmStartExactJhTeacherDataset", WARMSTART), + ("WarmStartExactJhTeacherContract", WARMSTART), + ("warmstart_teacher_trace_from_jsonl_path", WARMSTART), + ("merge_warmstart_teacher_traces_deterministic", WARMSTART), + ("validate_warmstart_exact_jh_teacher_contract", PLANNER_RUNTIME), + ("load_warmstart_exact_jh_teacher_dataset", PLANNER_RUNTIME), + ("WarmStartExactJhAgent::same_task_live_trace", WARMSTART), + ("merge_warmstart_trace_deterministic", TUNER), + ("TunerPlannerAgentRuntime::rebuild_warmstart_agent", TUNER), + ("resolve_evaluator_runtime_profile", TUNER), + ("ResolvedEvaluatorRuntimeProfile", TUNER), + ("ResolvedMemoryAccountingKind", TUNER), + ("EvaluatorWorkerCgroup", TUNER), + ("resolve_required_tuner_eval_cgroup_parent", TUNER), + ("annealer_progress_from_elapsed", TUNER), + ("annealer_temperature", TUNER), + ("VerifiedTheoremInputs", TUNER), + ("canonical_tune_document_rejects_unknown_nested_fields", TUNER_TESTS), + ("canonical_tune_document_rejects_candidate_local_external_assets", TUNER_TESTS), + ("tune_exec_config_rejects_unknown_fields_and_malformed_theorem", TUNER_TESTS), + ("tune_executor_rejects_unsupported_certificate_uri_scheme", TUNER_TESTS), + ("exact_family_controller_rejects_missing_exact_reward_certificate", TUNER_TESTS), + ("exact_controller_rejects_finite_reward_map_without_complete_interval", TUNER_TESTS), + ("exact_reward_certificate_rejects_unrepresentable_reachable_reward", TUNER_TESTS), + ("exact_finite_reward_map_encodes_objective_difference_not_symbol_arithmetic", TUNER), + ("discounted_aiqi_exact_theorem_claims_remain_uncertified_by_family", TUNER), + ("annealer_schedule_matches_normative_log_linear_law", TUNER), + ("warmstart_exact_jh_rejects_nonidentity_finite_reward_map", TUNER_TESTS), + ("planner_percept_encoding_distinguishes_diagnostic_tokens", TUNER), + ("exact_state_observation_certificate_requires_injectivity_basis", TUNER_TESTS), + ("exact_state_observation_projection_supports_stream_hash", TUNER), + ("exact_state_observation_certificate_rejects_duplicate_state_ids", TUNER_TESTS), + ("warmstart_trace_refresh_merges_same_task_live_trace", TUNER_TESTS), + ("warmstart_exact_jh_json_binary_and_compile_roundtrip", SPEC_TESTS), + ("jsonl_trace_converter_rejects_malformed_and_inconsistent_records", WARMSTART), + ("planner_deployable_model_flag_reports_objective_target_and_diagnostics", TUNER_TESTS), + ("deterministic_table_peak_memory_can_make_baseline_nondeployable", TUNER_TESTS), + ("real_time_timing_certificate_sets_verified_timing_basis", TUNER_TESTS), + ("tune_cli_loads_binary_itsd_tune_document", TUNER_TESTS), + ("planner_run_parser_rejects_internal_tuner_bridge_environment", SPEC_TESTS), + ("planner_run_binary_rejects_internal_tuner_bridge_environment", SPEC_TESTS), + ("reject_tune_candidate_local_external_refs", SPEC_PARSER), + ("ensure_tune_baseline_candidate_is_canonical_json", SPEC_PARSER), +) + + +class AstIndex: + def __init__(self) -> None: + self.functions: set[str] = set() + self.structs: set[str] = set() + self.enums: set[str] = set() + self.struct_fields: set[str] = set() + self.enum_variants: set[str] = set() + self.impl_methods: set[str] = set() + self.test_functions: set[str] = set() + + def merge(self, other: "AstIndex") -> None: + self.functions.update(other.functions) + self.structs.update(other.structs) + self.enums.update(other.enums) + self.struct_fields.update(other.struct_fields) + self.enum_variants.update(other.enum_variants) + self.impl_methods.update(other.impl_methods) + self.test_functions.update(other.test_functions) + + +def rust_language(): + language = tree_sitter_rust.language() + if isinstance(language, tree_sitter.Language): + return language + return tree_sitter.Language(language) + + +def rust_parser(): + parser = tree_sitter.Parser() + language = rust_language() + try: + parser.language = language + except AttributeError: + parser.set_language(language) + return parser + + +def node_text(node) -> str: + return node.text.decode("utf-8") + + +def normalized_attribute_text(node) -> str: + return "".join(node_text(node).split()) + + +def is_test_attribute(node) -> bool: + return node.type == "attribute_item" and normalized_attribute_text(node) == "#[test]" + + +def function_has_test_attribute(node) -> bool: + prev = node.prev_named_sibling + while prev: + if is_test_attribute(prev): + return True + if prev.type not in ("attribute_item", "line_comment", "block_comment"): + return False + prev = prev.prev_named_sibling + return False + + +def impl_type_name(node) -> str | None: + if node.child_by_field_name("trait") is not None: + return None + + type_node = node.child_by_field_name("type") + if type_node is None: + return None + + type_text = "".join(node_text(type_node).split()) + if "<" in type_text: + type_text = type_text.split("<", maxsplit=1)[0] + if "::" in type_text: + type_text = type_text.rsplit("::", maxsplit=1)[1] + return type_text or None + + +def add_struct_fields(idx: AstIndex, node) -> None: + body = node.child_by_field_name("body") + if body is None or body.type != "field_declaration_list": + return + + for child in body.named_children: + if child.type != "field_declaration": + continue + field_name_node = child.child_by_field_name("name") + if field_name_node is not None: + idx.struct_fields.add(node_text(field_name_node)) + + +def add_enum_variants(idx: AstIndex, node, enum_name: str) -> None: + body = node.child_by_field_name("body") + if body is None or body.type != "enum_variant_list": + return + + for child in body.named_children: + if child.type != "enum_variant": + continue + variant_name_node = child.child_by_field_name("name") + if variant_name_node is not None: + idx.enum_variants.add(f"{enum_name}::{node_text(variant_name_node)}") + + +def add_impl_methods(idx: AstIndex, node) -> None: + type_name = impl_type_name(node) + if type_name is None: + return + + body = node.child_by_field_name("body") + if body is None or body.type != "declaration_list": + return + + for child in body.named_children: + if child.type != "function_item": + continue + method_name_node = child.child_by_field_name("name") + if method_name_node is not None: + idx.impl_methods.add(f"{type_name}::{node_text(method_name_node)}") + + +def build_ast_index(source_code: bytes, source_name: str = "") -> AstIndex: + parser = rust_parser() + tree = parser.parse(source_code) + if tree.root_node.has_error: + raise ValueError(f"tree-sitter-rust parse error in {source_name}") + idx = AstIndex() + + def visit_item(node) -> None: + if node.type == "function_item": + name_node = node.child_by_field_name("name") + if name_node is not None: + name = node_text(name_node) + idx.functions.add(name) + if function_has_test_attribute(node): + idx.test_functions.add(name) + elif node.type == "struct_item": + name_node = node.child_by_field_name("name") + if name_node is not None: + idx.structs.add(node_text(name_node)) + add_struct_fields(idx, node) + elif node.type == "enum_item": + name_node = node.child_by_field_name("name") + if name_node is not None: + name = node_text(name_node) + idx.enums.add(name) + add_enum_variants(idx, node, name) + elif node.type == "impl_item": + add_impl_methods(idx, node) + elif node.type == "mod_item": + body = node.child_by_field_name("body") + if body is not None: + visit_item_scope(body) + + def visit_item_scope(node) -> None: + for child in node.named_children: + visit_item(child) + + visit_item_scope(tree.root_node) + return idx + + +def ref_exists_as_definition(ref: str, path: Path, idx: AstIndex) -> bool: + if path in (TUNER_TESTS, SPEC_TESTS): + return ref in idx.test_functions + if "::" not in ref: + return ( + ref in idx.functions + or ref in idx.structs + or ref in idx.enums + or ref in idx.struct_fields + ) + return ref in idx.enum_variants or ref in idx.impl_methods + + +def require_self_test(condition: bool, message: str) -> None: + if not condition: + raise RuntimeError(f"traceability self-test failed: {message}") + + +def run_self_tests() -> None: + source = b""" + #[test] + fn real_test_anchor() {} + + struct Settings { + peak_memory_bytes: u64, + } + + fn peak_memory_bytes() {} + + enum ObjectiveTarget { + PlannerDeployableModel, + } + + impl Settings { + fn from_json_value() {} + } + + trait Decoder { + fn trait_only() {} + } + + impl Decoder for Settings { + fn trait_only() {} + } + + fn use_settings() { + let x = Settings { peak_memory_bytes: 0 }; + } + """ + idx = build_ast_index(source) + require_self_test("real_test_anchor" in idx.test_functions, "failed to find #[test] function") + require_self_test("peak_memory_bytes" in idx.struct_fields, "failed to find struct field") + require_self_test("peak_memory_bytes" in idx.functions, "failed to find free function") + require_self_test( + "ObjectiveTarget::PlannerDeployableModel" in idx.enum_variants, + "failed to find enum variant", + ) + require_self_test( + "Settings::from_json_value" in idx.impl_methods, + "failed to find inherent impl method", + ) + require_self_test( + "Settings::trait_only" not in idx.impl_methods, + "trait impl method was accepted as inherent method", + ) + require_self_test( + "from_json_value" not in idx.functions, + "inherent impl method was accepted as free function", + ) + + source_no_def = b""" + fn use_settings() { + let x = Settings { peak_memory_bytes: 0 }; + } + """ + idx_no_def = build_ast_index(source_no_def) + require_self_test( + "peak_memory_bytes" not in idx_no_def.struct_fields, + "struct literal field initializer was accepted as a field definition", + ) + require_self_test( + not ref_exists_as_definition("peak_memory_bytes", TUNER, idx_no_def), + "struct literal field initializer satisfied plain anchor lookup", + ) + + +def main() -> int: + run_self_tests() + + doc = TRACEABILITY.read_text(encoding="utf-8") + + # Pre-parse indices + indices: dict[Path, AstIndex] = {} + for path in TUNER_SOURCES: + indices[path] = build_ast_index(path.read_bytes(), str(path.relative_to(ROOT))) + indices[WARMSTART] = build_ast_index(WARMSTART.read_bytes(), str(WARMSTART.relative_to(ROOT))) + indices[WARMSTART_CONTRACT] = build_ast_index( + WARMSTART_CONTRACT.read_bytes(), str(WARMSTART_CONTRACT.relative_to(ROOT)) + ) + indices[PLANNER_RUNTIME] = build_ast_index( + PLANNER_RUNTIME.read_bytes(), str(PLANNER_RUNTIME.relative_to(ROOT)) + ) + indices[TUNER_TESTS] = build_ast_index(TUNER_TESTS.read_bytes(), str(TUNER_TESTS.relative_to(ROOT))) + indices[SPEC_TESTS] = build_ast_index(SPEC_TESTS.read_bytes(), str(SPEC_TESTS.relative_to(ROOT))) + indices[SPEC_PARSER] = build_ast_index(SPEC_PARSER.read_bytes(), str(SPEC_PARSER.relative_to(ROOT))) + + tuner_aggregate = AstIndex() + for path in TUNER_SOURCES: + tuner_aggregate.merge(indices[path]) + + missing: list[str] = [] + for ref, path in REQUIRED_REFS: + if ref not in doc: + missing.append(f"{TRACEABILITY.relative_to(ROOT)} does not cite `{ref}`") + + idx = tuner_aggregate if path == TUNER else indices[path] + if path == TUNER: + target = "tuner implementation sources" + else: + target = str(path.relative_to(ROOT)) + if not ref_exists_as_definition(ref, path, idx): + missing.append(f"{target} do not define `{ref}`") + + if missing: + print("Tuner traceability check failed:", file=sys.stderr) + for item in missing: + print(f"- {item}", file=sys.stderr) + return 1 + + print(f"Checked {len(REQUIRED_REFS)} Tuner V1 traceability anchors.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/compare_bench_two_json.lua b/scripts/compare_bench_two_json.lua index d9a50b11..79f063a4 100755 --- a/scripts/compare_bench_two_json.lua +++ b/scripts/compare_bench_two_json.lua @@ -6,7 +6,7 @@ local function die(msg) end local CORE_OPERATIONS = { h = true, compress = true, decompress = true } -local CORE_SUBJECTS = { ppmd = true, ctw = true, rosa = true, rwkv = true, neural_mixture = true } +local CORE_SUBJECTS = { ppmd = true, ["fac-ctw"] = true, rosa = true, rwkv7 = true, neural_mixture = true } local CORE_SIZES = { ["1048576"] = true, ["4194304"] = true, ["10000000"] = true } local REQUIRED_COLUMNS = { @@ -16,6 +16,15 @@ local REQUIRED_COLUMNS = { "compression_backend", } +local OPTIONAL_PROVENANCE_COLUMNS = { + "suite_spec_path", + "suite_spec_sha256", + "build_mode", + "build_features", +} + +local LEGACY_UNKNOWN = "__legacy_unknown__" + local baseline_path, candidate_path do @@ -43,6 +52,17 @@ do end local SEP = "\0" +local EPS = 1e-12 + +local function canonicalize_subject(subject) + if subject == "rwkv" then + return "rwkv7" + end + if subject == "ctw" then + return "fac-ctw" -- this script is specific to the semantics of two.json, so this maintains support for comparison with old results. Because AC-CTW won't be used in the future for two.json, this is safe. + end + return subject +end local function chomp_cr(s) return (s:gsub("\r$", "")) @@ -79,6 +99,47 @@ local function make_key(operation, subject, size_bytes, compression_backend) return operation .. SEP .. subject .. SEP .. size_bytes .. SEP .. compression_backend end +local function format_compare_key(row) + return "operation=" .. (row._operation or "") + .. ", subject=" .. (row._subject or "") + .. ", size_bytes=" .. (row._size_bytes or "") + .. ", compression_backend=" .. (row._compression_backend or "") +end + +local function duplicate_row_message(path, headers, existing, duplicate) + local parts = { + "duplicate comparison row in " .. path, + "key: " .. format_compare_key(duplicate), + "first line: " .. tostring(existing._line), + "duplicate line: " .. tostring(duplicate._line), + } + + local diffs = {} + for _, h in ipairs(headers) do + if h ~= "" then + local a = existing[h] or "" + local b = duplicate[h] or "" + if a ~= b then + diffs[#diffs + 1] = h .. ": " .. a .. " != " .. b + end + end + end + + if #diffs > 0 then + parts[#parts + 1] = "differing columns: " .. table.concat(diffs, "; ") + end + + parts[#parts + 1] = + "comparison rows must be unique by operation, subject, size_bytes, and compression_backend" + if (existing.cpu or "") ~= (duplicate.cpu or "") then + parts[#parts + 1] = + "hint: this summary appears to mix CPU affinities; rerun with INFOTHEORY_BENCH_FRESH=1, " + .. "set one INFOTHEORY_BENCH_CPU, or compare a summary filtered to one CPU" + end + + return table.concat(parts, "\n") +end + local function load_rows(path) local f = io.open(path, "r") if not f then @@ -96,8 +157,10 @@ local function load_rows(path) validate_required_columns(header_index, path) local rows = {} + local line_number = 1 for line in f:lines() do + line_number = line_number + 1 if line ~= "" then local vals = split_tsv(line) local row = {} @@ -105,23 +168,29 @@ local function load_rows(path) for j, h in ipairs(headers) do row[h] = vals[j] or "" end + for _, name in ipairs(OPTIONAL_PROVENANCE_COLUMNS) do + if not header_index[name] then + row[name] = LEGACY_UNKNOWN + end + end local operation = row.operation or "" - local subject = row.subject or "" + local subject = canonicalize_subject(row.subject or "") + row.subject = subject local size_bytes = row.size_bytes or "" local compression_backend = row.compression_backend or "" local key = make_key(operation, subject, size_bytes, compression_backend) - if rows[key] then - die("duplicate row in " .. path .. ": " - .. operation .. "\t" .. subject .. "\t" .. size_bytes .. "\t" .. compression_backend) - end - row._operation = operation row._subject = subject row._size_bytes = size_bytes row._compression_backend = compression_backend + row._line = line_number + + if rows[key] then + die(duplicate_row_message(path, headers, rows[key], row)) + end rows[key] = row end @@ -131,6 +200,39 @@ local function load_rows(path) return rows end +local function collect_single_value(rows, path, field) + local seen = {} + for _, row in pairs(rows) do + local value = chomp_cr(row[field] or "") + if value ~= "" then + seen[value] = true + end + end + + local count, only = 0, nil + for value in pairs(seen) do + count = count + 1 + only = value + end + + if count == 0 then + die("missing required provenance value in " .. path .. ": " .. field) + end + if count > 1 then + die("multiple distinct provenance values in " .. path .. ": " .. field) + end + return only +end + +local function collect_provenance(rows, path) + return { + suite_spec_path = collect_single_value(rows, path, "suite_spec_path"), + suite_spec_sha256 = collect_single_value(rows, path, "suite_spec_sha256"), + build_mode = collect_single_value(rows, path, "build_mode"), + build_features = collect_single_value(rows, path, "build_features"), + } +end + local function num(row, field) local v = row[field] if not v or v:match("^%s*$") then @@ -164,16 +266,16 @@ local function compare(base, cand) local br, cr = num(base, "real_seconds_median"), num(cand, "real_seconds_median") if br and cr then local lim = math.max(br * 1.05, br + 0.02) - if cr > lim then - reasons[#reasons + 1] = ("real_seconds_median %.6g > %.6g"):format(cr, lim) + if cr > lim + EPS then + reasons[#reasons + 1] = ("real_seconds_median %.12g > %.12g"):format(cr, lim) end end local bm, cm = num(base, "rss_kib_median"), num(cand, "rss_kib_median") if bm and cm then local lim = math.max(bm * 1.03, bm + 4096.0) - if cm > lim then - reasons[#reasons + 1] = ("rss_kib_median %.6g > %.6g"):format(cm, lim) + if cm > lim + EPS then + reasons[#reasons + 1] = ("rss_kib_median %.12g > %.12g"):format(cm, lim) end end @@ -192,6 +294,18 @@ end local baseline_rows = load_rows(baseline_path) local candidate_rows = load_rows(candidate_path) +local baseline_provenance = collect_provenance(baseline_rows, baseline_path) +local candidate_provenance = collect_provenance(candidate_rows, candidate_path) + +if baseline_provenance.suite_spec_sha256 ~= candidate_provenance.suite_spec_sha256 then + if baseline_provenance.suite_spec_sha256 ~= LEGACY_UNKNOWN + and candidate_provenance.suite_spec_sha256 ~= LEGACY_UNKNOWN then + die("suite spec digest mismatch: baseline " + .. baseline_provenance.suite_spec_sha256 + .. " != candidate " + .. candidate_provenance.suite_spec_sha256) + end +end local key_set, keys = {}, {} for k in pairs(baseline_rows) do @@ -232,6 +346,14 @@ local full_warnings, core_failures = 0, 0 print("baseline\t" .. baseline_path) print("candidate\t" .. candidate_path) +print("baseline_suite_spec_path\t" .. baseline_provenance.suite_spec_path) +print("baseline_suite_spec_sha256\t" .. baseline_provenance.suite_spec_sha256) +print("baseline_build_mode\t" .. baseline_provenance.build_mode) +print("baseline_build_features\t" .. baseline_provenance.build_features) +print("candidate_suite_spec_path\t" .. candidate_provenance.suite_spec_path) +print("candidate_suite_spec_sha256\t" .. candidate_provenance.suite_spec_sha256) +print("candidate_build_mode\t" .. candidate_provenance.build_mode) +print("candidate_build_features\t" .. candidate_provenance.build_features) print("scope\tstatus\toperation\tsubject\tsize_bytes\tcompression_backend\treasons") for _, key in ipairs(keys) do diff --git a/scripts/ctw_profile_format.lua b/scripts/ctw_profile_format.lua new file mode 100755 index 00000000..b1261615 --- /dev/null +++ b/scripts/ctw_profile_format.lua @@ -0,0 +1,290 @@ +#!/usr/bin/env luajit + +local ok_cjson, cjson = pcall(require, "cjson") +if not ok_cjson then + io.stderr:write("ERROR: lua-cjson is required (module 'cjson' not found)\n") + os.exit(1) +end + +local function die(msg) + io.stderr:write("ERROR: " .. msg .. "\n") + os.exit(1) +end + +local function is_null(value) + return value == nil or value == cjson.null +end + +local function table_or_empty(value) + if is_null(value) then + return {} + end + return value +end + +local function strip_ansi(s) + return (s:gsub("\27%[[%d;?]*[ -/]*[@-~]", "")) +end + +local function trim(s) + return (s:gsub("^%s+", ""):gsub("%s+$", "")) +end + +local function comma_int(value) + if is_null(value) then + return "-" + end + local s = string.format("%.0f", value) + local sign = "" + if s:sub(1, 1) == "-" then + sign = "-" + s = s:sub(2) + end + local rev = s:reverse():gsub("(%d%d%d)", "%1,") + local out = rev:reverse():gsub("^,", "") + return sign .. out +end + +local function short_count(value) + if is_null(value) then + return "-" + end + local abs_value = math.abs(value) + if abs_value >= 1e9 then + return string.format("%.2fG", value / 1e9) + end + if abs_value >= 1e6 then + return string.format("%.2fM", value / 1e6) + end + if abs_value >= 1e3 then + return string.format("%.1fk", value / 1e3) + end + return string.format("%.0f", value) +end + +local function human_bytes(bytes) + if is_null(bytes) then + return "-" + end + local units = {"B", "KiB", "MiB", "GiB", "TiB"} + local value = bytes + local unit = 1 + while math.abs(value) >= 1024 and unit < #units do + value = value / 1024 + unit = unit + 1 + end + if unit == 1 then + return string.format("%.0f %s", value, units[unit]) + end + if math.abs(value) >= 100 then + return string.format("%.1f %s", value, units[unit]) + end + return string.format("%.2f %s", value, units[unit]) +end + +local function human_seconds(seconds) + if is_null(seconds) then + return "-" + end + if seconds < 60 then + return string.format("%.2fs", seconds) + end + local minutes = math.floor(seconds / 60) + local rest = seconds - minutes * 60 + if minutes < 60 then + return string.format("%dm%05.2fs", minutes, rest) + end + local hours = math.floor(minutes / 60) + minutes = minutes - hours * 60 + return string.format("%dh%02dm%05.2fs", hours, minutes, rest) +end + +local function percent(numer, denom) + if is_null(numer) or is_null(denom) or denom == 0 then + return "-" + end + return string.format("%.1f%%", 100.0 * numer / denom) +end + +local function number_or_nil(value) + if is_null(value) then + return nil + end + return value +end + +local function predicted_archive_bytes(snapshot) + local bits = number_or_nil(snapshot.bits) + if bits == nil then + return nil + end + return bits / 8.0 +end + +local function snapshot_row(snapshot) + local telemetry = table_or_empty(snapshot.telemetry) + local rss = table_or_empty(snapshot.rss) + local bpb = number_or_nil(snapshot.bits_per_byte) + local archive = predicted_archive_bytes(snapshot) + local hwm = number_or_nil(rss.vm_hwm_bytes) or number_or_nil(rss.vm_rss_bytes) + return string.format( + "%-10s %-10s %-10s %-11s %-11s %-11s %-10s %-10s %-10s %-9s %-9s %-7s", + short_count(snapshot.bytes_seen), + human_seconds(snapshot.elapsed_seconds), + bpb and string.format("%.6f", bpb) or "-", + archive and human_bytes(archive) or "-", + human_bytes(hwm), + human_bytes(telemetry.total_bytes), + short_count(telemetry.nodes_len), + short_count(telemetry.segments_len), + short_count(telemetry.segment_bits), + short_count(telemetry.history_segments), + short_count(telemetry.history_invert_segments), + telemetry.trees and tostring(#telemetry.trees) or "-" + ) +end + +local function print_snapshot_header() + print(string.format( + "%-10s %-10s %-10s %-11s %-11s %-11s %-10s %-10s %-10s %-9s %-9s %-7s", + "bytes", "elapsed", "bits/B", "ideal_out", "rss_hwm", "reserved", "nodes", "segments", "seg_bits", "histSeg", "invHist", "trees" + )) + print(string.rep("-", 132)) +end + +local function arena_payload_bytes(telemetry) + if not is_null(telemetry.tree_payload_bytes) then + return telemetry.tree_payload_bytes + end + local node_bytes = (telemetry.nodes_len or 0) * 32 + local segment_bytes = (telemetry.segments_len or 0) * 40 + return node_bytes + segment_bytes +end + +local function explicit_node_equivalent_bytes(telemetry) + local logical_nodes = (telemetry.nodes_len or 0) + (telemetry.segment_bits or 0) + return logical_nodes * 32 +end + +local function print_kv(label, value) + print(string.format(" %-30s %s", label .. ":", value)) +end + +local function print_summary(final, snapshots) + local telemetry = table_or_empty(final.telemetry) + local rss = table_or_empty(final.rss) + local archive = predicted_archive_bytes(final) + local logical_nodes = (telemetry.nodes_len or 0) + (telemetry.segment_bits or 0) + local payload_bytes = arena_payload_bytes(telemetry) + local explicit_bytes = explicit_node_equivalent_bytes(telemetry) + local capacity_slack = telemetry.tree_arena_slack_bytes + if is_null(capacity_slack) then + local capacity_arena_bytes = (telemetry.nodes_capacity or 0) * 32 + (telemetry.segments_capacity or 0) * 40 + capacity_slack = capacity_arena_bytes - payload_bytes + end + local saved_vs_explicit = explicit_bytes - payload_bytes + local history_payload_segments = (telemetry.history_segments or 0) + (telemetry.history_invert_segments or 0) + + print("") + print("Final CTW profile summary") + print(string.rep("=", 72)) + print_kv("snapshots", comma_int(snapshots)) + print_kv("mode", tostring(final.mode or "-")) + print_kv("base depth", tostring(final.depth or telemetry.base_depth or "-")) + print_kv("bytes seen", comma_int(final.bytes_seen)) + print_kv("elapsed", human_seconds(final.elapsed_seconds)) + if not is_null(final.bits_per_byte) then + print_kv("rate", string.format("%.9f bits/byte", final.bits_per_byte)) + end + if archive ~= nil then + print_kv("ideal archive payload", human_bytes(archive) .. " (" .. comma_int(archive) .. " bytes)") + end + print_kv("RSS current", human_bytes(rss.vm_rss_bytes)) + print_kv("RSS high-water", human_bytes(rss.vm_hwm_bytes)) + print_kv("telemetry reserved total", human_bytes(telemetry.total_bytes)) + print_kv("tree arena reserved", human_bytes(telemetry.tree_bytes)) + print_kv("shared history reserved", human_bytes(telemetry.shared_history_bytes)) + print_kv("shared history payload", human_bytes(telemetry.shared_history_payload_bytes)) + print_kv("shared history slack", human_bytes(telemetry.shared_history_slack_bytes)) + print_kv("shared log cache reserved", human_bytes(telemetry.shared_log_cache_bytes)) + print_kv("history length", comma_int(telemetry.shared_history_len_bits) .. " bits") + print_kv("history capacity", comma_int(telemetry.shared_history_capacity_bits) .. " bits") + print_kv("nodes", comma_int(telemetry.nodes_len) .. " / cap " .. comma_int(telemetry.nodes_capacity) .. " (" .. percent(telemetry.nodes_len, telemetry.nodes_capacity) .. " full)") + print_kv("segments", comma_int(telemetry.segments_len) .. " / cap " .. comma_int(telemetry.segments_capacity) .. " (" .. percent(telemetry.segments_len, telemetry.segments_capacity) .. " full)") + print_kv("arena payload at len", human_bytes(payload_bytes)) + print_kv("arena capacity slack", human_bytes(capacity_slack)) + print_kv("total allocator slack", human_bytes(telemetry.total_slack_bytes)) + print_kv("represented logical nodes", comma_int(logical_nodes)) + print_kv("explicit-node equivalent", human_bytes(explicit_bytes)) + print_kv("payload saved by segments", human_bytes(saved_vs_explicit)) + print_kv("exact segments", comma_int(telemetry.exact_segments)) + print_kv("history-anchor segments", comma_int(history_payload_segments)) + print_kv("const segments", comma_int(telemetry.const_segments)) + print_kv("segment bits", comma_int(telemetry.segment_bits)) + print_kv("max segment len", comma_int(telemetry.max_segment_len)) + + if history_payload_segments == 0 then + print_kv("history-anchor audit", "none observed") + else + print_kv("history-anchor audit", "present; bounded ring history is not automatically exact") + end + + if telemetry.trees ~= nil and #telemetry.trees > 0 then + print("") + print("Final per-tree arena") + print(string.rep("-", 116)) + print(string.format( + "%-4s %-6s %-10s %-10s %-8s %-10s %-10s %-8s %-10s %-8s %-8s", + "bit", "depth", "nodes", "node_cap", "nfull", "segments", "seg_cap", "sfull", "seg_bits", "maxseg", "histSeg" + )) + print(string.rep("-", 116)) + for _, tree in ipairs(telemetry.trees) do + local tree_hist = (tree.history_segments or 0) + (tree.history_invert_segments or 0) + print(string.format( + "%-4s %-6s %-10s %-10s %-8s %-10s %-10s %-8s %-10s %-8s %-8s", + tostring(tree.bit_index), + tostring(tree.max_depth), + short_count(tree.nodes_len), + short_count(tree.nodes_capacity), + percent(tree.nodes_len, tree.nodes_capacity), + short_count(tree.segments_len), + short_count(tree.segments_capacity), + percent(tree.segments_len, tree.segments_capacity), + short_count(tree.segment_bits), + tostring(tree.max_segment_len or "-"), + short_count(tree_hist) + )) + end + end +end + +local snapshots = 0 +local final = nil +local printed_header = false + +for raw_line in io.lines() do + local line = trim(strip_ansi(raw_line)) + if line ~= "" then + local ok, decoded = pcall(cjson.decode, line) + if not ok then + die("failed to decode JSONL line " .. tostring(snapshots + 1) .. ": " .. tostring(decoded)) + end + if decoded.kind ~= "ctw_profile_snapshot" then + die("line " .. tostring(snapshots + 1) .. " is not a ctw_profile_snapshot") + end + if not printed_header then + print_snapshot_header() + printed_header = true + end + snapshots = snapshots + 1 + final = decoded + print(snapshot_row(decoded)) + io.stdout:flush() + end +end + +if final == nil then + die("no ctw-profile JSONL snapshots were read from stdin") +end + +print_summary(final, snapshots) diff --git a/scripts/delegate_tuner_cgroup_v2.sh b/scripts/delegate_tuner_cgroup_v2.sh new file mode 100755 index 00000000..af3e3a56 --- /dev/null +++ b/scripts/delegate_tuner_cgroup_v2.sh @@ -0,0 +1,260 @@ +#!/bin/sh +set -eu + +die() { + echo "Error: $*" >&2 + exit 1 +} + +usage() { + cat <<'EOF' +Usage: + delegate_tuner_cgroup_v2.sh setup [cgroup-name] + delegate_tuner_cgroup_v2.sh run-in-session [cgroup-name] -- [args...] + +Commands: + setup + Root-only. Create/delegate a strict-mode cgroup-v2 subtree with this layout: + /sys/fs/cgroup// + session/ (for the long-lived tuner parent process) + evals/ (pass this to --evaluator-cgroup-parent; tuner creates per-eval children here) + + run-in-session + Root-only launcher helper. Runs an arbitrary command as , places + that process into /sys/fs/cgroup//session, then resumes it. + Use this when host delegation containment rules block unprivileged placement + of the initial process into the delegated subtree. + +Examples: + sudo ./scripts/delegate_tuner_cgroup_v2.sh setup theo infotheory-tuner + + sudo ./scripts/delegate_tuner_cgroup_v2.sh run-in-session theo infotheory-tuner -- \ + env INFOTHEORY_TUNER_EVAL_CGROUP_PARENT=/sys/fs/cgroup/infotheory-tuner/evals \ + cargo run -p infotheory --features "tuner cli backend-ctw" -- \ + tune spec.json --rss-mode hybrid_strict_max --max-evaluations 1 + +Security model: + - Root operations are limited to cgroup subtree setup/delegation and initial + process placement into the delegated session cgroup. + - Spec parsing/compression/scoring logic remains in the unprivileged + process. +EOF +} + +require_root() { + if [ "$(id -u)" -ne 0 ]; then + die "this helper must be run as root" + fi +} + +validate_user() { + user=$1 + if ! id "$user" >/dev/null 2>&1; then + die "user '$user' does not exist" + fi +} + +validate_name() { + name=$1 + case $name in + */*|.*|*..*|*:*|"") + die "cgroup-name must be a simple directory name" + ;; + esac +} + +ensure_cgroup_v2_root() { + root=$1 + [ -f "$root/cgroup.controllers" ] || die "$root is not a cgroup-v2 unified hierarchy" + grep -qw memory "$root/cgroup.controllers" || + die "cgroup-v2 memory controller is not available in $root/cgroup.controllers" +} + +enable_memory_subtree_control() { + node=$1 + subtree="$node/cgroup.subtree_control" + [ -f "$subtree" ] || die "missing $subtree" + if ! grep -qw memory "$subtree"; then + [ -w "$subtree" ] || die "cannot write $subtree to enable +memory" + printf '+memory\n' > "$subtree" || die "failed to enable +memory in $subtree" + fi + grep -qw memory "$subtree" || die "memory controller is not enabled in $subtree" +} + +delegate_node_to_user() { + node=$1 + owner_group=$2 + chown "$owner_group" "$node" || die "failed to chown directory '$node' to $owner_group" + chmod 0750 "$node" || die "failed to chmod 0750 '$node'" + # Delegate only the files required for cgroup-v2 subtree management. + for file in cgroup.procs cgroup.subtree_control cgroup.threads; do + if [ -e "$node/$file" ]; then + chown "$owner_group" "$node/$file" || die "failed to chown '$node/$file' to $owner_group" + fi + done +} + +run_as_user() { + user=$1 + shift + if command -v runuser >/dev/null 2>&1; then + runuser -u "$user" -- "$@" + else + su -s /bin/sh "$user" -c 'exec "$@"' sh "$@" + fi +} + +cmd_setup() { + user=$1 + name=$2 + root=/sys/fs/cgroup + base="$root/$name" + session="$base/session" + evals="$base/evals" + + require_root + validate_user "$user" + validate_name "$name" + ensure_cgroup_v2_root "$root" + + group="$(id -gn "$user")" || die "failed to resolve primary group for user '$user'" + owner_group="$user:$group" + + # Root operation: enable memory controller for children of /sys/fs/cgroup. + enable_memory_subtree_control "$root" + + # Root operation: create delegated subtree layout. + mkdir -p "$session" "$evals" + + # Root operation: enable memory controller where strict-mode evaluator child + # cgroups will be created. + enable_memory_subtree_control "$base" + enable_memory_subtree_control "$evals" + + # Root operation: grant delegatee ownership only on delegation files + dirs. + delegate_node_to_user "$base" "$owner_group" + delegate_node_to_user "$session" "$owner_group" + delegate_node_to_user "$evals" "$owner_group" + + cat < [args...] +EOF +} + +cmd_run_in_session() { + user=$1 + name=$2 + shift 2 + [ "${1:-}" = "--" ] || die "run-in-session requires '-- [args...]'" + shift + [ "$#" -gt 0 ] || die "run-in-session requires a command after '--'" + + require_root + validate_user "$user" + validate_name "$name" + + root=/sys/fs/cgroup + base="$root/$name" + session="$base/session" + evals="$base/evals" + [ -d "$base" ] || die "missing $base; run setup first" + [ -d "$session" ] || die "missing $session; run setup first" + [ -d "$evals" ] || die "missing $evals; run setup first" + [ -w "$session/cgroup.procs" ] || die "session cgroup is not writable: $session/cgroup.procs" + + tmp_root=${TMPDIR:-/tmp} + pid_file="$(mktemp "$tmp_root/infotheory-tuner-cgroup-pid.XXXXXX")" + wrapper_file="" + trap 'rm -f "$pid_file"; [ -n "$wrapper_file" ] && rm -f "$wrapper_file"' EXIT HUP INT TERM + wrapper_file="$(mktemp "$tmp_root/infotheory-tuner-cgroup-wrapper.XXXXXX")" + cat > "$wrapper_file" <<'WRAP' +#!/bin/sh +set -eu +pid_file=$1 +shift +printf '%s\n' "$$" > "$pid_file" +kill -s STOP "$$" +exec "$@" +WRAP + chmod 0700 "$wrapper_file" + chown "$user:$(id -gn "$user")" "$wrapper_file" "$pid_file" + + # Root operation: start unprivileged command and place it into delegated + # session cgroup before resuming. Invoke via /bin/sh so this works even on + # hosts that mount /tmp with noexec. + run_as_user "$user" /bin/sh "$wrapper_file" "$pid_file" "$@" & + launcher_pid=$! + + i=0 + while [ ! -s "$pid_file" ]; do + if ! kill -0 "$launcher_pid" 2>/dev/null; then + wait "$launcher_pid" || true + die "delegated command exited before PID capture (check command path/permissions)" + fi + i=$((i + 1)) + [ "$i" -le 200 ] || die "timed out waiting for delegated command PID capture" + sleep 0.05 + done + target_pid="$(cat "$pid_file")" + case $target_pid in + ""|*[!0-9]*) + die "captured invalid target PID: '$target_pid'" + ;; + esac + echo "$target_pid" > "$session/cgroup.procs" || + die "failed to place PID $target_pid into $session/cgroup.procs" + kill -s CONT "$target_pid" || die "failed to resume PID $target_pid" + wait "$launcher_pid" +} + +main() { + if [ "${1:-}" = "--help" ] || [ "${1:-}" = "-h" ]; then + usage + exit 0 + fi + + subcommand=${1:-setup} + case "$subcommand" in + setup) + shift || true + user=${1:-} + [ -n "$user" ] || { + usage >&2 + exit 2 + } + name=${2:-infotheory-tuner} + cmd_setup "$user" "$name" + ;; + run-in-session) + shift || true + user=${1:-} + [ -n "$user" ] || { + usage >&2 + exit 2 + } + shift || true + name=infotheory-tuner + if [ "${1:-}" != "--" ]; then + name=${1:-} + shift || true + fi + cmd_run_in_session "$user" "$name" "$@" + ;; + *) + usage >&2 + exit 2 + ;; + esac +} + +main "$@" diff --git a/scripts/legacy_aixi_convert.lua b/scripts/legacy_aixi_convert.lua new file mode 100755 index 00000000..bc505f24 --- /dev/null +++ b/scripts/legacy_aixi_convert.lua @@ -0,0 +1,1672 @@ +#!/usr/bin/env luajit + +local ok_cjson, cjson = pcall(require, "cjson") +if not ok_cjson then + io.stderr:write("ERROR: lua-cjson is required (module 'cjson' not found)\n") + os.exit(1) +end + +-- Keep JSON number output stable and compact for human-facing configs. +cjson.encode_number_precision(14) + +local function die(msg) + io.stderr:write("ERROR: " .. msg .. "\n") + os.exit(1) +end + +local function trim(s) + return (s:gsub("^%s+", ""):gsub("%s+$", "")) +end + +local function lower(s) + if s == nil then + return "" + end + return string.lower(trim(tostring(s))) +end + +local function dirname(path) + local idx = path:match("^.*()/") + if not idx then + return "." + end + if idx == 1 then + return "/" + end + return path:sub(1, idx - 1) +end + +local function path_join(base, rel) + if rel:sub(1, 1) == "/" then + return rel + end + if base == "" or base == "." then + return rel + end + if base:sub(-1) == "/" then + return base .. rel + end + return base .. "/" .. rel +end + +local function file_exists(path) + local f = io.open(path, "rb") + if f then + f:close() + return true + end + return false +end + +local function read_file(path) + local f, err = io.open(path, "rb") + if not f then + die("cannot open '" .. path .. "': " .. tostring(err)) + end + local content = f:read("*a") + f:close() + return content +end + +local function read_stdin() + local chunks = {} + while true do + local chunk = io.read(8192) + if chunk == nil then + break + end + chunks[#chunks + 1] = chunk + end + return table.concat(chunks) +end + +local function as_int(value, field) + if value == nil then + return nil + end + local n = tonumber(value) + if not n then + die("field '" .. field .. "' must be numeric") + end + if n ~= n or n == math.huge or n == -math.huge then + die("field '" .. field .. "' must be finite") + end + if n >= 0 then + return math.floor(n) + end + return math.ceil(n) +end + +local function as_num(value, field) + if value == nil then + return nil + end + local n = tonumber(value) + if not n then + die("field '" .. field .. "' must be numeric") + end + if n ~= n or n == math.huge or n == -math.huge then + die("field '" .. field .. "' must be finite") + end + return n +end + +local function as_bool(value) + return value == true +end + +local function int_with_default(value, field, default_value) + local n = as_int(value, field) + if n == nil then + return default_value + end + return n +end + +local function int_with_default_min(value, field, default_value, min_value) + local n = int_with_default(value, field, default_value) + if n < min_value then + return min_value + end + return n +end + +local function int_with_default_nonnegative(value, field, default_value) + local n = int_with_default(value, field, default_value) + if n < 0 then + return 0 + end + return n +end + +local function nonnegative_num_with_default(value, field, default_value) + local n = as_num(value, field) + if n == nil then + return default_value + end + if n < 0 then + return 0 + end + return n +end + +local function positive_num_with_default(value, field, default_value) + local n = as_num(value, field) + if n == nil then + return default_value + end + if n <= 0 then + return default_value + end + return n +end + +local function closed_unit_num_with_default(value, field, default_value) + local n = as_num(value, field) + if n == nil then + return default_value + end + if n < 0 then + return 0 + end + if n > 1 then + return 1 + end + return n +end + +local function open_unit_num_with_default(value, field, default_value) + local n = as_num(value, field) + if n == nil then + return default_value + end + if n <= 0 then + return default_value + end + if n > 1 then + return 1 + end + return n +end + +local function next_power_of_two(n) + local v = int_with_default_min(n, "return_bins", 1, 1) + local p = 1 + while p < v do + p = p * 2 + end + return p +end + +local function deep_copy(value) + if type(value) ~= "table" then + return value + end + local out = {} + for k, v in pairs(value) do + out[k] = deep_copy(v) + end + return out +end + +local function normalize_rate_backend_name(name) + local aliases = { + ["rosaplus"] = "rosaplus", + ["rosa"] = "rosaplus", + ["ctw"] = "ctw", + ["ac-ctw"] = "ctw", + ["ctw-context-tree"] = "ctw", + ["fac-ctw"] = "fac-ctw", + ["facctw"] = "fac-ctw", + ["zpaq"] = "zpaq", + ["mamba"] = "mamba", + ["mamba1"] = "mamba", + ["rwkv7"] = "rwkv7", + ["rwkv"] = "rwkv7", + ["match"] = "match", + ["sparse-match"] = "sparse-match", + ["sparse_match"] = "sparse-match", + ["sparsematch"] = "sparse-match", + ["ppmd"] = "ppmd", + ["ppm"] = "ppmd", + ["sequitur"] = "sequitur", + ["mixture"] = "mixture", + ["mix"] = "mixture", + ["particle"] = "particle", + ["particles"] = "particle", + ["calibrated"] = "calibrated", + ["cal"] = "calibrated", + } + local norm = aliases[lower(name)] + if not norm then + die("unknown legacy rate backend '" .. tostring(name) .. "'") + end + return norm +end + +local function signed_reward_bounds(bits) + if bits <= 0 then + return 0, 0 + end + if bits >= 63 then + return -9223372036854775808, 9223372036854775807 + end + local max = (2 ^ (bits - 1)) - 1 + local min = -(2 ^ (bits - 1)) + return min, max +end + +local function parse_observation_key_mode_str(raw) + local s = lower(raw or "full") + if s == "full" or s == "full-stream" or s == "stream" or s == "full_stream" then + return "full_stream" + end + if s == "last" then + return "last" + end + if s == "hash" or s == "stream-hash" or s == "stream_hash" then + return "stream_hash" + end + return "first" +end + +local function parse_observation_stream_len(value) + return int_with_default_min(value.observation_stream_len, "observation_stream_len", 1, 1) +end + +local function parse_vm_observation_stream_len(value) + if type(value) ~= "table" then + return 1 + end + local stream_len = as_int(value.stream_len, "vm_observation.stream_len") + or as_int(value.observation_stream_len, "vm_observation.observation_stream_len") + or 1 + if stream_len < 1 then + return 1 + end + return stream_len +end + +local function parse_observation_stream_len_for_env(root, env_name) + if env_name == "vm" then + if type(root.vm_observation) == "table" then + return parse_vm_observation_stream_len(root.vm_observation) + end + return parse_observation_stream_len(root) + end + return parse_observation_stream_len(root) +end + +local function parse_observation_key_mode_for_env(root, env_name) + if env_name == "vm" then + if type(root.vm_observation) == "table" then + local vm_obs = root.vm_observation + local mode = vm_obs.key_mode or vm_obs.observation_key_mode or "full" + return parse_observation_key_mode_str(mode) + end + return parse_observation_key_mode_str(root.observation_key_mode or "full") + end + return parse_observation_key_mode_str(root.observation_key_mode or "full") +end + +local function decode_hex_bytes(text, field) + if #text % 2 ~= 0 then + die(field .. " contains odd-length hex payload") + end + local out = {} + for i = 1, #text, 2 do + local chunk = text:sub(i, i + 1) + local value = tonumber(chunk, 16) + if value == nil then + die(field .. " contains invalid hex payload") + end + out[#out + 1] = string.char(value) + end + return table.concat(out) +end + +local function encode_hex_bytes(bytes) + return (bytes:gsub('.', function(c) + return string.format('%02x', string.byte(c)) + end)) +end + +local function decode_legacy_payload(text, encoding, field) + local value = text or "" + local enc = lower(encoding or "utf8") + if enc == "hex" then + return decode_hex_bytes(value, field) + end + return value +end + +local function resolve_read_path(base_dir, path) + if path:sub(1, 1) == "/" then + return path + end + local candidate = path_join(base_dir, path) + if file_exists(candidate) then + return candidate + end + return path +end + +local function as_table(value, field) + if type(value) ~= "table" then + die("field '" .. field .. "' must be an object") + end + return value +end + +local function default_vm_stats_backend(root) + local algo = lower(root.algorithm or "ctw") + local ct_depth = int_with_default_min(root.ct_depth, "ct_depth", 20, 1) + + if algo == "ctw" or algo == "ac-ctw" or algo == "ctw-context-tree" then + return { kind = "ctw", depth = ct_depth } + end + + if algo == "fac-ctw" then + return { + kind = "fac-ctw", + base_depth = ct_depth, + num_percept_bits = 8, + encoding_bits = 8, + } + end + + if algo == "sequitur" then + return { + kind = "sequitur", + context_bytes = int_with_default_min(root.context_bytes, "context_bytes", 64, 1), + } + end + + if algo == "mamba" or algo == "mamba1" then + local method = root.mamba_method + if method ~= nil then + return { kind = "mamba", method = method } + end + local model_path = root.mamba_model_path + if not model_path then + die("legacy default mamba backend requires mamba_model_path") + end + return { kind = "mamba", model_path = model_path } + end + + if algo == "rwkv" or algo == "rwkv7" then + local method = root.rwkv_method + if method ~= nil then + return { kind = "rwkv7", method = method } + end + local model_path = root.rwkv_model_path + if not model_path then + die("legacy default rwkv backend requires rwkv_model_path") + end + return { kind = "rwkv7", model_path = model_path } + end + + if algo == "zpaq" then + return { + kind = "zpaq", + method = tostring(root.method or "2"), + } + end + + if algo == "mixture" or algo == "mix" then + local spec_path = root.mixture_spec + if not spec_path then + die("legacy default mixture backend requires mixture_spec") + end + return { + kind = "mixture", + spec_path = spec_path, + } + end + + if algo == "rosa" or algo == "rosaplus" then + return { kind = "rosaplus" } + end + + return { kind = "rosaplus" } +end + +local function backend_from_legacy_cfg(cfg, root, base_dir, observation_bits, reward_bits) + if cfg == nil then + return nil + end + + local cfg_type = type(cfg) + local cfg_obj + if cfg_type == "string" then + cfg_obj = { name = cfg } + elseif cfg_type == "table" then + cfg_obj = cfg + else + die("legacy backend override must be an object or string") + end + + local raw_name = cfg_obj.name or cfg_obj.rate_backend or cfg_obj.kind or (cfg_type == "string" and cfg or nil) or "rosaplus" + local name = normalize_rate_backend_name(raw_name) + + if name == "rosaplus" then + return { kind = "rosaplus" } + end + + if name == "ctw" then + return { + kind = "ctw", + depth = int_with_default_min( + cfg_obj.ct_depth ~= nil and cfg_obj.ct_depth or cfg_obj.depth, + "rate_backend.depth", + 32, + 1 + ), + } + end + + if name == "fac-ctw" then + local encoding_bits = int_with_default_min(cfg_obj.encoding_bits, "rate_backend.encoding_bits", 8, 1) + local percept_bits = as_int(cfg_obj.num_percept_bits, "rate_backend.num_percept_bits") + or (observation_bits + reward_bits) + if percept_bits < 1 then + percept_bits = 1 + end + return { + kind = "fac-ctw", + base_depth = int_with_default_min( + cfg_obj.base_depth ~= nil and cfg_obj.base_depth or cfg_obj.ct_depth, + "rate_backend.base_depth", + 32, + 1 + ), + num_percept_bits = percept_bits, + encoding_bits = encoding_bits, + } + end + + if name == "mamba" then + local method = cfg_obj.method or cfg_obj.mamba_method + if method ~= nil then + return { kind = "mamba", method = method } + end + local model_path = cfg_obj.mamba_model_path or cfg_obj.model_path or root.mamba_model_path + if not model_path then + die("legacy mamba backend requires method/model_path") + end + return { kind = "mamba", model_path = model_path } + end + + if name == "rwkv7" then + local method = cfg_obj.method or cfg_obj.rwkv_method + if method ~= nil then + return { kind = "rwkv7", method = method } + end + local model_path = cfg_obj.rwkv_model_path or cfg_obj.model_path or root.rwkv_model_path + if not model_path then + die("legacy rwkv backend requires method/model_path") + end + return { kind = "rwkv7", model_path = model_path } + end + + if name == "zpaq" then + return { + kind = "zpaq", + method = tostring(cfg_obj.method or cfg_obj.zpaq_method or root.method or "2"), + } + end + + if name == "match" then + local min_len = int_with_default_min(cfg_obj.min_len, "rate_backend.min_len", 4, 1) + local max_len = int_with_default_min(cfg_obj.max_len, "rate_backend.max_len", 255, 1) + if max_len < min_len then + max_len = min_len + end + return { + kind = "match", + hash_bits = int_with_default_min(cfg_obj.hash_bits, "rate_backend.hash_bits", 20, 1), + min_len = min_len, + max_len = max_len, + base_mix = nonnegative_num_with_default(cfg_obj.base_mix, "rate_backend.base_mix", 0.02), + confidence_scale = positive_num_with_default(cfg_obj.confidence_scale, "rate_backend.confidence_scale", 1.0), + } + end + + if name == "sparse-match" then + local min_len = int_with_default_min(cfg_obj.min_len, "rate_backend.min_len", 3, 1) + local max_len = int_with_default_min(cfg_obj.max_len, "rate_backend.max_len", 64, 1) + if max_len < min_len then + max_len = min_len + end + return { + kind = "sparse-match", + hash_bits = int_with_default_min(cfg_obj.hash_bits, "rate_backend.hash_bits", 19, 1), + min_len = min_len, + max_len = max_len, + gap_min = int_with_default_min(cfg_obj.gap_min, "rate_backend.gap_min", 1, 0), + gap_max = int_with_default_min(cfg_obj.gap_max, "rate_backend.gap_max", 2, 0), + base_mix = nonnegative_num_with_default(cfg_obj.base_mix, "rate_backend.base_mix", 0.05), + confidence_scale = positive_num_with_default(cfg_obj.confidence_scale, "rate_backend.confidence_scale", 1.0), + } + end + + if name == "ppmd" then + return { + kind = "ppmd", + order = int_with_default_min(cfg_obj.order, "rate_backend.order", 10, 1), + memory_mb = int_with_default_min(cfg_obj.memory_mb, "rate_backend.memory_mb", 64, 1), + } + end + + if name == "sequitur" then + return { + kind = "sequitur", + context_bytes = int_with_default_min(cfg_obj.context_bytes, "rate_backend.context_bytes", 64, 1), + } + end + + if name == "mixture" then + if type(cfg_obj.spec) == "table" then + return { kind = "mixture", spec = deep_copy(cfg_obj.spec) } + end + local spec_path = cfg_obj.mixture_spec or cfg_obj.spec_path or cfg_obj.path + if type(cfg_obj.spec) == "string" and spec_path == nil then + spec_path = cfg_obj.spec + end + if spec_path == nil then + spec_path = root.mixture_spec + end + if not spec_path then + die("legacy mixture backend requires inline spec or mixture_spec/spec_path") + end + return { kind = "mixture", spec_path = spec_path } + end + + if name == "particle" then + if type(cfg_obj.spec) == "table" then + return { kind = "particle", spec = deep_copy(cfg_obj.spec) } + end + local spec_path = cfg_obj.particle_spec or cfg_obj.spec_path or cfg_obj.path + if type(cfg_obj.spec) == "string" and spec_path == nil then + spec_path = cfg_obj.spec + end + if spec_path == nil then + spec_path = root.particle_spec + end + if spec_path then + return { kind = "particle", spec_path = spec_path } + end + local inline = deep_copy(cfg_obj) + inline.kind = "particle" + inline.name = nil + inline.rate_backend = nil + return inline + end + + if name == "calibrated" then + if type(cfg_obj.spec) == "table" then + return { kind = "calibrated", spec = deep_copy(cfg_obj.spec) } + end + local spec_path = cfg_obj.calibrated_spec or cfg_obj.spec_path or cfg_obj.path + if type(cfg_obj.spec) == "string" and spec_path == nil then + spec_path = cfg_obj.spec + end + if spec_path == nil then + spec_path = root.calibrated_spec + end + if spec_path then + return { kind = "calibrated", spec_path = spec_path } + end + local inline = deep_copy(cfg_obj) + inline.kind = "calibrated" + inline.name = nil + inline.rate_backend = nil + return inline + end + + die("unsupported legacy backend kind '" .. tostring(raw_name) .. "'") +end + +local function parse_vm_observation_policy(mode) + local m = lower(mode or "guest") + if m == "raw" or m == "raw-bytes" or m == "bytes" or m == "stream" then + return "raw_output" + end + if m == "hash" or m == "output-hash" then + return "output_hash" + end + if m == "shared-memory" or m == "shared_mem" or m == "shared" then + return "shared_memory" + end + return "from_guest" +end + +local function parse_vm_observation_stream_mode(mode) + local m = lower(mode or "pad-truncate") + if m == "pad" then + return "pad" + end + if m == "truncate" then + return "truncate" + end + return "pad_truncate" +end + +local function parse_payload_encoding(mode) + local m = lower(mode or "utf8") + if m == "hex" then + return "hex" + end + return "utf8" +end + +local function parse_wire_encoding(mode) + local m = lower(mode or "hex") + if m == "utf8" or m == "text" then + return "utf8" + end + if m == "hex" then + return "hex" + end + return "hex" +end + +local function parse_vm_fuzz_mutator(name) + local n = lower(name) + if n == "flip_bit" or n == "flipbit" then + return "flip_bit" + end + if n == "flip_byte" or n == "flipbyte" then + return "flip_byte" + end + if n == "insert" or n == "insert_byte" or n == "insertbyte" then + return "insert_byte" + end + if n == "delete" or n == "delete_byte" or n == "deletebyte" then + return "delete_byte" + end + if n == "splice" or n == "splice_seed" or n == "splice-seed" then + return "splice_seed" + end + if n == "reset" or n == "reset_seed" or n == "reset-seed" then + return "reset_seed" + end + if n == "havoc" then + return "havoc" + end + return nil +end + +local function make_asset_registry() + local assets = {} + local by_path = {} + local used = {} + + local function sanitize(name) + local out = tostring(name or "asset") + out = out:gsub("[^%w_%-]+", "_") + out = out:gsub("_+", "_") + out = out:gsub("^_+", "") + out = out:gsub("_+$", "") + if out == "" then + out = "asset" + end + return out + end + + local function add(path, hint) + if path == nil then + return nil + end + local key = tostring(path) + local existing = by_path[key] + if existing then + return existing + end + local base = sanitize(hint) + local id = base + local i = 2 + while used[id] do + id = base .. "_" .. i + i = i + 1 + end + used[id] = true + by_path[key] = id + assets[#assets + 1] = { + id = id, + path = key, + } + return id + end + + local function list() + return assets + end + + return { + add = add, + list = list, + } +end + +local function convert_legacy_vm_action_source(raw, base_dir) + local source = type(raw) == "table" and raw or {} + local mode = lower(source.mode or "literal") + + if mode == "fuzz" then + local fuzz = type(source.fuzz) == "table" and source.fuzz or source + local seed_encoding = parse_payload_encoding(fuzz.seed_encoding) + local dict_encoding = parse_payload_encoding(fuzz.dict_encoding) + + local seeds = {} + if type(fuzz.seed_paths) == "table" then + for idx, path in ipairs(fuzz.seed_paths) do + if type(path) == "string" then + local resolved = resolve_read_path(base_dir, path) + local bytes = read_file(resolved) + seeds[#seeds + 1] = encode_hex_bytes(bytes) + else + die("vm_actions.fuzz.seed_paths[" .. idx .. "] must be a string") + end + end + end + if type(fuzz.seed_inputs) == "table" then + for idx, text in ipairs(fuzz.seed_inputs) do + if type(text) ~= "string" then + die("vm_actions.fuzz.seed_inputs[" .. idx .. "] must be a string") + end + local bytes = decode_legacy_payload(text, seed_encoding, "vm_actions.fuzz.seed_inputs") + seeds[#seeds + 1] = encode_hex_bytes(bytes) + end + end + + local mutators = {} + if type(fuzz.mutators) == "table" then + for _, name in ipairs(fuzz.mutators) do + if type(name) == "string" then + local canonical = parse_vm_fuzz_mutator(name) + if canonical ~= nil then + mutators[#mutators + 1] = canonical + end + end + end + end + + local dictionary = {} + if type(fuzz.dictionary) == "table" then + for idx, text in ipairs(fuzz.dictionary) do + if type(text) ~= "string" then + die("vm_actions.fuzz.dictionary[" .. idx .. "] must be a string") + end + local bytes = decode_legacy_payload(text, dict_encoding, "vm_actions.fuzz.dictionary") + dictionary[#dictionary + 1] = encode_hex_bytes(bytes) + end + end + + if #mutators == 0 then + mutators[1] = "havoc" + end + + local min_len = int_with_default_min(fuzz.min_len, "vm_actions.fuzz.min_len", 1, 1) + local max_len = int_with_default_min(fuzz.max_len, "vm_actions.fuzz.max_len", 4096, 1) + if max_len < min_len then + max_len = min_len + end + local rng_seed = int_with_default_nonnegative(fuzz.rng_seed, "vm_actions.fuzz.rng_seed", 0) + + if #seeds == 0 then + seeds[1] = "" + end + + return { + kind = "fuzz", + seeds = seeds, + encoding = "hex", + mutators = mutators, + min_len = min_len, + max_len = max_len, + dictionary = dictionary, + rng_seed = rng_seed, + } + end + + local actions = {} + local raw_actions = source.actions + if type(raw_actions) == "table" then + for idx, entry in ipairs(raw_actions) do + if type(entry) == "string" then + local bytes = decode_legacy_payload(entry, "utf8", "vm_actions.actions") + actions[#actions + 1] = { payload = encode_hex_bytes(bytes) } + elseif type(entry) == "table" then + local payload = entry.payload + if payload ~= nil and type(payload) ~= "string" then + die("vm_actions.actions[" .. idx .. "].payload must be a string") + end + local bytes = decode_legacy_payload(payload or "", entry.encoding, "vm_actions.actions") + local action = { payload = encode_hex_bytes(bytes) } + if entry.name ~= nil then + action.name = tostring(entry.name) + end + actions[#actions + 1] = action + else + die("vm_actions.actions[" .. idx .. "] must be a string or object") + end + end + end + + if #actions == 0 then + actions[1] = { payload = "" } + end + + return { + kind = "literal", + encoding = "hex", + actions = actions, + } +end + +local function convert_legacy_vm_environment(root, base_dir, assets) + local vm = as_table(root.vm_config, "vm_config") + + local observation_bits = int_with_default_min(root.observation_bits, "observation_bits", 16, 1) + local reward_bits = int_with_default_min(root.reward_bits, "reward_bits", 8, 1) + local agent_horizon = int_with_default_min(root.agent_horizon, "agent_horizon", 3, 1) + + local firecracker_path = vm.firecracker_config or vm.config or root.firecracker_config + if firecracker_path == nil then + die("legacy vm config requires vm_config.firecracker_config") + end + if type(firecracker_path) ~= "string" then + die("vm_config.firecracker_config must be a string") + end + local firecracker_asset = assets.add(firecracker_path, "firecracker") + + local vm_observation = type(vm.observation) == "table" and vm.observation + or (type(root.vm_observation) == "table" and root.vm_observation) + or nil + + local vm_reward = type(vm.reward) == "table" and vm.reward + or (type(root.vm_reward) == "table" and root.vm_reward) + or {} + + local vm_actions = type(vm.actions) == "table" and vm.actions + or (type(root.vm_actions) == "table" and root.vm_actions) + or {} + + local vm_trace = type(vm.trace) == "table" and vm.trace + or (type(root.vm_trace) == "table" and root.vm_trace) + or nil + + local vm_filter = type(vm.filter) == "table" and vm.filter + or (type(root.vm_filter) == "table" and root.vm_filter) + or nil + + local protocol = type(vm.protocol) == "table" and vm.protocol + or (type(root.vm_protocol) == "table" and root.vm_protocol) + or {} + + local action_source = convert_legacy_vm_action_source(vm_actions, base_dir) + local action_count + if action_source.kind == "literal" then + action_count = #action_source.actions + else + action_count = #action_source.mutators + end + + local reward_policy + do + local reward_mode = lower(vm_reward.mode or "guest") + if reward_mode == "pattern" then + local pattern = vm_reward.pattern + if type(pattern) ~= "string" then + die("vm_reward.pattern is required when vm_reward.mode=pattern") + end + reward_policy = { + kind = "pattern", + pattern = pattern, + base_reward = as_int(vm_reward.base_reward, "vm_reward.base_reward") or 0, + bonus_reward = as_int(vm_reward.bonus_reward, "vm_reward.bonus_reward") or 10, + } + else + reward_policy = { kind = "from_guest" } + end + end + + local reward_shaping + do + local shaping = nil + if type(vm.reward_shaping) == "table" then + shaping = vm.reward_shaping + elseif type(root.vm_reward_shaping) == "table" then + shaping = root.vm_reward_shaping + elseif type(vm.reward) == "table" and type(vm.reward.shaping) == "table" then + shaping = vm.reward.shaping + end + + if shaping ~= nil then + local mode = lower(shaping.mode or "none") + if mode == "entropy-reduction" or mode == "entropy_reduction" then + local baseline_path = shaping.baseline_path + if type(baseline_path) ~= "string" then + die("vm_reward_shaping.baseline_path is required in entropy-reduction mode") + end + reward_shaping = { + kind = "entropy_reduction", + baseline_asset = assets.add(baseline_path, "baseline"), + max_order = as_int(shaping.max_order, "vm_reward_shaping.max_order") or 8, + scale = as_num(shaping.scale, "vm_reward_shaping.scale") or 10.0, + crash_bonus = as_int(shaping.crash_bonus, "vm_reward_shaping.crash_bonus"), + timeout_bonus = as_int(shaping.timeout_bonus, "vm_reward_shaping.timeout_bonus"), + } + elseif mode == "trace-entropy" or mode == "trace_entropy" then + reward_shaping = { + kind = "trace_entropy", + max_order = as_int(shaping.max_order, "vm_reward_shaping.max_order") or 8, + scale = as_num(shaping.scale, "vm_reward_shaping.scale") or 1.0, + normalize = as_bool(shaping.normalize), + } + end + end + end + + local action_filter + if vm_filter ~= nil then + local novelty_prior_asset + if type(vm_filter.novelty_prior_path) == "string" then + novelty_prior_asset = assets.add(vm_filter.novelty_prior_path, "novelty_prior") + end + local reject_reward = as_int(vm_filter.reject_reward, "vm_filter.reject_reward") + if reject_reward == nil then + reject_reward = -(as_int(vm.step_cost, "vm_config.step_cost") or 1) + end + action_filter = { + min_entropy = as_num(vm_filter.min_entropy, "vm_filter.min_entropy"), + max_entropy = as_num(vm_filter.max_entropy, "vm_filter.max_entropy"), + min_intrinsic_dependence = as_num(vm_filter.min_intrinsic_dependence, "vm_filter.min_intrinsic_dependence"), + min_novelty = as_num(vm_filter.min_novelty, "vm_filter.min_novelty"), + novelty_prior_asset = novelty_prior_asset, + max_order = as_int(vm_filter.max_order, "vm_filter.max_order") or 8, + reject_reward = reject_reward, + } + end + + local trace + if vm_trace ~= nil then + local shared_region_name = vm_trace.shared_region_name + or vm_trace.shared_region + or vm_trace.name + or ((lower(vm_trace.mode) == "shared-memory") and "trace" or nil) + or "trace" + trace = { + shared_region_name = shared_region_name, + max_bytes = as_int(vm_trace.max_bytes, "vm_trace.max_bytes") or 1000000, + reset_on_episode = as_bool(vm_trace.reset_on_episode), + } + end + + local stats_backend = backend_from_legacy_cfg( + vm.stats_backend or root.vm_stats_backend, + root, + base_dir, + observation_bits, + reward_bits + ) + if stats_backend == nil then + stats_backend = default_vm_stats_backend(root) + end + + local environment = { + kind = "nyx_vm", + firecracker_config_asset = firecracker_asset, + instance_id = tostring(vm.instance_id or "aixi-nyx"), + shared_region_name = tostring(vm.shared_region_name or "shared"), + shared_region_size = int_with_default_min(vm.shared_region_size, "vm_config.shared_region_size", 4096, 1), + shared_memory_policy = (function() + local policy = lower(vm.shared_memory_policy or root.shared_memory_policy or "snapshot") + if policy == "preserve" or policy == "keep" then + return "preserve" + end + return "snapshot" + end)(), + step_timeout_ms = int_with_default_min(vm.step_timeout_ms, "vm_config.step_timeout_ms", 100, 1), + boot_timeout_ms = int_with_default_min(vm.boot_timeout_ms, "vm_config.boot_timeout_ms", 30000, 1), + episode_steps = int_with_default_min(vm.episode_steps, "vm_config.episode_steps", agent_horizon, 1), + step_cost = as_int(vm.step_cost, "vm_config.step_cost") or 1, + observation_policy = parse_vm_observation_policy(vm_observation and vm_observation.mode or nil), + observation_bits = observation_bits, + observation_stream_len = parse_vm_observation_stream_len(vm_observation), + observation_stream_mode = parse_vm_observation_stream_mode(vm_observation and vm_observation.stream_mode or nil), + observation_pad_byte = as_int(vm_observation and vm_observation.pad_byte or nil, "vm_observation.pad_byte") or 0, + reward_bits = reward_bits, + reward_policy = reward_policy, + reward_shaping = reward_shaping, + action_source = action_source, + action_filter = action_filter, + protocol = { + action_prefix = tostring(protocol.action_prefix or "ACT "), + action_suffix = tostring(protocol.action_suffix or "\n"), + obs_prefix = tostring(protocol.obs_prefix or "OBS "), + rew_prefix = tostring(protocol.rew_prefix or "REW "), + done_prefix = tostring(protocol.done_prefix or "DONE "), + data_prefix = tostring(protocol.data_prefix or "DATA "), + wire_encoding = parse_wire_encoding(protocol.wire_encoding), + }, + stats_backend = stats_backend, + trace = trace, + debug_mode = as_bool(vm.verbose) or as_bool(vm.debug), + crash_log = (type(vm.crash_log) == "string") and vm.crash_log or nil, + } + + return environment, { + observation_bits = observation_bits, + reward_bits = reward_bits, + action_count = action_count, + } +end + +local BUILTIN_ENV = { + ["coin-flip"] = "coin_flip", + ["coin_flip"] = "coin_flip", + ["coinflip"] = "coin_flip", + ["ctw-test"] = "ctw_test", + ["ctw_test"] = "ctw_test", + ["tictactoe"] = "tic_tac_toe", + ["tic-tac-toe"] = "tic_tac_toe", + ["tic_tac_toe"] = "tic_tac_toe", + ["extended-tiger"] = "extended_tiger", + ["extended_tiger"] = "extended_tiger", + ["biased-rock-paper-scissor"] = "biased_rock_paper_scissor", + ["biased-rock-paper-scissors"] = "biased_rock_paper_scissor", + ["biased_rock_paper_scissor"] = "biased_rock_paper_scissor", + ["biased_rock_paper_scissors"] = "biased_rock_paper_scissor", + ["kuhn-poker"] = "kuhn_poker", + ["kuhn_poker"] = "kuhn_poker", +} + +local BUILTIN_DEFAULTS = { + coin_flip = { + observation_bits = 1, + reward_bits = 1, + agent_actions = 2, + min_reward = 0, + max_reward = 1, + }, + ctw_test = { + observation_bits = 1, + reward_bits = 1, + agent_actions = 2, + min_reward = 0, + max_reward = 1, + }, + biased_rock_paper_scissor = { + observation_bits = 2, + reward_bits = 2, + agent_actions = 3, + min_reward = -1, + max_reward = 1, + }, + extended_tiger = { + observation_bits = 3, + reward_bits = 8, + agent_actions = 4, + min_reward = -100, + max_reward = 30, + }, + tic_tac_toe = { + observation_bits = 18, + reward_bits = 3, + agent_actions = 9, + min_reward = -3, + max_reward = 2, + }, + kuhn_poker = { + observation_bits = 3, + reward_bits = 3, + agent_actions = 2, + min_reward = -2, + max_reward = 2, + }, +} + +local function legacy_max_order(root) + local n = as_int(root.rate_backend_max_order, "rate_backend_max_order") + or as_int(root.max_order, "max_order") + or as_int(root.rosa_max_order, "rosa_max_order") + or 20 + if n < 1 then + return 1 + end + return n +end + +local function mc_predictor_from_algorithm(root, interface) + local algo = lower(root.algorithm or "ctw") + local ct_depth = int_with_default_min(root.ct_depth, "ct_depth", 20, 1) + + if algo == "ctw" or algo == "fac-ctw" then + local percept_bits = interface.observation_bits * math.max(interface.observation_stream_len, 1) + + interface.reward_bits + return { + predictor = { + kind = "fac-ctw", + base_depth = ct_depth, + num_percept_bits = percept_bits, + encoding_bits = 8, + }, + } + end + + if algo == "ac-ctw" or algo == "ctw-context-tree" then + return { + predictor = { kind = "ctw", depth = ct_depth }, + } + end + + if algo == "rosa" or algo == "rosaplus" then + return { + predictor = { kind = "rosaplus", max_order = as_int(root.rosa_max_order, "rosa_max_order") or legacy_max_order(root) or -1 }, + } + end + + if algo == "rwkv" or algo == "rwkv7" then + local method = root.rwkv_method + if method ~= nil then + return { + predictor = { kind = "rwkv7", method = method }, + } + end + local model_path = root.rwkv_model_path + if not model_path then + die("legacy algorithm=rwkv requires rwkv_model_path/rwkv_method") + end + return { + predictor = { kind = "rwkv7", model_path = model_path }, + } + end + + if algo == "mamba" or algo == "mamba1" then + local method = root.mamba_method + if method ~= nil then + return { + predictor = { kind = "mamba", method = method }, + } + end + local model_path = root.mamba_model_path + if not model_path then + die("legacy algorithm=mamba requires mamba_model_path/mamba_method") + end + return { + predictor = { kind = "mamba", model_path = model_path }, + } + end + + if algo == "zpaq" then + return { + predictor = { + kind = "zpaq", + method = tostring(root.zpaq_method or "1"), + }, + } + end + + die("unknown legacy MC-AIXI algorithm '" .. tostring(root.algorithm) .. "'") +end + +local function aiqi_predictor_from_algorithm(root, return_bits) + local algo = lower(root.algorithm or "ac-ctw") + local ct_depth = int_with_default_min(root.ct_depth, "ct_depth", 20, 1) + + if algo == "ctw" or algo == "ac-ctw" or algo == "ctw-context-tree" then + return { + predictor = { kind = "ctw", depth = ct_depth }, + } + end + + if algo == "fac-ctw" then + return { + predictor = { + kind = "fac-ctw", + base_depth = ct_depth, + num_percept_bits = return_bits, + encoding_bits = 8, + }, + } + end + + if algo == "rosa" or algo == "rosaplus" then + return { + predictor = { kind = "rosaplus", max_order = as_int(root.rosa_max_order, "rosa_max_order") or legacy_max_order(root) or -1 }, + } + end + + if algo == "rwkv" or algo == "rwkv7" then + local method = root.rwkv_method + if method ~= nil then + return { + predictor = { kind = "rwkv7", method = method }, + } + end + local model_path = root.rwkv_model_path + if not model_path then + die("legacy AIQI algorithm=rwkv requires rwkv_model_path/rwkv_method") + end + return { + predictor = { kind = "rwkv7", model_path = model_path }, + } + end + + if algo == "mamba" or algo == "mamba1" then + local method = root.mamba_method + if method ~= nil then + return { + predictor = { kind = "mamba", method = method }, + } + end + local model_path = root.mamba_model_path + if not model_path then + die("legacy AIQI algorithm=mamba requires mamba_model_path/mamba_method") + end + return { + predictor = { kind = "mamba", model_path = model_path }, + } + end + + if algo == "zpaq" then + die("legacy AIQI algorithm=zpaq was unsupported and cannot be converted") + end + + die("unknown legacy AIQI algorithm '" .. tostring(root.algorithm) .. "'") +end + +local function render_number(n) + if n ~= n or n == math.huge or n == -math.huge then + die("cannot encode non-finite number") + end + if n == 0 then + return "0" + end + if n % 1 == 0 then + return string.format("%.0f", n) + end + local encoded = cjson.encode(n) + if encoded == "-0" then + return "0" + end + return encoded +end + +local function is_array(tbl) + if type(tbl) ~= "table" then + return false + end + local count = 0 + local max_key = 0 + for k, _ in pairs(tbl) do + if type(k) ~= "number" or k < 1 or k % 1 ~= 0 then + return false + end + if k > max_key then + max_key = k + end + count = count + 1 + end + return max_key == count +end + +local function sorted_keys(tbl) + local keys = {} + for k, _ in pairs(tbl) do + keys[#keys + 1] = k + end + table.sort(keys, function(a, b) + return tostring(a) < tostring(b) + end) + return keys +end + +local function render_json(value, indent) + if value == cjson.null then + return "null" + end + + local t = type(value) + if t == "nil" then + return "null" + end + if t == "boolean" then + return value and "true" or "false" + end + if t == "number" then + return render_number(value) + end + if t == "string" then + return cjson.encode(value) + end + if t ~= "table" then + die("unsupported JSON value type: " .. t) + end + + if is_array(value) then + if #value == 0 then + return "[]" + end + local pieces = {"["} + for i = 1, #value do + pieces[#pieces + 1] = string.rep(" ", indent + 2) + .. render_json(value[i], indent + 2) + .. (i < #value and "," or "") + end + pieces[#pieces + 1] = string.rep(" ", indent) .. "]" + return table.concat(pieces, "\n") + end + + local keys = sorted_keys(value) + if #keys == 0 then + return "{}" + end + + local pieces = {"{"} + for i, key in ipairs(keys) do + local rendered_key = cjson.encode(tostring(key)) + local rendered_value = render_json(value[key], indent + 2) + pieces[#pieces + 1] = string.rep(" ", indent + 2) + .. rendered_key + .. ": " + .. rendered_value + .. (i < #keys and "," or "") + end + pieces[#pieces + 1] = string.rep(" ", indent) .. "}" + return table.concat(pieces, "\n") +end + +local function convert_legacy(root, input_path) + local env_raw = lower(root.environment or "coin-flip") + if env_raw == "external" then + return nil, "External configs were deprecated" + end + + local assets = make_asset_registry() + local base_dir = dirname(input_path) + + local environment + local env_defaults + + if env_raw == "vm" then + local vm_env, vm_info = convert_legacy_vm_environment(root, base_dir, assets) + environment = vm_env + local min_reward, max_reward = signed_reward_bounds(vm_info.reward_bits) + env_defaults = { + observation_bits = vm_info.observation_bits, + reward_bits = vm_info.reward_bits, + agent_actions = vm_info.action_count, + min_reward = min_reward, + max_reward = max_reward, + } + else + local builtin = BUILTIN_ENV[env_raw] + if not builtin then + die("unknown legacy environment '" .. tostring(root.environment) .. "'") + end + environment = { + kind = "builtin", + name = builtin, + } + env_defaults = BUILTIN_DEFAULTS[builtin] + if env_defaults == nil then + die("internal error: missing builtin defaults for '" .. builtin .. "'") + end + end + + local observation_stream_len = parse_observation_stream_len_for_env(root, env_raw) + local observation_key_mode = parse_observation_key_mode_for_env(root, env_raw) + + local observation_bits = as_int(root.observation_bits, "observation_bits") + or env_defaults.observation_bits + if observation_bits < 1 then + observation_bits = env_defaults.observation_bits + end + local reward_bits = as_int(root.reward_bits, "reward_bits") + or env_defaults.reward_bits + if reward_bits < 1 then + reward_bits = env_defaults.reward_bits + end + + local min_reward = env_defaults.min_reward + local max_reward = env_defaults.max_reward + local reward_offset = as_int(root.reward_offset, "reward_offset") + if reward_offset == nil then + reward_offset = math.max(0, -min_reward) + end + if reward_offset == 0 then + reward_offset = 0 + end + + local interface = { + observation_bits = observation_bits, + observation_stream_len = observation_stream_len, + observation_key_mode = observation_key_mode, + reward_bits = reward_bits, + agent_actions = int_with_default_min(root.agent_actions, "agent_actions", env_defaults.agent_actions, 1), + min_reward = min_reward, + max_reward = max_reward, + reward_offset = reward_offset, + } + + local planner_raw = lower(root.planner or root.solver or "mc-aixi") + local planner_kind + if planner_raw == "mc-aixi" or planner_raw == "mc_aixi" then + planner_kind = "mc_aixi" + elseif planner_raw == "aiqi" then + planner_kind = "aiqi_discounted" + else + die("unsupported legacy planner/solver '" .. tostring(root.planner or root.solver) .. "'") + end + + local run_seed = as_int(root.random_seed, "random_seed") + or as_int(root.rng_seed, "rng_seed") + + local controller + local random_seed = run_seed + + if planner_kind == "mc_aixi" then + local predictor_override = nil + if root.rate_backend ~= nil then + predictor_override = backend_from_legacy_cfg( + root.rate_backend, + root, + base_dir, + observation_bits, + reward_bits + ) + end + + local predictor_pair + if predictor_override ~= nil then + if predictor_override.kind == "rosaplus" and predictor_override.max_order == nil then + predictor_override.max_order = legacy_max_order(root) or -1 + end + predictor_pair = { + predictor = predictor_override, + } + else + predictor_pair = mc_predictor_from_algorithm(root, interface) + end + + controller = { + kind = "mc_aixi", + predictor = predictor_pair.predictor, + agent_horizon = int_with_default_min(root.agent_horizon, "agent_horizon", 3, 1), + num_simulations = int_with_default_min(root.num_simulations, "num_simulations", 50, 1), + exploration_exploitation_ratio = positive_num_with_default(root.exploration_exploitation_ratio, "exploration_exploitation_ratio", 1.4), + discount_gamma = closed_unit_num_with_default(root.discount_gamma, "discount_gamma", 1.0), + } + + local planner_seed = as_int(root.mcaixi_random_seed, "mcaixi_random_seed") + if planner_seed ~= nil and random_seed == nil then + random_seed = planner_seed + end + else + local predictor_override = nil + if root.aiqi_rate_backend ~= nil then + predictor_override = backend_from_legacy_cfg( + root.aiqi_rate_backend, + root, + base_dir, + observation_bits, + reward_bits + ) + elseif root.rate_backend ~= nil then + predictor_override = backend_from_legacy_cfg( + root.rate_backend, + root, + base_dir, + observation_bits, + reward_bits + ) + end + + local discount_gamma + if root.discount_gamma == nil then + discount_gamma = 0.99 + else + discount_gamma = as_num(root.discount_gamma, "discount_gamma") + end + + local return_horizon = int_with_default_min( + root.return_horizon ~= nil and root.return_horizon or root.agent_horizon, + "return_horizon", + 3, + 1 + ) + local return_bins_raw = as_int(root.return_bins, "return_bins") + or as_int(root.aiqi_bins, "aiqi_bins") + or 16 + local return_bins = next_power_of_two(return_bins_raw) + local augmentation_period = int_with_default_min(as_int(root.augmentation_period, "augmentation_period") + or as_int(root.aiqi_period, "aiqi_period") + or as_int(root.return_horizon, "return_horizon") + or as_int(root.agent_horizon, "agent_horizon") + or 3, "augmentation_period", 3, return_horizon) + + local predictor_pair + if predictor_override ~= nil then + if predictor_override.kind == "rosaplus" and predictor_override.max_order == nil then + predictor_override.max_order = legacy_max_order(root) or -1 + end + predictor_pair = { + predictor = predictor_override, + } + else + predictor_pair = aiqi_predictor_from_algorithm(root, return_bins) + end + + controller = { + kind = "aiqi_discounted", + predictor = predictor_pair.predictor, + discount_gamma = open_unit_num_with_default(discount_gamma, "discount_gamma", 0.99), + return_horizon = return_horizon, + return_bins = return_bins, + augmentation_period = augmentation_period, + history_prune_keep_steps = (function() + local keep = as_int(root.history_prune_keep_steps, "history_prune_keep_steps") + if keep == nil then + return nil + end + if keep < 0 then + return 0 + end + return keep + end)(), + baseline_exploration = open_unit_num_with_default(as_num(root.baseline_exploration, "baseline_exploration") + or as_num(root.tau, "tau") + or 0.01, "baseline_exploration", 0.01), + } + + local planner_seed = as_int(root.aiqi_random_seed, "aiqi_random_seed") + if planner_seed ~= nil and random_seed == nil then + random_seed = planner_seed + end + end + + local vm_perf_only = as_bool(root.vm_perf_only) + local runtime = { + random_seed = random_seed, + log_every = int_with_default_min(root.log_every, "log_every", 1, 1), + perf = as_bool(root.perf), + vm_perf_only = vm_perf_only, + explore_epsilon = nonnegative_num_with_default(root.explore_epsilon, "explore_epsilon", 0.0), + explore_gamma = positive_num_with_default(root.explore_gamma, "explore_gamma", 1.0), + } + + if vm_perf_only then + local perf_cycles = as_int(root.perf_cycles, "perf_cycles") + if perf_cycles ~= nil and perf_cycles < 1 then + perf_cycles = 1 + end + local terminate_lifetime = as_int(root["terminate-lifetime"], "terminate-lifetime") + if terminate_lifetime ~= nil and terminate_lifetime < 1 then + terminate_lifetime = 1 + end + runtime.terminate_lifetime = perf_cycles or terminate_lifetime or 1000 + else + local learn_cycles = as_int(root.learn_cycles, "learn_cycles") + if learn_cycles ~= nil and learn_cycles < 0 then + learn_cycles = 0 + end + local eval_cycles = as_int(root.eval_cycles, "eval_cycles") + if eval_cycles ~= nil and eval_cycles < 0 then + eval_cycles = 0 + end + runtime.learn_cycles = learn_cycles + runtime.eval_cycles = eval_cycles + runtime.terminate_lifetime = int_with_default_min(root["terminate-lifetime"], "terminate-lifetime", 20, 1) + end + + local doc = { + schema_version = 1, + kind = "planner_run", + assets = assets.list(), + environment = environment, + interface = interface, + controller = controller, + runtime = runtime, + } + + return doc, nil +end + +local input_path = arg[1] +if not input_path or input_path == "" then + die("usage: " .. (arg[0] or "legacy_aixi_convert.lua") .. " ") +end + +local raw +if input_path == "-" then + raw = read_stdin() + input_path = "stdin.json" +else + raw = read_file(input_path) +end + +local ok, value = pcall(cjson.decode, raw) +if not ok then + die("invalid JSON input: " .. tostring(value)) +end +if type(value) ~= "table" then + die("top-level JSON value must be an object") +end + +if value.schema_version ~= nil and value.kind ~= nil then + io.write(render_json(value, 0)) + io.write("\n") + os.exit(0) +end + +local converted, message = convert_legacy(value, input_path) +if converted == nil then + io.write(message) + os.exit(0) +end + +io.write(render_json(converted, 0)) +io.write("\n") diff --git a/scripts/plot_two_json.sh b/scripts/plot_two_json.sh index 7ade6354..a2389231 100644 --- a/scripts/plot_two_json.sh +++ b/scripts/plot_two_json.sh @@ -22,13 +22,13 @@ need_cmd() { command -v "$1" >/dev/null 2>&1 || fail "Missing required command: case "${PLOT_SUITE}" in two-json|two_json|two|core|full) PLOT_SUITE=two-json - SUITE_DISPLAY="examples/two.json" + SUITE_DISPLAY="configs/bench/two.json" SUITE_PATH_PREFIX="infotheory-two-json" - SUITE_FOCUS_SUBJECTS="neural_mixture rwkv" + SUITE_FOCUS_SUBJECTS="neural_mixture rwkv7" ;; extra) PLOT_SUITE=extra - SUITE_DISPLAY="examples/extra.json" + SUITE_DISPLAY="configs/bench/extra.json" SUITE_PATH_PREFIX="infotheory-extra" SUITE_FOCUS_SUBJECTS="neural_mixture mamba" ;; @@ -58,7 +58,7 @@ Environment: INFOTHEORY_PLOT_SUITE=two-json|extra INFOTHEORY_PLOT_SUMMARY_TSV=/tmp/${SUITE_PATH_PREFIX}-summary-.tsv INFOTHEORY_BASELINE_SUMMARY_TSV=benchmarks/baselines/${SUITE_PATH_PREFIX}-summary-.tsv - INFOTHEORY_PLOT_SUBJECTS=rwkv + INFOTHEORY_PLOT_SUBJECTS=rwkv7 INFOTHEORY_PLOT_OUTPUT_DIR=/tmp/plotimgs INFOTHEORY_PLOT_WIDTH=2400 INFOTHEORY_PLOT_HEIGHT=1400 @@ -120,10 +120,20 @@ selected = {token for token in re.split(r"[\s,]+", raw_filter.strip()) if token} if not selected: raise SystemExit("INFOTHEORY_PLOT_SUBJECTS must contain at least one subject") +def canonicalize_subject(subject: str) -> str: + return "rwkv7" if subject == "rwkv" else subject + +selected = {canonicalize_subject(token) for token in selected} + with open(input_path, newline="") as fh: reader = csv.DictReader(fh, delimiter="\t") rows = list(reader) +for row in rows: + row["subject"] = canonicalize_subject(row["subject"]) + if "series" in row: + row["series"] = row["series"].replace(":rwkv", ":rwkv7") + known = {row["subject"] for row in rows} unknown = sorted(selected - known) if unknown: @@ -155,10 +165,22 @@ import sys input_path, field_name, field_value, output_path = sys.argv[1:5] +def canonicalize_subject(subject: str) -> str: + return "rwkv7" if subject == "rwkv" else subject + +if field_name == "subject": + field_value = canonicalize_subject(field_value) + with open(input_path, newline="") as fh: reader = csv.DictReader(fh, delimiter="\t") rows = list(reader) +if field_name == "subject": + for row in rows: + row["subject"] = canonicalize_subject(row.get("subject", "")) + if "series" in row: + row["series"] = row["series"].replace(":rwkv", ":rwkv7") + if not rows: raise SystemExit(f"{input_path}: no rows to filter") @@ -200,18 +222,62 @@ import sys current_path, baseline_path, output_path = sys.argv[1:4] +OPTIONAL_LEGACY_COLUMNS = ( + "suite_spec_path", + "suite_spec_sha256", + "build_mode", + "build_features", +) + +LEGACY_UNKNOWN = "__legacy_unknown__" + def load_rows(path): with open(path, newline="") as fh: reader = csv.DictReader(fh, delimiter="\t") rows = list(reader) return reader.fieldnames, rows -current_fields, current_rows = load_rows(current_path) -baseline_fields, baseline_rows = load_rows(baseline_path) -if current_fields != baseline_fields: - raise SystemExit( - "summary TSV columns do not match between current and baseline inputs" - ) +def normalize_rows(path): + fieldnames, rows = load_rows(path) + if fieldnames is None: + raise SystemExit(f"{path}: missing header") + + normalized_fieldnames = list(fieldnames) + for name in OPTIONAL_LEGACY_COLUMNS: + if name not in normalized_fieldnames: + normalized_fieldnames.append(name) + + normalized_rows = [] + for row in rows: + normalized = {name: row.get(name, "") for name in normalized_fieldnames} + subject = normalized.get("subject", "") + if subject == "rwkv": + normalized["subject"] = "rwkv7" + if "series" in normalized and normalized["series"]: + normalized["series"] = normalized["series"].replace(":rwkv", ":rwkv7") + for name in OPTIONAL_LEGACY_COLUMNS: + if name not in row: + normalized[name] = LEGACY_UNKNOWN + elif normalized[name] == "": + normalized[name] = LEGACY_UNKNOWN + normalized_rows.append(normalized) + return normalized_fieldnames, normalized_rows + +current_fields, current_rows = normalize_rows(current_path) +baseline_fields, baseline_rows = normalize_rows(baseline_path) + +all_fields = [] +seen_fields = set() +for field_list in (current_fields, baseline_fields): + for name in field_list: + if name not in seen_fields: + seen_fields.add(name) + all_fields.append(name) + +for rows in (current_rows, baseline_rows): + for row in rows: + for name in all_fields: + row.setdefault(name, "") key_fields = ("operation", "subject", "size_bytes", "compression_backend") current_keys = { @@ -230,7 +296,7 @@ if current_only or baseline_only: file=sys.stderr, ) -fieldnames = current_fields + [ +fieldnames = all_fields + [ "summary_source", "subject_overlay", "series_overlay", diff --git a/scripts/summarize_interpret.sh b/scripts/summarize_interpret.sh new file mode 100755 index 00000000..6f3a733b --- /dev/null +++ b/scripts/summarize_interpret.sh @@ -0,0 +1,306 @@ +#!/usr/bin/env bash +set -euo pipefail + +find_latest_summary() { + local root="/var/tmp/infotheory_bench" + local latest="" + + [[ -d "${root}" ]] || return 1 + + latest="$( + find "${root}" -mindepth 2 -maxdepth 2 -type f -name summary.tsv 2>/dev/null \ + | awk -F/ ' + { + stamp=$(NF-1) + if (stamp ~ /^[0-9]{8}-[0-9]{6}$/) { + print stamp "\t" $0 + } + } + ' \ + | sort -r \ + | head -n1 \ + | cut -f2- + )" + + [[ -n "${latest}" ]] || return 1 + printf '%s\n' "${latest}" +} + +summary="${1:-}" +if [[ -z "${summary}" ]]; then + if ! summary="$(find_latest_summary)"; then + echo "error: no summary.tsv found under /var/tmp/infotheory_bench" >&2 + exit 1 + fi +fi + +if [[ ! -f "${summary}" ]]; then + echo "usage: $0 [/path/to/summary.tsv]" >&2 + echo "error: file not found: ${summary}" >&2 + exit 1 +fi + +python3 - "${summary}" <<'PY' +import csv +import math +import pathlib +import sys + +summary_path = pathlib.Path(sys.argv[1]) + + +def parse_float(raw): + if raw is None: + return math.nan + text = str(raw).strip() + if not text: + return math.nan + lower = text.lower() + if lower in {"inf", "+inf"}: + return math.inf + if lower == "-inf": + return -math.inf + try: + return float(text) + except ValueError: + return math.nan + + +def parse_int(raw): + value = parse_float(raw) + if math.isfinite(value): + return int(value) + return None + + +def fmt(value, digits=9): + if math.isfinite(value): + return f"{value:.{digits}f}" + if value > 0: + return "inf" + if value < 0: + return "-inf" + return "nan" + + +rows = [] +with summary_path.open("r", encoding="utf-8") as fh: + reader = csv.DictReader(fh, delimiter="\t") + if reader.fieldnames is None: + print(f"Summary for: {summary_path}") + print() + print("No tabular header detected in summary.tsv.") + raise SystemExit(0) + for row in reader: + if not any((value or "").strip() for value in row.values()): + continue + rows.append(row) + +records = [] +for row in rows: + label = (row.get("label") or "").strip() + if not label: + continue + + bmean = parse_float(row.get("baseline_mean_s")) + cmean = parse_float(row.get("current_mean_s")) + bstd = parse_float(row.get("baseline_stddev_s")) + cstd = parse_float(row.get("current_stddev_s")) + ratio = parse_float(row.get("ratio_current_over_baseline")) + + if not math.isfinite(bstd): + bstd = 0.0 + if not math.isfinite(cstd): + cstd = 0.0 + + if not math.isfinite(ratio): + if bmean == 0: + ratio = math.inf + else: + ratio = cmean / bmean + + b_n = parse_int(row.get("baseline_n")) + c_n = parse_int(row.get("current_n")) + + bsem = parse_float(row.get("baseline_sem_s")) + csem = parse_float(row.get("current_sem_s")) + if not math.isfinite(bsem): + if b_n is not None and b_n > 0: + bsem = bstd / math.sqrt(b_n) + else: + bsem = bstd + if not math.isfinite(csem): + if c_n is not None and c_n > 0: + csem = cstd / math.sqrt(c_n) + else: + csem = cstd + + delta = parse_float(row.get("delta_s")) + if not math.isfinite(delta): + delta = cmean - bmean + + se_delta = parse_float(row.get("se_delta_s")) + if not math.isfinite(se_delta): + se_delta = math.sqrt((bsem * bsem) + (csem * csem)) + + t_like = parse_float(row.get("t_like")) + if not math.isfinite(t_like): + if se_delta == 0: + t_like = math.inf if delta != 0 else 0.0 + else: + t_like = abs(delta) / se_delta + + ci_low = parse_float(row.get("ci95_ratio_low")) + ci_high = parse_float(row.get("ci95_ratio_high")) + if not (math.isfinite(ci_low) and math.isfinite(ci_high)): + if ( + bmean > 0 + and cmean > 0 + and math.isfinite(bsem) + and math.isfinite(csem) + ): + log_ratio = math.log(cmean / bmean) + se_log_ratio = math.sqrt((bsem / bmean) ** 2 + (csem / cmean) ** 2) + ci_low = math.exp(log_ratio - 1.96 * se_log_ratio) + ci_high = math.exp(log_ratio + 1.96 * se_log_ratio) + else: + ci_low = ratio + ci_high = ratio + + residual_bits = parse_float(row.get("residual_bits_gaussian")) + + pct = (ratio - 1.0) * 100.0 + if pct > 0.5: + direction = "slower" + elif pct < -0.5: + direction = "faster" + else: + direction = "flat" + + apct = abs(pct) + if apct < 1.0: + size = "tiny" + elif apct < 3.0: + size = "small" + elif apct < 10.0: + size = "moderate" + else: + size = "large" + + if t_like < 1.0: + confidence = "low" + elif t_like < 2.0: + confidence = "medium" + else: + confidence = "high" + + ci_text = "n/a" + ci_support = False + if math.isfinite(ci_low) and math.isfinite(ci_high): + lo_pct = (ci_low - 1.0) * 100.0 + hi_pct = (ci_high - 1.0) * 100.0 + ci_text = f"[{lo_pct:+.2f}%, {hi_pct:+.2f}%]" + if direction == "slower" and ci_low > 1.0: + ci_support = True + if direction == "faster" and ci_high < 1.0: + ci_support = True + + records.append( + { + "label": label, + "pct": pct, + "change_text": f"{pct:+.2f}%", + "dir": direction, + "size": size, + "confidence": confidence, + "bmean": bmean, + "cmean": cmean, + "se_delta": se_delta, + "ci_text": ci_text, + "residual_bits": residual_bits, + "ci_support": ci_support, + "sort_key": apct, + } + ) + +records.sort(key=lambda item: float(item["sort_key"]), reverse=True) + +total = len(records) +slower = sum(1 for row in records if row["dir"] == "slower") +faster = sum(1 for row in records if row["dir"] == "faster") +flat = sum(1 for row in records if row["dir"] == "flat") + +strong_reg = sum( + 1 + for row in records + if row["dir"] == "slower" + and (row["size"] == "large" or row["confidence"] == "high" or row["ci_support"]) +) +strong_imp = sum( + 1 + for row in records + if row["dir"] == "faster" + and (row["size"] == "large" or row["confidence"] == "high" or row["ci_support"]) +) + +roundtrip_path = summary_path.with_name("roundtrip.tsv") +roundtrip_total = 0 +roundtrip_pass = 0 +roundtrip_fail = 0 +if roundtrip_path.is_file(): + with roundtrip_path.open("r", encoding="utf-8") as fh: + reader = csv.DictReader(fh, delimiter="\t") + for row in reader: + status = (row.get("status") or "").strip().lower() + if not status: + continue + roundtrip_total += 1 + if status == "pass": + roundtrip_pass += 1 + else: + roundtrip_fail += 1 + +print(f"Summary for: {summary_path}") +print() +print(f"Cases: {total} | Slower: {slower} | Faster: {faster} | Flat: {flat}") +if roundtrip_total > 0: + print(f"Roundtrip checks: {roundtrip_pass} passed / {roundtrip_total} total") + +if roundtrip_fail > 0: + print("Interpretation: invalid benchmark run; roundtrip verification reported failures.") +elif strong_reg > 0 and strong_imp > 0: + print("Interpretation: mixed result; there are both strong regressions and strong improvements.") +elif strong_reg > 0: + print("Interpretation: regression-leaning; inspect the top slower cases first.") +elif strong_imp > 0: + print("Interpretation: improvement-leaning; no strong regression stands out.") +else: + print("Interpretation: mostly flat/noisy; nothing stands out strongly from this summary alone.") + +print() +print( + f"{'label':<24} {'change':>12} {'dir':<7} {'size':<9} {'confidence':<9} " + f"{'baseline(s)':>14} {'current(s)':>14} {'se_delta(s)':>12} {'95% CI(change)':>20} {'resid_bits':>10}" +) +print("-" * 170) + +for row in records: + se_delta_text = fmt(float(row["se_delta"]), 9) + resid_bits = float(row["residual_bits"]) + resid_text = fmt(resid_bits, 3) if math.isfinite(resid_bits) else "n/a" + print( + f"{str(row['label']):<24} {str(row['change_text']):>12} {str(row['dir']):<7} " + f"{str(row['size']):<9} {str(row['confidence']):<9} " + f"{fmt(float(row['bmean']), 9):>14} {fmt(float(row['cmean']), 9):>14} " + f"{se_delta_text:>12} {str(row['ci_text']):>20} {resid_text:>10}" + ) + +print() +print("Legend:") +print(" change = (current / baseline - 1) * 100") +print(" confidence = |current_mean - baseline_mean| / se_delta") +print(" se_delta = sqrt(baseline_sem^2 + current_sem^2)") +print(" 95% CI(change) = delta-method interval from log(current/baseline)") +print(" resid_bits = 0.5 * log2(2*pi*e*pooled_residual_var_s2)") +print(" size = tiny <1%, small <3%, moderate <10%, large >=10%") +PY diff --git a/scripts/test_ci_local.sh b/scripts/test_ci_local.sh new file mode 100755 index 00000000..6af8cd30 --- /dev/null +++ b/scripts/test_ci_local.sh @@ -0,0 +1,233 @@ +#!/bin/sh +set -eu + +export LC_ALL=C + +ROOT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) + +say() { printf '%s\n' "$*"; } +fail() { printf 'ERROR: %s\n' "$*" >&2; exit 1; } +need_cmd() { command -v "$1" >/dev/null 2>&1 || fail "Missing required command: $1"; } + +build_mode() { + printf '%s' "${INFOTHEORY_BUILD_MODE:-native}" +} + +validate_build_mode() { + case "$(build_mode)" in + native|portable) ;; + *) fail "INFOTHEORY_BUILD_MODE must be one of: native, portable" ;; + esac +} + +portable_rustflags() { + case "$(uname -s)" in + Linux|FreeBSD|OpenBSD) printf '%s' "-C target-cpu=generic -C link-arg=-fuse-ld=lld" ;; + *) printf '%s' "-C target-cpu=generic" ;; + esac +} + +run_cargo_mode() { + validate_build_mode + mode=$(build_mode) + ci_jobs=${CARGO_BUILD_JOBS:-4} + ci_dev_lto=${CARGO_PROFILE_DEV_LTO:-false} + ci_test_lto=${CARGO_PROFILE_TEST_LTO:-false} + ci_dev_cgu=${CARGO_PROFILE_DEV_CODEGEN_UNITS:-16} + ci_test_cgu=${CARGO_PROFILE_TEST_CODEGEN_UNITS:-16} + if [ "$mode" = "portable" ]; then + mode_flags=$(portable_rustflags) + CARGO_BUILD_JOBS="$ci_jobs" \ + CARGO_PROFILE_DEV_LTO="$ci_dev_lto" \ + CARGO_PROFILE_TEST_LTO="$ci_test_lto" \ + CARGO_PROFILE_DEV_CODEGEN_UNITS="$ci_dev_cgu" \ + CARGO_PROFILE_TEST_CODEGEN_UNITS="$ci_test_cgu" \ + CARGO_BUILD_RUSTFLAGS="$mode_flags" \ + RUSTDOCFLAGS="${RUSTDOCFLAGS:-$mode_flags}" \ + cargo "$@" + else + CARGO_BUILD_JOBS="$ci_jobs" \ + CARGO_PROFILE_DEV_LTO="$ci_dev_lto" \ + CARGO_PROFILE_TEST_LTO="$ci_test_lto" \ + CARGO_PROFILE_DEV_CODEGEN_UNITS="$ci_dev_cgu" \ + CARGO_PROFILE_TEST_CODEGEN_UNITS="$ci_test_cgu" \ + cargo "$@" + fi +} + +cargo_check_warn() { + ( + cd "$ROOT_DIR" && \ + RUSTFLAGS="-D warnings" \ + run_cargo_mode check "$@" + ) +} + +ci_rustdoc_tmp=${TMPDIR:-/tmp}/infotheory-rustdoc-cov.$$.json +cleanup() { + rm -f "$ci_rustdoc_tmp" >/dev/null 2>&1 || true +} +trap cleanup EXIT HUP INT TERM + +cmd_rust_line_coverage() { + say "[test_ci] Rust line coverage gate (>= ${INFOTHEORY_CI_FAIL_UNDER_LINES:-85}%)..." + if ! cargo llvm-cov --version >/dev/null 2>&1; then + fail "cargo-llvm-cov is required. Install it with: cargo install cargo-llvm-cov --locked" + fi + ( + cd "$ROOT_DIR" && \ + run_cargo_mode llvm-cov -p infotheory --tests --features "cli all-backends" --locked --summary-only --fail-under-lines "${INFOTHEORY_CI_FAIL_UNDER_LINES:-85}" + ) +} + +cmd_rustdoc_coverage() { + say "[test_ci] Rustdoc coverage gate (must remain 100%)..." + if ! cargo +nightly --version >/dev/null 2>&1; then + fail "nightly toolchain is required for rustdoc coverage. Install it with: rustup toolchain install nightly" + fi + ( + cd "$ROOT_DIR" && \ + run_cargo_mode +nightly rustdoc -p infotheory --all-features -- -Z unstable-options --show-coverage --output-format json > "$ci_rustdoc_tmp" + ) + python3 - "$ci_rustdoc_tmp" <<'PY' +import json +import sys + +data = json.load(open(sys.argv[1], encoding="utf-8")) +files = data.get("files", data) +total = sum(v.get("total", 0) for v in files.values()) +with_docs = sum(v.get("with_docs", 0) for v in files.values()) +pct = 100.0 if total == 0 else (with_docs * 100.0 / total) +print(f"Rustdoc documented items: {with_docs}/{total} ({pct:.2f}%)") +if with_docs != total: + print("Rustdoc coverage gate failed (<100.0%).", file=sys.stderr) + sys.exit(1) +PY +} + +cmd_feature_gates() { + say "[test_ci] Feature-gate compile matrix (curated fast subset)..." + + cargo_check_warn -p zpaq_rs --locked + cargo_check_warn -p benchman --locked + cargo_check_warn --manifest-path vendor/gameengine/Cargo.toml --features builtin --locked + + cargo_check_warn -p infotheory --locked + cargo_check_warn -p infotheory --no-default-features --locked + cargo_check_warn -p infotheory --no-default-features --features backend-ctw --locked + cargo_check_warn -p infotheory --no-default-features --features all-backends --locked + cargo_check_warn -p infotheory --no-default-features --features aixi-gameengine --locked + cargo_check_warn -p infotheory --no-default-features --features "tuner backend-ctw" --locked + cargo_check_warn -p infotheory --features cli --locked + cargo_check_warn -p infotheory --no-default-features --features cli --locked + cargo_check_warn -p infotheory --no-default-features --features "cli all-backends" --locked + ( + cd "$ROOT_DIR" && \ + PYO3_BUILD_EXTENSION_MODULE=1 \ + RUSTFLAGS="-D warnings" \ + run_cargo_mode check -p infotheory_py --no-default-features --features "tuner backend-ctw" --locked + ) + + say "[test_ci] Test-harness compile slices..." + ( + cd "$ROOT_DIR" && \ + run_cargo_mode test -p infotheory --no-run --locked && \ + run_cargo_mode test -p infotheory --no-default-features --features "cli all-backends" --no-run --locked && \ + run_cargo_mode test -p infotheory --no-default-features --features "tuner backend-ctw" --no-run --locked + ) + + if [ "${INFOTHEORY_CI_INCLUDE_VM:-0}" = "1" ]; then + say "[test_ci] VM compile slices enabled (INFOTHEORY_CI_INCLUDE_VM=1)..." + ( + cd "$ROOT_DIR" && \ + run_cargo_mode test -p infotheory --no-default-features --features vm --no-run --locked && \ + run_cargo_mode test -p infotheory --no-default-features --features "vm backend-ctw" --no-run --locked + ) + else + say "[test_ci] Skipping VM compile slices (set INFOTHEORY_CI_INCLUDE_VM=1 to include)." + fi +} + +cmd_python_gates() { + say "[test_ci] Python extension + coverage gate..." + need_cmd uv + + venv_dir=${INFOTHEORY_CI_PYTHON_VENV:-"$ROOT_DIR/.venv"} + venv_py=$venv_dir/bin/python + + if [ ! -x "$venv_py" ]; then + (cd "$ROOT_DIR" && uv venv "$venv_dir") + fi + + (cd "$ROOT_DIR" && uv pip install --python "$venv_py" "maturin>=1.7,<2" "pytest>=8.0" "pytest-cov>=7.0.0") + + ( + cd "$ROOT_DIR" && \ + VIRTUAL_ENV="$venv_dir" \ + PATH="$venv_dir/bin:$PATH" \ + "$venv_py" -m maturin develop --profile python-release --manifest-path crates/infotheory_py/Cargo.toml + ) + + ( + cd "$ROOT_DIR" && \ + "$venv_py" -m pytest --cov=infotheory_rs --cov-report=term-missing --cov-report=xml:target/python-coverage.xml --cov-fail-under=100 python/tests + ) + + say "[test_ci] Python aixi-gameengine smoke gate..." + ( + cd "$ROOT_DIR" && \ + VIRTUAL_ENV="$venv_dir" \ + PATH="$venv_dir/bin:$PATH" \ + "$venv_py" -m maturin develop --profile python-release --manifest-path crates/infotheory_py/Cargo.toml --features python-extension,all-backends,aixi-gameengine && \ + "$venv_py" -m pytest -q python/tests/test_aixi_gameengine.py + ) + + if [ "${INFOTHEORY_CI_INCLUDE_VM:-0}" = "1" ]; then + say "[test_ci] Python VM smoke gate enabled (INFOTHEORY_CI_INCLUDE_VM=1)..." + ( + cd "$ROOT_DIR" && \ + VIRTUAL_ENV="$venv_dir" \ + PATH="$venv_dir/bin:$PATH" \ + "$venv_py" -m maturin develop --profile python-release --manifest-path crates/infotheory_py/Cargo.toml --features python-extension,all-backends,vm && \ + "$venv_py" -m pytest -q python/tests/test_vm.py + ) + else + say "[test_ci] Skipping Python VM smoke (set INFOTHEORY_CI_INCLUDE_VM=1 to include)." + fi +} + +cmd_main() { + say "[test_ci] Running local CI preflight (fast comprehensive gates)..." + need_cmd cargo + need_cmd python3 + validate_build_mode + say "[test_ci] Build mode: $(build_mode)" + + if [ "${INFOTHEORY_CI_SKIP_RUST_LINE_COVERAGE:-0}" = "1" ]; then + say "[test_ci] Skipping Rust line coverage gate (INFOTHEORY_CI_SKIP_RUST_LINE_COVERAGE=1)." + else + cmd_rust_line_coverage + fi + + if [ "${INFOTHEORY_CI_SKIP_RUSTDOC_COVERAGE:-0}" = "1" ]; then + say "[test_ci] Skipping rustdoc coverage gate (INFOTHEORY_CI_SKIP_RUSTDOC_COVERAGE=1)." + else + cmd_rustdoc_coverage + fi + + if [ "${INFOTHEORY_CI_SKIP_FEATURE_GATES:-0}" = "1" ]; then + say "[test_ci] Skipping feature-gate compile matrix (INFOTHEORY_CI_SKIP_FEATURE_GATES=1)." + else + cmd_feature_gates + fi + + if [ "${INFOTHEORY_CI_SKIP_PYTHON:-0}" = "1" ]; then + say "[test_ci] Skipping Python coverage/smoke gates (INFOTHEORY_CI_SKIP_PYTHON=1)." + else + cmd_python_gates + fi + + say "[test_ci] All local CI preflight gates passed." +} + +cmd_main "$@" diff --git a/scripts/workload_presets.sh b/scripts/workload_presets.sh index a97ac667..38d1727a 100755 --- a/scripts/workload_presets.sh +++ b/scripts/workload_presets.sh @@ -38,7 +38,7 @@ workload_portable_input() { "${repo_root}/README.md" \ "${repo_root}/LICENSE-APACHE" \ "${repo_root}/Cargo.toml" \ - "${repo_root}/examples/two.json" >> "${out_path}" + "${repo_root}/configs/bench/two.json" >> "${out_path}" done truncate -s "${bytes}" "${out_path}" printf '%s\n' "${out_path}" @@ -66,7 +66,7 @@ configure_workload_preset() { two-json) WORKLOAD_LABELS=("two_json_rate_ac") WORKLOAD_COMMANDS=( - "\"${bin_path}\" compress \"${WORKLOAD_INPUT}\" \"${out_dir}/two_json_rate_ac.itc\" --compression-backend rate-ac --rate-backend mixture --method \"${repo_root}/examples/two.json\"" + "\"${bin_path}\" compress \"${WORKLOAD_INPUT}\" \"${out_dir}/two_json_rate_ac.itc\" --compression-backend rate-ac --rate-backend mixture --method \"${repo_root}/configs/bench/two.json\"" ) ;; rwkv-all) diff --git a/src/aixi/agent.rs b/src/aixi/agent.rs deleted file mode 100644 index b152ea71..00000000 --- a/src/aixi/agent.rs +++ /dev/null @@ -1,694 +0,0 @@ -//! The core AIXI agent implementation. -//! -//! This module defines the `Agent` struct, which ties together a world model -//! (Predictor) and a planner (SearchTree) to form a complete autonomous entity. - -use crate::RateBackend; -use crate::aixi::common::{ - Action, ObservationKeyMode, PerceptVal, RandomGenerator, Reward, decode, encode, - observation_repr_from_stream, -}; -use crate::aixi::mcts::{AgentSimulator, SearchTree}; -#[cfg(feature = "backend-mamba")] -use crate::aixi::model::MambaPredictor; -#[cfg(feature = "backend-rwkv")] -use crate::aixi::model::RwkvPredictor; -use crate::aixi::model::{ - CtwPredictor, FacCtwPredictor, Predictor, RateBackendBitPredictor, RosaPredictor, ZpaqPredictor, -}; -use crate::aixi::rate_backend::{adapt_rate_backend_for_bit_tokens, rate_backend_contains_zpaq}; -#[cfg(feature = "backend-mamba")] -use crate::load_mamba_model_from_path; -#[cfg(feature = "backend-rwkv")] -use crate::load_rwkv7_model_from_path; -use crate::{validate_rate_backend, validate_zpaq_rate_method}; - -/// Configuration parameters for an AIXI agent. -#[derive(Clone)] -pub struct AgentConfig { - /// The predictive algorithm to use ("ctw", "rosa", "rwkv", "mamba", "zpaq"). - pub algorithm: String, - /// Context depth for the CTW model. - pub ct_depth: usize, - /// Planning horizon for MCTS. - pub agent_horizon: usize, - /// Number of bits used to encode observations. - pub observation_bits: usize, - /// Number of observation symbols per action (stream length). - pub observation_stream_len: usize, - /// Strategy for mapping observation streams into search keys. - pub observation_key_mode: ObservationKeyMode, - /// Number of bits used to encode rewards. - pub reward_bits: usize, - /// Number of possible actions. - pub agent_actions: usize, - /// Number of MCTS simulations per planning step. - pub num_simulations: usize, - /// Constant governing exploration vs exploitation in UCT. - pub exploration_exploitation_ratio: f64, - /// Discount factor for future rewards (1.0 = undiscounted). - pub discount_gamma: f64, - /// Minimum possible instantaneous reward in the environment. - pub min_reward: Reward, - /// Maximum possible instantaneous reward in the environment. - pub max_reward: Reward, - /// Reward offset applied before encoding rewards as unsigned bits. - /// - /// Paper-compatible encoding shifts rewards by an offset so all encoded values are non-negative. - pub reward_offset: Reward, - /// Optional deterministic RNG seed for planning/simulation behavior. - /// - /// When `None`, a fresh runtime-derived seed is used. - pub random_seed: Option, - /// Optional generic rate backend override. - /// - /// When set, this takes precedence over `algorithm` and routes MC-AIXI - /// through the shared `RateBackend` abstraction. - pub rate_backend: Option, - /// Max-order hint for `rate_backend` constructors that use it (for example ROSA). - pub rate_backend_max_order: i64, - /// Path to the RWKV model weights (if using "rwkv"). - pub rwkv_model_path: Option, - /// Optional RWKV method string for hosted/browser-safe construction. - pub rwkv_method: Option, - /// Path to the Mamba model weights (if using "mamba"). - pub mamba_model_path: Option, - /// Optional Mamba method string for hosted/browser-safe construction. - pub mamba_method: Option, - /// Maximum Markov order for the ROSA model (if using "rosa"). - pub rosa_max_order: Option, - /// ZPAQ method string for the rate model (if using "zpaq"). - pub zpaq_method: Option, -} - -impl AgentConfig { - /// Validate configuration constraints for MC-AIXI. - pub fn validate(&self) -> Result<(), String> { - if self.agent_actions == 0 { - return Err("agent_actions must be >= 1".to_string()); - } - if self.agent_horizon == 0 { - return Err("agent_horizon must be >= 1".to_string()); - } - if self.num_simulations == 0 { - return Err("num_simulations must be >= 1".to_string()); - } - if self.exploration_exploitation_ratio <= 0.0 { - return Err("exploration_exploitation_ratio must be > 0".to_string()); - } - if !(0.0..=1.0).contains(&self.discount_gamma) { - return Err(format!( - "discount_gamma must be in [0, 1] for MC-AIXI, got {}", - self.discount_gamma - )); - } - if self.max_reward < self.min_reward { - return Err(format!( - "max_reward must be >= min_reward (got {} < {})", - self.max_reward, self.min_reward - )); - } - - let min_shifted = (self.min_reward as i128) + (self.reward_offset as i128); - let max_shifted = (self.max_reward as i128) + (self.reward_offset as i128); - if min_shifted < 0 { - return Err(format!( - "reward_offset too small: min_reward + reward_offset must be >= 0 (got {})", - min_shifted - )); - } - if self.reward_bits < 64 { - let max_enc = (1u128 << self.reward_bits) - 1; - if (max_shifted as u128) > max_enc { - return Err(format!( - "reward_bits too small for configured reward range: max shifted reward {} exceeds {}", - max_shifted, max_enc - )); - } - } - - if let Some(rate_backend) = &self.rate_backend { - validate_rate_backend(rate_backend) - .map_err(|err| format!("invalid rate_backend: {err}"))?; - if rate_backend_contains_zpaq(rate_backend) { - return Err( - "MC-AIXI strict generic rate_backend support requires reversible action conditioning; configured rate_backend contains zpaq which does not provide the reversible action conditioning required by \"A Monte-Carlo AIXI Approximation\"" - .to_string(), - ); - } - return Ok(()); - } - - match self.algorithm.as_str() { - "ctw" | "fac-ctw" | "ac-ctw" | "ctw-context-tree" | "rosa" => {} - #[cfg(feature = "backend-rwkv")] - "rwkv" => { - let has_method = self - .rwkv_method - .as_deref() - .map(str::trim) - .is_some_and(|v| !v.is_empty()); - let has_path = self - .rwkv_model_path - .as_deref() - .map(str::trim) - .is_some_and(|v| !v.is_empty()); - if !(has_method || has_path) { - return Err( - "algorithm=rwkv requires rwkv_model_path or rwkv_method when no rate_backend override is configured" - .to_string(), - ); - } - } - #[cfg(not(feature = "backend-rwkv"))] - "rwkv" => return Err("algorithm=rwkv requires backend-rwkv feature".to_string()), - #[cfg(feature = "backend-mamba")] - "mamba" => { - let has_method = self - .mamba_method - .as_deref() - .map(str::trim) - .is_some_and(|v| !v.is_empty()); - let has_path = self - .mamba_model_path - .as_deref() - .map(str::trim) - .is_some_and(|v| !v.is_empty()); - if !(has_method || has_path) { - return Err( - "algorithm=mamba requires mamba_model_path or mamba_method when no rate_backend override is configured" - .to_string(), - ); - } - } - #[cfg(not(feature = "backend-mamba"))] - "mamba" => return Err("algorithm=mamba requires backend-mamba feature".to_string()), - "zpaq" => { - let method = self.zpaq_method.as_deref().unwrap_or("1"); - if let Err(err) = validate_zpaq_rate_method(method) { - return Err(format!("Invalid zpaq method for AIXI: {err}")); - } - } - other => return Err(format!("Unknown algorithm: {other}")), - } - - Ok(()) - } -} - -/// A complete MC-AIXI agent. -/// -/// The agent maintains an internal world model and a planning tree. It can -/// be used for both live interaction with an environment and for -/// "imaginary" simulations during planning. -pub struct Agent { - /// The world model used for prediction. - model: Box, - /// The MCTS planner, temporarily taken during search. - planner: Option, - /// Configuration settings. - config: AgentConfig, - - /// Total number of interaction cycles. - age: u64, - /// Accumulated reward. - total_reward: f64, - - /// Pre-calculated bit depth for actions based on `agent_actions`. - action_bits: usize, - - /// Internal PRNG for simulations. - rng: RandomGenerator, - - /// Recycled buffer for observation generation during planning. - obs_buffer: Vec, - /// Recycled buffer for symbol processing. - sym_buffer: Vec, -} - -impl Agent { - /// Creates a new `Agent` with the given configuration. - pub fn new(config: AgentConfig) -> Self { - Self::try_new(config).unwrap_or_else(|err| panic!("Invalid MC-AIXI config: {err}")) - } - - /// Creates a new `Agent` with the given configuration, returning a validation error on failure. - pub fn try_new(config: AgentConfig) -> Result { - config.validate()?; - - let mut action_bits = 0; - let mut c = 1; - let mut i = 1; - while i < config.agent_actions { - i *= 2; - action_bits = c; - c += 1; - } - if config.agent_actions == 1 { - action_bits = 1; - } - - let model = build_model(&config)?; - - let rng = if let Some(seed) = config.random_seed { - RandomGenerator::from_seed(seed) - } else { - RandomGenerator::new() - }; - - Ok(Self { - model, - planner: Some(SearchTree::new()), - config, - age: 0, - total_reward: 0.0, - action_bits, - rng, - obs_buffer: Vec::with_capacity(128), - sym_buffer: Vec::with_capacity(64), - }) - } - - fn clone_for_simulation(&self, seed: u64) -> Self { - Self { - model: self.model.boxed_clone(), - planner: None, - config: self.config.clone(), - age: self.age, - total_reward: self.total_reward, - action_bits: self.action_bits, - rng: self.rng.fork_with(seed), - obs_buffer: Vec::with_capacity(128), - sym_buffer: Vec::with_capacity(64), - } - } - - /// Resets the agent's interaction statistics. - pub fn reset(&mut self) { - self.age = 0; - self.total_reward = 0.0; - } - - /// Primary interface for decision making. - /// - /// Uses MCTS to find the action that maximizes expected future reward. - pub fn get_planned_action( - &mut self, - prev_obs_stream: &[PerceptVal], - prev_rew: Reward, - prev_act: Action, - ) -> Action { - let mut planner = self.planner.take().expect("Planner missing"); - let num_sim = self.config.num_simulations; - let action = planner.search(self, prev_obs_stream, prev_rew, prev_act, num_sim); - self.planner = Some(planner); - action - } - - /// Updates the world model with real-world percepts. - pub fn model_update_percept(&mut self, observation: PerceptVal, reward: Reward) { - self.model_update_percept_stream(&[observation], reward); - } - - /// Updates the world model with an observation stream and a terminal reward. - pub fn model_update_percept_stream(&mut self, observations: &[PerceptVal], reward: Reward) { - debug_assert!( - !observations.is_empty() || self.config.observation_bits == 0, - "percept update missing observation stream" - ); - let mut percept_syms = Vec::new(); - for &obs in observations { - encode(&mut percept_syms, obs, self.config.observation_bits); - } - crate::aixi::common::encode_reward_offset( - &mut percept_syms, - reward, - self.config.reward_bits, - self.config.reward_offset, - ); - - for &sym in &percept_syms { - self.model.commit_update(sym); - } - - self.total_reward += reward as f64; - } - - /// Computes the observation key used for search-tree branching. - pub fn observation_repr_from_stream(&self, observations: &[PerceptVal]) -> Vec { - observation_repr_from_stream( - self.config.observation_key_mode, - observations, - self.config.observation_bits, - ) - } - - /// Explicitly updates the world model with an action. - pub fn model_update_action_external(&mut self, action: Action) { - self.sym_buffer.clear(); - encode(&mut self.sym_buffer, action, self.action_bits); - - for &sym in &self.sym_buffer { - self.model.commit_update_history(sym); - } - } -} - -fn build_model(config: &AgentConfig) -> Result, String> { - if let Some(rate_backend) = config.rate_backend.clone() { - let bit_backend = adapt_rate_backend_for_bit_tokens(rate_backend); - let predictor = RateBackendBitPredictor::new(bit_backend, config.rate_backend_max_order)?; - return Ok(Box::new(predictor)); - } - - match config.algorithm.as_str() { - // FAC-CTW is the default and recommended CTW variant in - // "A Monte-Carlo AIXI Approximation". - "ctw" | "fac-ctw" => { - let obs_len = config.observation_stream_len.max(1); - let percept_bits = (config.observation_bits * obs_len) + config.reward_bits; - Ok(Box::new(FacCtwPredictor::new( - config.ct_depth, - percept_bits, - ))) - } - // AC-CTW is the legacy single-tree variant - "ac-ctw" | "ctw-context-tree" => Ok(Box::new(CtwPredictor::new(config.ct_depth))), - "rosa" => { - let max_order = config.rosa_max_order.unwrap_or(20); - Ok(Box::new(RosaPredictor::new(max_order))) - } - #[cfg(feature = "backend-rwkv")] - "rwkv" => { - if let Some(method) = config - .rwkv_method - .as_deref() - .map(str::trim) - .filter(|v| !v.is_empty()) - { - let predictor = RwkvPredictor::from_method(method) - .map_err(|err| format!("Invalid RWKV method for AIXI: {err}"))?; - Ok(Box::new(predictor)) - } else { - let path = config.rwkv_model_path.as_ref().ok_or_else(|| { - "RWKV model path required when rwkv_method is not configured".to_string() - })?; - let model_arc = load_rwkv7_model_from_path(path); - Ok(Box::new(RwkvPredictor::new(model_arc))) - } - } - #[cfg(not(feature = "backend-rwkv"))] - "rwkv" => Err("RWKV backend disabled at compile time".to_string()), - #[cfg(feature = "backend-mamba")] - "mamba" => { - if let Some(method) = config - .mamba_method - .as_deref() - .map(str::trim) - .filter(|v| !v.is_empty()) - { - let predictor = MambaPredictor::from_method(method) - .map_err(|err| format!("Invalid Mamba method for AIXI: {err}"))?; - Ok(Box::new(predictor)) - } else { - let path = config.mamba_model_path.as_ref().ok_or_else(|| { - "Mamba model path required when mamba_method is not configured".to_string() - })?; - let model_arc = load_mamba_model_from_path(path); - Ok(Box::new(MambaPredictor::new(model_arc))) - } - } - #[cfg(not(feature = "backend-mamba"))] - "mamba" => Err("Mamba backend disabled at compile time".to_string()), - "zpaq" => { - let method = config - .zpaq_method - .clone() - .unwrap_or_else(|| "1".to_string()); - if let Err(err) = validate_zpaq_rate_method(&method) { - return Err(format!("Invalid zpaq method for AIXI: {err}")); - } - Ok(Box::new(ZpaqPredictor::new(method, 2f64.powi(-24)))) - } - _ => Err(format!("Unknown algorithm: {}", config.algorithm)), - } -} - -impl AgentSimulator for Agent { - fn get_num_actions(&self) -> usize { - self.config.agent_actions - } - - fn get_num_observation_bits(&self) -> usize { - self.config.observation_bits - } - - fn observation_stream_len(&self) -> usize { - self.config.observation_stream_len.max(1) - } - - fn observation_key_mode(&self) -> ObservationKeyMode { - self.config.observation_key_mode - } - - fn get_num_reward_bits(&self) -> usize { - self.config.reward_bits - } - - fn horizon(&self) -> usize { - self.config.agent_horizon - } - - fn max_reward(&self) -> Reward { - self.config.max_reward - } - - fn min_reward(&self) -> Reward { - self.config.min_reward - } - - fn reward_offset(&self) -> i64 { - self.config.reward_offset - } - - fn get_explore_exploit_ratio(&self) -> f64 { - self.config.exploration_exploitation_ratio - } - - fn discount_gamma(&self) -> f64 { - self.config.discount_gamma - } - - fn model_update_action(&mut self, action: Action) { - self.sym_buffer.clear(); - encode(&mut self.sym_buffer, action, self.action_bits); - - for &sym in &self.sym_buffer { - self.model.update_history(sym); - } - } - - fn gen_percept_and_update(&mut self, bits: usize) -> u64 { - self.sym_buffer.clear(); - for _ in 0..bits { - let prob_1 = self.model.predict_one(); - let sym = self.rng.gen_bool(prob_1); - self.model.update(sym); - self.sym_buffer.push(sym); - } - decode(&self.sym_buffer, bits) - } - - fn begin_simulation(&mut self) { - self.model.begin_rollback_scope(); - } - - fn gen_percepts_and_update(&mut self) -> (Vec, Reward) { - let obs_bits = self.config.observation_bits; - let obs_len = self.config.observation_stream_len.max(1); - - self.obs_buffer.clear(); - for _ in 0..obs_len { - let p = self.gen_percept_and_update(obs_bits); - self.obs_buffer.push(p); - } - - let obs_repr = observation_repr_from_stream( - self.config.observation_key_mode, - &self.obs_buffer, - obs_bits, - ); - let rew_bits = self.config.reward_bits; - let rew_u = self.gen_percept_and_update(rew_bits); - let rew = (rew_u as i64) - self.config.reward_offset; - - // Mark that we've completed a percept cycle (ready for next action) - - (obs_repr, rew) - } - - fn gen_range(&mut self, end: usize) -> usize { - self.rng.gen_range(end) - } - - fn gen_f64(&mut self) -> f64 { - self.rng.gen_f64() - } - - fn model_revert(&mut self, steps: usize) { - if self.model.rollback_scope() { - return; - } - let obs_bits = self.config.observation_bits * self.config.observation_stream_len.max(1); - let percept_bits = obs_bits + self.config.reward_bits; - - for _ in 0..steps { - for _ in 0..percept_bits { - self.model.revert(); - } - for _ in 0..self.action_bits { - self.model.pop_history(); - } - } - } - - fn boxed_clone_with_seed(&self, seed: u64) -> Box { - Box::new(self.clone_for_simulation(seed)) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use std::sync::{Arc, Mutex}; - - #[derive(Clone, Default)] - struct CallCounts { - update: usize, - commit_update: usize, - update_history: usize, - commit_update_history: usize, - begin_scope: usize, - rollback_scope: usize, - revert: usize, - pop_history: usize, - } - - #[derive(Clone)] - struct InstrumentedPredictor { - counts: Arc>, - } - - impl InstrumentedPredictor { - fn new(counts: Arc>) -> Self { - Self { counts } - } - } - - impl Predictor for InstrumentedPredictor { - fn update(&mut self, _sym: bool) { - self.counts.lock().unwrap().update += 1; - } - - fn commit_update(&mut self, _sym: bool) { - self.counts.lock().unwrap().commit_update += 1; - } - - fn update_history(&mut self, _sym: bool) { - self.counts.lock().unwrap().update_history += 1; - } - - fn commit_update_history(&mut self, _sym: bool) { - self.counts.lock().unwrap().commit_update_history += 1; - } - - fn revert(&mut self) { - self.counts.lock().unwrap().revert += 1; - } - - fn pop_history(&mut self) { - self.counts.lock().unwrap().pop_history += 1; - } - - fn begin_rollback_scope(&mut self) { - self.counts.lock().unwrap().begin_scope += 1; - } - - fn rollback_scope(&mut self) -> bool { - self.counts.lock().unwrap().rollback_scope += 1; - true - } - - fn predict_prob(&mut self, sym: bool) -> f64 { - if sym { 0.75 } else { 0.25 } - } - - fn model_name(&self) -> String { - "InstrumentedPredictor".to_string() - } - - fn boxed_clone(&self) -> Box { - Box::new(self.clone()) - } - } - - fn basic_config() -> AgentConfig { - AgentConfig { - algorithm: "ac-ctw".to_string(), - ct_depth: 8, - agent_horizon: 2, - observation_bits: 2, - observation_stream_len: 2, - observation_key_mode: ObservationKeyMode::FullStream, - reward_bits: 3, - agent_actions: 4, - num_simulations: 2, - exploration_exploitation_ratio: 1.0, - discount_gamma: 0.95, - min_reward: -2, - max_reward: 3, - reward_offset: 2, - random_seed: Some(7), - rate_backend: None, - rate_backend_max_order: 8, - rwkv_model_path: None, - rwkv_method: None, - mamba_model_path: None, - mamba_method: None, - rosa_max_order: None, - zpaq_method: None, - } - } - - #[test] - fn external_history_updates_use_committed_predictor_paths() { - let mut agent = Agent::try_new(basic_config()).expect("valid agent config"); - let counts = Arc::new(Mutex::new(CallCounts::default())); - agent.model = Box::new(InstrumentedPredictor::new(counts.clone())); - - agent.model_update_percept_stream(&[1, 2], 1); - agent.model_update_action_external(3); - - let snapshot = counts.lock().unwrap().clone(); - assert_eq!(snapshot.commit_update, 7); - assert_eq!(snapshot.commit_update_history, 2); - assert_eq!(snapshot.update, 0); - assert_eq!(snapshot.update_history, 0); - } - - #[test] - fn simulation_revert_prefers_predictor_scope_when_available() { - let mut agent = Agent::try_new(basic_config()).expect("valid agent config"); - let counts = Arc::new(Mutex::new(CallCounts::default())); - agent.model = Box::new(InstrumentedPredictor::new(counts.clone())); - - AgentSimulator::begin_simulation(&mut agent); - agent.model_revert(3); - - let snapshot = counts.lock().unwrap().clone(); - assert_eq!(snapshot.begin_scope, 1); - assert_eq!(snapshot.rollback_scope, 1); - assert_eq!(snapshot.revert, 0); - assert_eq!(snapshot.pop_history, 0); - } -} diff --git a/src/aixi/aiqi.rs b/src/aixi/aiqi.rs deleted file mode 100644 index 0487d544..00000000 --- a/src/aixi/aiqi.rs +++ /dev/null @@ -1,1479 +0,0 @@ -//! AIQI implementation from "A Model-Free Universal AI". -//! -//! This module implements a model-free universal agent that predicts -//! discretized H-step returns directly from augmented interaction history. -//! The implementation follows the phase-indexed periodic augmentation in -//! "A Model-Free Universal AI": -//! for return horizon `H` and period `N >= H`, each phase model only inserts -//! returns at indices `i % N == phase`. - -use crate::aixi::common::{Action, PerceptVal, RandomGenerator, Reward}; -use crate::aixi::model::{CtwPredictor, FacCtwPredictor, Predictor, RateBackendBitPredictor}; -use crate::aixi::rate_backend::rate_backend_contains_zpaq; -#[cfg(feature = "backend-rwkv")] -use crate::load_rwkv7_model_from_path; -use crate::{RateBackend, validate_rate_backend}; - -/// Configuration parameters for an AIQI agent. -#[derive(Clone)] -pub struct AiqiConfig { - /// Predictive backend. - /// - /// - `ac-ctw` / `ctw` / `ctw-context-tree`: AIQI-CTW path from - /// "A Model-Free Universal AI". - /// - `fac-ctw`: factorized CTW extension. - /// - `rosa` / `rwkv`: pluggable predictor extensions. - /// - `zpaq`: intentionally unsupported for AIQI strict conditioning. - pub algorithm: String, - /// Context depth for CTW/FAC-CTW backends. - pub ct_depth: usize, - /// Number of bits used to encode observations. - pub observation_bits: usize, - /// Number of observation symbols per environment step. - pub observation_stream_len: usize, - /// Number of bits used to encode rewards. - pub reward_bits: usize, - /// Number of valid actions. - pub agent_actions: usize, - /// Minimum possible environment reward. - pub min_reward: Reward, - /// Maximum possible environment reward. - pub max_reward: Reward, - /// Offset applied before encoding reward bits. - pub reward_offset: Reward, - /// Discount factor used when constructing H-step returns. - pub discount_gamma: f64, - /// Return horizon `H`. - pub return_horizon: usize, - /// Number of discretization bins `M` for returns. - /// - /// This implementation uses exact fixed-width binary encoding of return bins, - /// so `return_bins` must be a power of two. - pub return_bins: usize, - /// Augmentation period `N` (must satisfy `N >= H`). - pub augmentation_period: usize, - /// Optional history retention knob for bounded memory growth. - /// - /// - `None`: keep full history (default behavior, no pruning). - /// - `Some(k)`: keep at least the most recent `k` steps, while also - /// preserving all steps still required for exact return construction and - /// deferred phase-model advancement. - pub history_prune_keep_steps: Option, - /// Baseline epsilon-greedy exploration probability `tau`. - pub baseline_exploration: f64, - /// Optional deterministic RNG seed for action selection/exploration. - /// - /// When `None`, a fresh runtime-derived seed is used. - pub random_seed: Option, - /// Optional generic rate backend. - /// - /// When set, this takes precedence over `algorithm` and routes AIQI - /// prediction through the shared `RateBackend` abstraction. - pub rate_backend: Option, - /// Max-order hint for `rate_backend` constructors that use it (for example ROSA). - pub rate_backend_max_order: i64, - /// Optional RWKV model path. - /// - /// Required only when selecting `algorithm="rwkv"` and no `rate_backend` - /// override is configured. - pub rwkv_model_path: Option, - /// Optional ROSA max order. - pub rosa_max_order: Option, - /// Optional ZPAQ method string. - pub zpaq_method: Option, -} - -impl AiqiConfig { - /// Validate configuration constraints. - pub fn validate(&self) -> Result<(), String> { - if self.agent_actions == 0 { - return Err("agent_actions must be >= 1".to_string()); - } - if self.return_horizon == 0 { - return Err("return_horizon must be >= 1".to_string()); - } - if self.return_bins == 0 { - return Err("return_bins must be >= 1".to_string()); - } - if !self.return_bins.is_power_of_two() { - return Err(format!( - "return_bins must be a power of two for exact binary return encoding, got {}", - self.return_bins - )); - } - if self.augmentation_period < self.return_horizon { - return Err(format!( - "augmentation_period must be >= return_horizon (got N={}, H={})", - self.augmentation_period, self.return_horizon - )); - } - if !(0.0 < self.discount_gamma && self.discount_gamma < 1.0) { - return Err(format!( - "discount_gamma must be in (0, 1) for AIQI as defined in \"A Model-Free Universal AI\", got {}", - self.discount_gamma - )); - } - if !(0.0 < self.baseline_exploration && self.baseline_exploration <= 1.0) { - return Err(format!( - "baseline_exploration (tau) must be in (0, 1] for AIQI as defined in \"A Model-Free Universal AI\", got {}", - self.baseline_exploration - )); - } - if self.max_reward < self.min_reward { - return Err(format!( - "max_reward must be >= min_reward (got {} < {})", - self.max_reward, self.min_reward - )); - } - - // `rate_backend` takes precedence over `algorithm`; only validate - // algorithm choices when no backend override is configured. - if self.rate_backend.is_none() { - match self.algorithm.as_str() { - "ctw" | "fac-ctw" | "ac-ctw" | "ctw-context-tree" | "rosa" => {} - "zpaq" => { - return Err( - "AIQI strict mode does not support algorithm=zpaq: zpaq backends do not provide strict frozen conditioning" - .to_string(), - ) - } - #[cfg(feature = "backend-rwkv")] - "rwkv" => {} - #[cfg(not(feature = "backend-rwkv"))] - "rwkv" => { - return Err("algorithm=rwkv requires backend-rwkv feature".to_string()) - } - other => return Err(format!("Unknown AIQI algorithm: {other}")), - } - } - - if let Some(rate_backend) = &self.rate_backend { - validate_rate_backend(rate_backend) - .map_err(|err| format!("invalid rate_backend: {err}"))?; - if !rate_backend_supports_aiqi_frozen_conditioning(rate_backend) { - return Err( - "AIQI strict mode requires frozen context updates; configured rate_backend contains zpaq which does not provide strict frozen conditioning" - .to_string(), - ); - } - } - - #[cfg(feature = "backend-rwkv")] - if self.rate_backend.is_none() && self.algorithm == "rwkv" { - match self.rwkv_model_path.as_deref() { - Some(path) if !path.trim().is_empty() => {} - _ => { - return Err( - "algorithm=rwkv requires rwkv_model_path when no rate_backend override is configured; for method-string RWKV configure rate_backend rwkv/rwkv7" - .to_string(), - ) - } - } - } - - let min_shifted = (self.min_reward as i128) + (self.reward_offset as i128); - let max_shifted = (self.max_reward as i128) + (self.reward_offset as i128); - if min_shifted < 0 { - return Err(format!( - "reward_offset too small: min_reward + reward_offset must be >= 0 (got {})", - min_shifted - )); - } - if self.reward_bits < 64 { - let max_enc = (1u128 << self.reward_bits) - 1; - if (max_shifted as u128) > max_enc { - return Err(format!( - "reward_bits too small for configured reward range: max shifted reward {} exceeds {}", - max_shifted, max_enc - )); - } - } - - Ok(()) - } -} - -#[derive(Clone, Debug)] -struct StepRecord { - action: Action, - observations: Vec, - reward: Reward, -} - -struct PhaseModel { - predictor: Box, - // Largest step index for which this phase model has consumed - // the augmented stream up to and including that step's percept. - last_augmented_step: usize, -} - -/// AIQI agent with phase-indexed augmented return predictors. -pub struct AiqiAgent { - config: AiqiConfig, - phases: Vec, - steps: Vec, - return_bins_by_step: Vec>, - // Global 1-based index of steps[0] / return_bins_by_step[0]. - history_base_step: usize, - // Total number of transitions observed so far (global 1-based max step index). - total_steps_observed: usize, - action_bits: usize, - return_bits: usize, - use_generic_planner: bool, - distribution_uses_training_updates: bool, - rng: RandomGenerator, -} - -impl AiqiAgent { - /// Construct a new AIQI agent. - pub fn new(config: AiqiConfig) -> Result { - config.validate()?; - - let action_bits = bits_for_cardinality(config.agent_actions); - let return_bits = bits_for_cardinality(config.return_bins); - let use_generic_planner = aiqi_requires_generic_planner(&config); - let distribution_uses_training_updates = config.rate_backend.is_none() - && matches!( - config.algorithm.as_str(), - "ctw" | "fac-ctw" | "ac-ctw" | "ctw-context-tree" - ); - - let mut phases = Vec::with_capacity(config.augmentation_period); - for _ in 0..config.augmentation_period { - phases.push(PhaseModel { - predictor: build_predictor(&config, return_bits)?, - last_augmented_step: 0, - }); - } - - let rng = if let Some(seed) = config.random_seed { - RandomGenerator::from_seed(seed) - } else { - RandomGenerator::new() - }; - - Ok(Self { - action_bits, - return_bits, - use_generic_planner, - distribution_uses_training_updates, - config, - phases, - steps: Vec::new(), - return_bins_by_step: Vec::new(), - history_base_step: 1, - total_steps_observed: 0, - rng, - }) - } - - /// Number of transitions incorporated so far. - pub fn steps_observed(&self) -> usize { - self.total_steps_observed - } - - /// Returns the configured number of actions. - pub fn num_actions(&self) -> usize { - self.config.agent_actions - } - - /// Select the next action from the current history. - pub fn get_planned_action(&mut self) -> Action { - let q_values = self.estimate_q_values(); - let greedy_action = argmax_with_fixed_tie_break(&q_values) as u64; - if self.config.baseline_exploration > 0.0 - && self - .rng - .gen_bool(self.config.baseline_exploration.clamp(0.0, 1.0)) - { - self.rng.gen_range(self.config.agent_actions) as u64 - } else { - greedy_action - } - } - - /// Select the next action, adding optional extra exploration. - /// - /// The extra exploration probability is combined as - /// `p = 1 - (1 - tau) * (1 - extra)`, where `tau` is the baseline - /// exploration in [`AiqiConfig`]. - pub fn get_planned_action_with_extra_exploration(&mut self, extra_exploration: f64) -> Action { - let extra = extra_exploration.clamp(0.0, 1.0); - let tau = self.config.baseline_exploration.clamp(0.0, 1.0); - let effective = 1.0 - (1.0 - tau) * (1.0 - extra); - let q_values = self.estimate_q_values(); - let greedy_action = argmax_with_fixed_tie_break(&q_values) as u64; - if effective > 0.0 && self.rng.gen_bool(effective) { - self.rng.gen_range(self.config.agent_actions) as u64 - } else { - greedy_action - } - } - - /// Record one environment transition `(action, observations, reward)`. - /// - /// This appends to history and, when enough future rewards are known, - /// computes and learns one newly available discretized return. - pub fn observe_transition( - &mut self, - action: Action, - observations: &[PerceptVal], - reward: Reward, - ) -> Result<(), String> { - if action as usize >= self.config.agent_actions { - return Err(format!( - "action out of range: action={} but agent_actions={}", - action, self.config.agent_actions - )); - } - - let expected_obs = self.config.observation_stream_len.max(1); - if observations.len() != expected_obs { - return Err(format!( - "observation stream length mismatch: expected {}, got {}", - expected_obs, - observations.len() - )); - } - - if reward < self.config.min_reward || reward > self.config.max_reward { - return Err(format!( - "reward out of configured range: reward={} not in [{}, {}]", - reward, self.config.min_reward, self.config.max_reward - )); - } - - let obs_max = max_value_for_bits(self.config.observation_bits); - for &obs in observations { - if obs > obs_max { - return Err(format!( - "observation value {} does not fit observation_bits={} (max={})", - obs, self.config.observation_bits, obs_max - )); - } - } - - let rew_shifted = (reward as i128) + (self.config.reward_offset as i128); - if rew_shifted < 0 { - return Err(format!( - "encoded reward became negative after offset: reward={} offset={}", - reward, self.config.reward_offset - )); - } - if self.config.reward_bits < 64 { - let max_enc = (1u128 << self.config.reward_bits) - 1; - if (rew_shifted as u128) > max_enc { - return Err(format!( - "encoded reward {} exceeds reward_bits={} capacity {}", - rew_shifted, self.config.reward_bits, max_enc - )); - } - } - - self.steps.push(StepRecord { - action, - observations: observations.to_vec(), - reward, - }); - self.total_steps_observed += 1; - self.return_bins_by_step.push(None); - - self.maybe_learn_new_return()?; - self.maybe_prune_history(); - Ok(()) - } - - fn maybe_learn_new_return(&mut self) -> Result<(), String> { - let t = self.total_steps_observed; - let h = self.config.return_horizon; - if t < h { - return Ok(()); - } - - // Newly available return index (1-based): i = t - H + 1. - let i = t + 1 - h; - let bin = self.compute_return_bin(i); - let local_idx = self.local_index(i)?; - self.return_bins_by_step[local_idx] = Some(bin); - - let phase = i % self.config.augmentation_period; - self.advance_phase_model_to_step(phase, i) - } - - fn estimate_q_values(&mut self) -> Vec { - if self.use_generic_planner { - return self.estimate_q_values_generic(); - } - - let step = self.total_steps_observed + 1; - let phase = step % self.config.augmentation_period; - let config = &self.config; - let steps = &self.steps; - let return_bins_by_step = &self.return_bins_by_step; - let history_base_step = self.history_base_step; - let action_bits = self.action_bits; - let return_bits = self.return_bits; - - let mut q_values = vec![0.0; self.config.agent_actions]; - let mut pushed_fast_forward = 0usize; - - { - let model = &mut self.phases[phase]; - let start = (model.last_augmented_step + 1).max(history_base_step); - let end = step.saturating_sub(1); - if start <= end { - for idx in start..=end { - pushed_fast_forward += push_step_tokens_history( - config, - history_base_step, - steps, - return_bins_by_step, - action_bits, - return_bits, - model.predictor.as_mut(), - phase, - idx, - ); - } - } - - for action in 0..self.config.agent_actions { - let pushed_action = push_encoded_bits_history( - model.predictor.as_mut(), - action as u64, - self.action_bits, - ); - let dist = Self::predict_return_distribution( - self.config.return_bins, - self.return_bits, - model.predictor.as_mut(), - self.distribution_uses_training_updates, - ); - q_values[action] = expectation_from_distribution(&dist); - pop_history_bits(model.predictor.as_mut(), pushed_action); - } - - pop_history_bits(model.predictor.as_mut(), pushed_fast_forward); - } - - q_values - } - - fn estimate_q_values_generic(&mut self) -> Vec { - let step = self.total_steps_observed + 1; - let phase = step % self.config.augmentation_period; - - let model = &self.phases[phase]; - let mut context_predictor = model.predictor.boxed_clone(); - - let start = (model.last_augmented_step + 1).max(self.history_base_step); - let end = step.saturating_sub(1); - if start <= end { - for idx in start..=end { - push_augmented_step_tokens_commit( - &self.config, - self.history_base_step, - &self.steps, - &self.return_bins_by_step, - self.action_bits, - self.return_bits, - context_predictor.as_mut(), - phase, - idx, - ) - .expect("generic planner retained history must contain required augmented return"); - } - } - - let mut q_values = vec![0.0; self.config.agent_actions]; - for action in 0..self.config.agent_actions { - let mut action_predictor = context_predictor.boxed_clone(); - let _ = push_encoded_bits_commit_history( - action_predictor.as_mut(), - action as u64, - self.action_bits, - ); - let dist = Self::predict_return_distribution_from_base_predictor( - self.config.return_bins, - self.return_bits, - action_predictor.as_ref(), - ); - q_values[action] = expectation_from_distribution(&dist); - } - - q_values - } - - fn predict_return_distribution( - return_bins: usize, - return_bits: usize, - predictor: &mut dyn Predictor, - use_training_updates: bool, - ) -> Vec { - debug_assert!(return_bins.is_power_of_two()); - if return_bins == 1 { - return vec![1.0]; - } - - let mut probs = vec![0.0; return_bins]; - for (bin, slot) in probs.iter_mut().enumerate() { - let mut p = 1.0f64; - let mut v = bin as u64; - for _ in 0..return_bits { - let bit = (v & 1) == 1; - v >>= 1; - let q = predictor.predict_prob(bit).clamp(1e-12, 1.0 - 1e-12); - p *= q; - if use_training_updates { - predictor.update(bit); - } else { - predictor.update_history(bit); - } - } - if use_training_updates { - revert_bits(predictor, return_bits); - } else { - pop_history_bits(predictor, return_bits); - } - *slot = p; - } - - let sum: f64 = probs.iter().sum(); - if !sum.is_finite() || sum <= 0.0 { - let u = 1.0 / (return_bins as f64); - probs.fill(u); - return probs; - } - - for p in &mut probs { - *p /= sum; - } - probs - } - - fn predict_return_distribution_from_base_predictor( - return_bins: usize, - return_bits: usize, - base_predictor: &dyn Predictor, - ) -> Vec { - debug_assert!(return_bins.is_power_of_two()); - if return_bins == 1 { - return vec![1.0]; - } - - let mut probs = vec![0.0; return_bins]; - for (bin, slot) in probs.iter_mut().enumerate() { - let mut predictor = base_predictor.boxed_clone(); - let mut p = 1.0f64; - let mut v = bin as u64; - for _ in 0..return_bits { - let bit = (v & 1) == 1; - v >>= 1; - let q = predictor.predict_prob(bit).clamp(1e-12, 1.0 - 1e-12); - p *= q; - predictor.commit_update(bit); - } - *slot = p; - } - - let sum: f64 = probs.iter().sum(); - if !sum.is_finite() || sum <= 0.0 { - let u = 1.0 / (return_bins as f64); - probs.fill(u); - return probs; - } - - for p in &mut probs { - *p /= sum; - } - probs - } - - fn advance_phase_model_to_step( - &mut self, - phase: usize, - target_step: usize, - ) -> Result<(), String> { - let config = &self.config; - let steps = &self.steps; - let return_bins_by_step = &self.return_bins_by_step; - let history_base_step = self.history_base_step; - let action_bits = self.action_bits; - let return_bits = self.return_bits; - let model = &mut self.phases[phase]; - if target_step <= model.last_augmented_step { - return Ok(()); - } - - let start = (model.last_augmented_step + 1).max(history_base_step); - for idx in start..=target_step { - push_augmented_step_tokens_commit( - config, - history_base_step, - steps, - return_bins_by_step, - action_bits, - return_bits, - model.predictor.as_mut(), - phase, - idx, - )?; - } - - model.last_augmented_step = target_step; - Ok(()) - } - - fn compute_return_bin(&self, start_step: usize) -> u64 { - let h = self.config.return_horizon; - let gamma = self.config.discount_gamma; - - debug_assert!(gamma > 0.0 && gamma < 1.0); - let reward_range = (self.config.max_reward - self.config.min_reward) as f64; - - // Paper definition: R_{t,H} = (1-gamma) * sum_{k=0}^{H-1} gamma^k r_{t+k}. - let mut total = 0.0f64; - let mut gk = 1.0f64; - for k in 0..h { - let idx = start_step + k; - let local_idx = self - .local_index(idx) - .expect("return computation requires in-range history"); - let r = self.steps[local_idx].reward; - let rn = if reward_range <= 0.0 { - 0.0 - } else { - ((r - self.config.min_reward) as f64 / reward_range).clamp(0.0, 1.0) - }; - total += gk * rn; - gk *= gamma; - } - let ret = ((1.0 - gamma) * total).clamp(0.0, 1.0); - - let mut bin = (ret * (self.config.return_bins as f64)).floor() as u64; - let max_bin = (self.config.return_bins as u64).saturating_sub(1); - if bin > max_bin { - bin = max_bin; - } - bin - } - - fn local_index(&self, global_step: usize) -> Result { - if global_step < self.history_base_step || global_step > self.total_steps_observed { - return Err(format!( - "global step {} out of retained history range [{}, {}]", - global_step, self.history_base_step, self.total_steps_observed - )); - } - Ok(global_step - self.history_base_step) - } - - fn maybe_prune_history(&mut self) { - let Some(keep_steps) = self.config.history_prune_keep_steps else { - return; - }; - if self.steps.is_empty() { - return; - } - - let min_phase_committed = self - .phases - .iter() - .map(|phase| phase.last_augmented_step) - .min() - .unwrap_or(0); - - // For the next return update, we must retain steps from - // (t+2-H) onward (1-based indexing). Everything before that is no - // longer needed for exact H-step return construction. - let next_start_needed = self - .total_steps_observed - .saturating_add(2) - .saturating_sub(self.config.return_horizon); - let returns_safe_drop_upto = next_start_needed.saturating_sub(1); - - let mut safe_drop_upto = min_phase_committed.min(returns_safe_drop_upto); - - // Optional retention floor: keep at least `keep_steps` most recent - // transitions in memory for diagnostics/debugging. - let keep_floor_drop_upto = self.total_steps_observed.saturating_sub(keep_steps); - safe_drop_upto = safe_drop_upto.min(keep_floor_drop_upto); - - if safe_drop_upto < self.history_base_step { - return; - } - - let drain_count = safe_drop_upto - self.history_base_step + 1; - if drain_count == 0 || drain_count > self.steps.len() { - return; - } - - self.steps.drain(0..drain_count); - self.return_bins_by_step.drain(0..drain_count); - self.history_base_step += drain_count; - } -} - -fn push_step_tokens_history( - config: &AiqiConfig, - history_base_step: usize, - steps: &[StepRecord], - return_bins_by_step: &[Option], - action_bits: usize, - return_bits: usize, - predictor: &mut dyn Predictor, - phase: usize, - idx: usize, -) -> usize { - let mut pushed = 0usize; - pushed += push_action_tokens_history(history_base_step, steps, action_bits, predictor, idx); - - if idx % config.augmentation_period == phase { - let local_idx = idx - history_base_step; - if let Some(bin) = return_bins_by_step[local_idx] { - pushed += push_encoded_bits_history(predictor, bin, return_bits); - } - } - - pushed + push_percept_tokens_history(config, history_base_step, steps, predictor, idx) -} - -fn push_augmented_step_tokens_commit( - config: &AiqiConfig, - history_base_step: usize, - steps: &[StepRecord], - return_bins_by_step: &[Option], - action_bits: usize, - return_bits: usize, - predictor: &mut dyn Predictor, - phase: usize, - idx: usize, -) -> Result { - let mut pushed = 0usize; - pushed += - push_action_tokens_commit_history(history_base_step, steps, action_bits, predictor, idx); - - if idx % config.augmentation_period == phase { - let local_idx = idx - history_base_step; - let bin = return_bins_by_step[local_idx].ok_or_else(|| { - format!( - "missing return bin for step {} in phase {} while pushing augmented history", - idx, phase - ) - })?; - pushed += push_encoded_bits_commit(predictor, bin, return_bits); - } - - Ok(pushed - + push_percept_tokens_commit_history(config, history_base_step, steps, predictor, idx)) -} - -fn push_action_tokens_history( - history_base_step: usize, - steps: &[StepRecord], - action_bits: usize, - predictor: &mut dyn Predictor, - idx: usize, -) -> usize { - let action = steps[idx - history_base_step].action; - push_encoded_bits_history(predictor, action, action_bits) -} - -fn push_action_tokens_commit_history( - history_base_step: usize, - steps: &[StepRecord], - action_bits: usize, - predictor: &mut dyn Predictor, - idx: usize, -) -> usize { - let action = steps[idx - history_base_step].action; - push_encoded_bits_commit_history(predictor, action, action_bits) -} - -fn push_percept_tokens_history( - config: &AiqiConfig, - history_base_step: usize, - steps: &[StepRecord], - predictor: &mut dyn Predictor, - idx: usize, -) -> usize { - let step = &steps[idx - history_base_step]; - let mut pushed = 0usize; - for &obs in &step.observations { - pushed += push_encoded_bits_history(predictor, obs, config.observation_bits); - } - pushed - + push_encoded_reward_history( - predictor, - step.reward, - config.reward_bits, - config.reward_offset, - ) -} - -fn push_percept_tokens_commit_history( - config: &AiqiConfig, - history_base_step: usize, - steps: &[StepRecord], - predictor: &mut dyn Predictor, - idx: usize, -) -> usize { - let step = &steps[idx - history_base_step]; - let mut pushed = 0usize; - for &obs in &step.observations { - pushed += push_encoded_bits_commit_history(predictor, obs, config.observation_bits); - } - pushed - + push_encoded_reward_commit_history( - predictor, - step.reward, - config.reward_bits, - config.reward_offset, - ) -} - -fn build_predictor(config: &AiqiConfig, return_bits: usize) -> Result, String> { - if let Some(rate_backend) = config.rate_backend.clone() { - let bit_backend = adapt_rate_backend_for_bit_tokens(rate_backend); - let predictor = RateBackendBitPredictor::new(bit_backend, config.rate_backend_max_order)?; - return Ok(Box::new(predictor)); - } - - match config.algorithm.as_str() { - "ctw" | "ac-ctw" | "ctw-context-tree" => Ok(Box::new(CtwPredictor::new(config.ct_depth))), - "fac-ctw" => { - // AIQI-FAC-CTW extension: factorized return-bit modeling. - Ok(Box::new(FacCtwPredictor::new(config.ct_depth, return_bits))) - } - "rosa" => { - let max_order = config - .rosa_max_order - .unwrap_or(config.rate_backend_max_order); - let bit_backend = adapt_rate_backend_for_bit_tokens(RateBackend::RosaPlus); - let predictor = RateBackendBitPredictor::new(bit_backend, max_order)?; - Ok(Box::new(predictor)) - } - #[cfg(feature = "backend-rwkv")] - "rwkv" => { - let path = config.rwkv_model_path.as_ref().ok_or_else(|| { - "algorithm=rwkv requires rwkv_model_path when no rate_backend override is configured; for method-string RWKV configure rate_backend rwkv/rwkv7" - .to_string() - })?; - let model_arc = load_rwkv7_model_from_path(path); - let bit_backend = - adapt_rate_backend_for_bit_tokens(RateBackend::Rwkv7 { model: model_arc }); - let predictor = RateBackendBitPredictor::new(bit_backend, config.rate_backend_max_order)?; - Ok(Box::new(predictor)) - } - #[cfg(not(feature = "backend-rwkv"))] - "rwkv" => Err("algorithm=rwkv requires backend-rwkv feature".to_string()), - "zpaq" => Err( - "AIQI strict mode does not support algorithm=zpaq; configure a backend with strict frozen conditioning" - .to_string(), - ), - _ => Err(format!("Unknown AIQI algorithm: {}", config.algorithm)), - } -} - -fn adapt_rate_backend_for_bit_tokens(backend: RateBackend) -> RateBackend { - crate::aixi::rate_backend::adapt_rate_backend_for_bit_tokens(backend) -} - -fn rate_backend_supports_aiqi_frozen_conditioning(backend: &RateBackend) -> bool { - !rate_backend_contains_zpaq(backend) -} - -fn aiqi_requires_generic_planner(config: &AiqiConfig) -> bool { - config.rate_backend.is_some() - || !matches!( - config.algorithm.as_str(), - "ctw" | "fac-ctw" | "ac-ctw" | "ctw-context-tree" - ) -} - -fn bits_for_cardinality(cardinality: usize) -> usize { - let n = cardinality.max(1); - let mut bits = 0usize; - while (1usize << bits) < n { - bits += 1; - } - bits.max(1) -} - -fn max_value_for_bits(bits: usize) -> u64 { - if bits >= 64 { - u64::MAX - } else if bits == 0 { - 0 - } else { - (1u64 << bits) - 1 - } -} - -fn push_encoded_bits_commit(predictor: &mut dyn Predictor, value: u64, bits: usize) -> usize { - let mut v = value; - for _ in 0..bits { - predictor.commit_update((v & 1) == 1); - v >>= 1; - } - bits -} - -fn push_encoded_bits_history(predictor: &mut dyn Predictor, value: u64, bits: usize) -> usize { - let mut v = value; - for _ in 0..bits { - predictor.update_history((v & 1) == 1); - v >>= 1; - } - bits -} - -fn push_encoded_bits_commit_history( - predictor: &mut dyn Predictor, - value: u64, - bits: usize, -) -> usize { - let mut v = value; - for _ in 0..bits { - predictor.commit_update_history((v & 1) == 1); - v >>= 1; - } - bits -} - -fn push_encoded_reward_history( - predictor: &mut dyn Predictor, - reward: Reward, - bits: usize, - offset: Reward, -) -> usize { - let shifted = (reward as i128) + (offset as i128); - let as_u64 = if shifted <= 0 { - 0 - } else if shifted > (u64::MAX as i128) { - u64::MAX - } else { - shifted as u64 - }; - push_encoded_bits_history(predictor, as_u64, bits) -} - -fn push_encoded_reward_commit_history( - predictor: &mut dyn Predictor, - reward: Reward, - bits: usize, - offset: Reward, -) -> usize { - let shifted = (reward as i128) + (offset as i128); - let as_u64 = if shifted <= 0 { - 0 - } else if shifted > (u64::MAX as i128) { - u64::MAX - } else { - shifted as u64 - }; - push_encoded_bits_commit_history(predictor, as_u64, bits) -} - -fn pop_history_bits(predictor: &mut dyn Predictor, bits: usize) { - for _ in 0..bits { - predictor.pop_history(); - } -} - -fn revert_bits(predictor: &mut dyn Predictor, bits: usize) { - for _ in 0..bits { - predictor.revert(); - } -} - -fn expectation_from_distribution(probs: &[f64]) -> f64 { - if probs.is_empty() { - return 0.0; - } - let m = probs.len() as f64; - probs - .iter() - .enumerate() - .map(|(i, p)| (i as f64 / m) * p) - .sum::() -} - -fn argmax_with_fixed_tie_break(values: &[f64]) -> usize { - let mut best_value = f64::NEG_INFINITY; - let mut best_idx = 0usize; - for (i, &v) in values.iter().enumerate() { - if v > best_value { - best_value = v; - best_idx = i; - } - } - best_idx -} - -#[cfg(test)] -mod tests { - use super::*; - use std::sync::{Arc, Mutex}; - - fn basic_config() -> AiqiConfig { - AiqiConfig { - algorithm: "ac-ctw".to_string(), - ct_depth: 8, - observation_bits: 1, - observation_stream_len: 1, - reward_bits: 1, - agent_actions: 2, - min_reward: 0, - max_reward: 1, - reward_offset: 0, - discount_gamma: 0.99, - return_horizon: 2, - return_bins: 8, - augmentation_period: 2, - history_prune_keep_steps: None, - baseline_exploration: 0.01, - random_seed: Some(7), - rate_backend: None, - rate_backend_max_order: 20, - rwkv_model_path: None, - rosa_max_order: None, - zpaq_method: None, - } - } - - #[derive(Clone, Default)] - struct CountingPredictor { - update_calls: usize, - commit_update_calls: usize, - update_history_calls: usize, - commit_update_history_calls: usize, - revert_calls: usize, - pop_history_calls: usize, - } - - impl Predictor for CountingPredictor { - fn update(&mut self, _sym: bool) { - self.update_calls += 1; - } - - fn commit_update(&mut self, _sym: bool) { - self.commit_update_calls += 1; - } - - fn update_history(&mut self, _sym: bool) { - self.update_history_calls += 1; - } - - fn commit_update_history(&mut self, _sym: bool) { - self.commit_update_history_calls += 1; - } - - fn revert(&mut self) { - self.revert_calls += 1; - } - - fn pop_history(&mut self) { - self.pop_history_calls += 1; - } - - fn predict_prob(&mut self, sym: bool) -> f64 { - if sym { 0.75 } else { 0.25 } - } - - fn model_name(&self) -> String { - "CountingPredictor".to_string() - } - - fn boxed_clone(&self) -> Box { - Box::new(self.clone()) - } - } - - #[derive(Clone, Default)] - struct SharedCallCounts { - update: usize, - commit_update: usize, - update_history: usize, - commit_update_history: usize, - } - - #[derive(Clone)] - struct SharedCountingPredictor { - counts: Arc>, - } - - impl SharedCountingPredictor { - fn new(counts: Arc>) -> Self { - Self { counts } - } - } - - impl Predictor for SharedCountingPredictor { - fn update(&mut self, _sym: bool) { - self.counts.lock().unwrap().update += 1; - } - - fn commit_update(&mut self, _sym: bool) { - self.counts.lock().unwrap().commit_update += 1; - } - - fn update_history(&mut self, _sym: bool) { - self.counts.lock().unwrap().update_history += 1; - } - - fn commit_update_history(&mut self, _sym: bool) { - self.counts.lock().unwrap().commit_update_history += 1; - } - - fn revert(&mut self) {} - - fn pop_history(&mut self) {} - - fn predict_prob(&mut self, sym: bool) -> f64 { - if sym { 0.75 } else { 0.25 } - } - - fn model_name(&self) -> String { - "SharedCountingPredictor".to_string() - } - - fn boxed_clone(&self) -> Box { - Box::new(self.clone()) - } - } - - #[derive(Clone, Default)] - struct ReturnLearningPredictor { - saw_training_one: bool, - } - - impl Predictor for ReturnLearningPredictor { - fn update(&mut self, sym: bool) { - if sym { - self.saw_training_one = true; - } - } - - fn commit_update(&mut self, sym: bool) { - if sym { - self.saw_training_one = true; - } - } - - fn update_history(&mut self, _sym: bool) {} - - fn commit_update_history(&mut self, _sym: bool) {} - - fn revert(&mut self) {} - - fn pop_history(&mut self) {} - - fn predict_prob(&mut self, sym: bool) -> f64 { - let p1 = if self.saw_training_one { 0.75 } else { 0.25 }; - if sym { p1 } else { 1.0 - p1 } - } - - fn model_name(&self) -> String { - "ReturnLearningPredictor".to_string() - } - - fn boxed_clone(&self) -> Box { - Box::new(self.clone()) - } - } - - #[test] - fn config_rejects_invalid_period() { - let mut cfg = basic_config(); - cfg.augmentation_period = 1; - cfg.return_horizon = 2; - let err = cfg - .validate() - .expect_err("N < H must be rejected to match \"A Model-Free Universal AI\""); - assert!(err.contains("augmentation_period")); - } - - #[test] - fn config_rejects_non_power_of_two_return_bins() { - let mut cfg = basic_config(); - cfg.return_bins = 3; - let err = cfg - .validate() - .expect_err("non-power-of-two return_bins should be rejected"); - assert!(err.contains("power of two")); - } - - #[test] - fn config_rejects_zpaq_algorithm_in_strict_mode() { - let mut cfg = basic_config(); - cfg.algorithm = "zpaq".to_string(); - let err = cfg - .validate() - .expect_err("strict AIQI must reject zpaq algorithm mode"); - assert!(err.contains("strict mode")); - } - - #[test] - fn config_rejects_zpaq_rate_backend_in_strict_mode() { - let mut cfg = basic_config(); - cfg.rate_backend = Some(RateBackend::Zpaq { - method: "1".to_string(), - }); - let err = cfg - .validate() - .expect_err("strict AIQI must reject zpaq rate backend"); - assert!(err.contains("strict frozen conditioning")); - } - - #[test] - fn config_rejects_nonpaper_gamma_or_tau() { - let mut cfg = basic_config(); - cfg.discount_gamma = 1.0; - let err = cfg - .validate() - .expect_err("gamma=1 must be rejected for strict paper AIQI"); - assert!(err.contains("discount_gamma")); - - cfg = basic_config(); - cfg.baseline_exploration = 0.0; - let err = cfg - .validate() - .expect_err("tau=0 must be rejected for strict paper AIQI"); - assert!(err.contains("baseline_exploration")); - } - - #[test] - fn aiqi_estimates_action_values_after_observations() { - let mut agent = AiqiAgent::new(basic_config()).expect("valid aiqi config"); - for _ in 0..8 { - agent - .observe_transition(1, &[1], 1) - .expect("transition should be accepted"); - } - - let action = agent.get_planned_action(); - assert!(action <= 1); - } - - #[test] - fn fac_ctw_predictor_uses_return_bit_width() { - let mut cfg = basic_config(); - cfg.algorithm = "fac-ctw".to_string(); - cfg.return_bins = 8; // return_bits=3 - - let agent = AiqiAgent::new(cfg).expect("valid aiqi config"); - let name = agent.phases[0].predictor.model_name(); - assert!( - name.contains("k=3"), - "FAC-CTW should factorize over return bits only, model_name={name}" - ); - } - - #[test] - fn ac_ctw_path_uses_single_tree_predictor() { - let mut cfg = basic_config(); - cfg.algorithm = "ac-ctw".to_string(); - - let agent = AiqiAgent::new(cfg).expect("valid aiqi config"); - let name = agent.phases[0].predictor.model_name(); - assert!( - name.starts_with("AC-CTW"), - "ac-ctw should map to the single-tree CTW predictor, model_name={name}" - ); - } - - #[test] - fn ctw_alias_matches_ac_ctw_predictor() { - let mut cfg = basic_config(); - cfg.algorithm = "ctw".to_string(); - - let agent = AiqiAgent::new(cfg).expect("valid aiqi config"); - let name = agent.phases[0].predictor.model_name(); - assert!( - name.starts_with("AC-CTW"), - "ctw alias should map to paper AIQI-CTW predictor, model_name={name}" - ); - } - - #[test] - fn distribution_rollout_uses_update_and_revert_when_requested() { - let mut predictor = CountingPredictor::default(); - let probs = AiqiAgent::predict_return_distribution(4, 2, &mut predictor, true); - - assert_eq!(probs.len(), 4); - assert_eq!(predictor.update_calls, 8); - assert_eq!(predictor.revert_calls, 8); - assert_eq!(predictor.update_history_calls, 0); - assert_eq!(predictor.pop_history_calls, 0); - } - - #[test] - fn distribution_rollout_uses_history_path_when_not_requested() { - let mut predictor = CountingPredictor::default(); - let probs = AiqiAgent::predict_return_distribution(4, 2, &mut predictor, false); - - assert_eq!(probs.len(), 4); - assert_eq!(predictor.update_calls, 0); - assert_eq!(predictor.revert_calls, 0); - assert_eq!(predictor.update_history_calls, 8); - assert_eq!(predictor.pop_history_calls, 8); - } - - #[test] - fn generic_distribution_rollout_trains_on_return_symbols() { - let predictor = ReturnLearningPredictor::default(); - let probs = AiqiAgent::predict_return_distribution_from_base_predictor(4, 2, &predictor); - - assert_eq!(probs.len(), 4); - assert!((probs.iter().sum::() - 1.0).abs() < 1e-12); - assert!( - probs[3] > probs[1], - "training on the first return bit should make bin 11 likelier than 01; got {:?}", - probs - ); - assert!( - (probs[0] - 0.5625).abs() < 1e-12, - "expected exact normalized mass for 00, got {:?}", - probs - ); - } - - #[test] - fn ac_ctw_rollout_uses_training_updates() { - let mut cfg = basic_config(); - cfg.algorithm = "ac-ctw".to_string(); - - let agent = AiqiAgent::new(cfg).expect("valid aiqi config"); - assert!( - agent.distribution_uses_training_updates, - "ac-ctw should use update/revert during return distribution rollout" - ); - } - - #[test] - fn return_bin_for_gamma_less_than_one_matches_paper_h_step_return() { - let mut cfg = basic_config(); - cfg.discount_gamma = 0.5; - cfg.return_bins = 8; - - let mut agent = AiqiAgent::new(cfg).expect("valid aiqi config"); - agent - .observe_transition(0, &[0], 1) - .expect("first transition stored"); - agent - .observe_transition(0, &[0], 0) - .expect("second transition should produce first return"); - - let bin = agent.return_bins_by_step[0].expect("first return should be available"); - // Paper target: R_{t,H} = (1-gamma) * sum_{k=0}^{H-1} gamma^k r_{t+k}. - // For rewards [1, 0], gamma=0.5, H=2 this equals 0.5. - // With M=8 bins this maps to floor(8 * 0.5) = 4. - assert_eq!(bin, 4); - } - - #[test] - fn optional_history_pruning_bounds_retained_state_without_losing_progress() { - let mut cfg = basic_config(); - cfg.return_horizon = 3; - cfg.augmentation_period = 4; - cfg.history_prune_keep_steps = Some(8); - - let mut agent = AiqiAgent::new(cfg).expect("valid aiqi config"); - for i in 0..256usize { - let action = (i % 2) as u64; - let obs = [(i % 2) as u64]; - let rew = (i % 2) as i64; - agent - .observe_transition(action, &obs, rew) - .expect("transition should be accepted"); - } - - // Global progress should be preserved even when retained history is bounded. - assert_eq!(agent.steps_observed(), 256); - assert!( - agent.history_base_step > 1, - "history should have been pruned" - ); - assert!( - agent.steps.len() < agent.steps_observed(), - "retained history should be smaller than total observed" - ); - - let action = agent.get_planned_action(); - assert!(action <= 1); - } - - #[test] - fn committed_phase_advancement_uses_commit_predictor_paths() { - let mut agent = AiqiAgent::new(basic_config()).expect("valid aiqi config"); - let counts = Arc::new(Mutex::new(SharedCallCounts::default())); - agent.phases[1].predictor = Box::new(SharedCountingPredictor::new(counts.clone())); - agent.phases[1].last_augmented_step = 0; - agent.history_base_step = 1; - agent.total_steps_observed = 1; - agent.steps = vec![StepRecord { - action: 1, - observations: vec![1], - reward: 1, - }]; - agent.return_bins_by_step = vec![Some(3)]; - - agent - .advance_phase_model_to_step(1, 1) - .expect("phase advancement should succeed"); - - let snapshot = counts.lock().unwrap().clone(); - assert_eq!(snapshot.commit_update, 3); - assert_eq!(snapshot.commit_update_history, 3); - assert_eq!(snapshot.update, 0); - assert_eq!(snapshot.update_history, 0); - } - - #[test] - fn generic_planner_trains_on_returns_and_freezes_conditioning_tokens() { - let mut cfg = basic_config(); - cfg.rate_backend = Some(RateBackend::Match { - hash_bits: 16, - min_len: 2, - max_len: 16, - base_mix: 0.05, - confidence_scale: 1.0, - }); - - let mut agent = AiqiAgent::new(cfg).expect("valid aiqi config"); - let counts = Arc::new(Mutex::new(SharedCallCounts::default())); - agent.phases[1].predictor = Box::new(SharedCountingPredictor::new(counts.clone())); - agent.phases[1].last_augmented_step = 0; - agent.history_base_step = 1; - agent.total_steps_observed = 2; - agent.steps = vec![ - StepRecord { - action: 1, - observations: vec![1], - reward: 1, - }, - StepRecord { - action: 0, - observations: vec![0], - reward: 0, - }, - ]; - agent.return_bins_by_step = vec![Some(3), None]; - - let q_values = agent.estimate_q_values_generic(); - - assert_eq!(q_values.len(), agent.config.agent_actions); - let snapshot = counts.lock().unwrap().clone(); - assert_eq!(snapshot.update, 0); - assert_eq!(snapshot.update_history, 0); - assert!( - snapshot.commit_update > 0, - "generic planner should train on augmented return symbols" - ); - assert!( - snapshot.commit_update_history > 0, - "generic planner should keep action/percept conditioning frozen" - ); - } -} diff --git a/src/aixi/common.rs b/src/aixi/common.rs deleted file mode 100644 index 7d517525..00000000 --- a/src/aixi/common.rs +++ /dev/null @@ -1,321 +0,0 @@ -//! Common types and utilities for the AIXI implementation. - -/// Represents a single bit (0 or 1) in the agent's interaction history. -pub type Symbol = bool; - -/// A list of symbols, used to represent encoded observations, rewards, or actions. -pub type SymbolList = Vec; - -/// Represents an action that the agent can perform. -pub type Action = u64; - -/// Represents a reward received by the agent from the environment. -pub type Reward = i64; - -/// A generic value for a percept component (either an observation or a reward). -pub type PerceptVal = u64; - -/// Strategy for mapping an observation stream into a single percept key for tree search. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum ObservationKeyMode { - /// Use the full observation stream as the key (paper-accurate expectimax). - FullStream, - /// Use the first observation symbol as the key. - First, - /// Use the last observation symbol as the key. - Last, - /// Hash the entire observation stream into a single key. - StreamHash, -} - -/// Compute a percept key from an observation stream. -pub fn observation_key_from_stream( - mode: ObservationKeyMode, - observations: &[PerceptVal], - observation_bits: usize, -) -> PerceptVal { - match mode { - ObservationKeyMode::FullStream => { - debug_assert!( - false, - "observation_key_from_stream called with FullStream; use observation_repr_from_stream" - ); - // Fallback to hash in release builds to avoid panics. - observation_key_from_stream( - ObservationKeyMode::StreamHash, - observations, - observation_bits, - ) - } - ObservationKeyMode::First => observations.first().copied().unwrap_or(0), - ObservationKeyMode::Last => observations.last().copied().unwrap_or(0), - ObservationKeyMode::StreamHash => { - let mask = if observation_bits >= 64 { - u64::MAX - } else if observation_bits == 0 { - 0 - } else { - (1u64 << observation_bits) - 1 - }; - let mut h = 0u64; - for &obs in observations { - let v = obs & mask; - h = h.rotate_left(7) ^ v; - } - h - } - } -} - -/// Compute the observation representation used for tree branching. -/// -/// - `FullStream` returns the full stream (paper-accurate expectimax). -/// - Other modes collapse to a single-key vector. -pub fn observation_repr_from_stream( - mode: ObservationKeyMode, - observations: &[PerceptVal], - observation_bits: usize, -) -> Vec { - match mode { - ObservationKeyMode::FullStream => observations.to_vec(), - _ => vec![observation_key_from_stream( - mode, - observations, - observation_bits, - )], - } -} - -/// A high-performance random number generator using the XorShift64* algorithm. -#[derive(Clone, Copy)] -pub struct RandomGenerator { - state: u64, -} - -impl RandomGenerator { - #[inline] - fn initial_seed() -> u64 { - #[cfg(feature = "backend-zpaq")] - { - if let Ok(bytes) = zpaq_rs::random_bytes(8) { - let mut seed_arr = [0u8; 8]; - seed_arr.copy_from_slice(&bytes); - return u64::from_le_bytes(seed_arr); - } - } - - #[cfg(target_arch = "wasm32")] - { - // `SystemTime::now()` is unavailable on `wasm32-unknown-unknown` without WASI. - return 0xCAFEBABEDEADBEEF ^ 0x9E3779B97F4A7C15; - } - - #[cfg(not(target_arch = "wasm32"))] - #[allow(clippy::cast_possible_truncation)] - { - let nanos = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.as_nanos() as u64) - .unwrap_or(0xCAFEBABEDEADBEEF); - return nanos ^ 0x9E3779B97F4A7C15; - } - - #[allow(unreachable_code)] - 0xCAFEBABEDEADBEEF - } - - /// Creates a new `RandomGenerator` with a fresh seed. - pub fn new() -> Self { - let seed = Self::initial_seed(); - let state = if seed == 0 { 0xCAFEBABEDEADBEEF } else { seed }; - Self { state } - } - - /// Creates a new `RandomGenerator` from an explicit seed. - /// - /// A zero seed is remapped to a fixed non-zero constant to avoid the - /// xorshift zero-state trap. - pub fn from_seed(seed: u64) -> Self { - let state = if seed == 0 { 0xCAFEBABEDEADBEEF } else { seed }; - Self { state } - } - - /// Generates the next pseudo-random `u64`. - pub fn next_u64(&mut self) -> u64 { - // xorshift64* - let mut x = self.state; - x ^= x >> 12; - x ^= x << 25; - x ^= x >> 27; - self.state = x; - x.wrapping_mul(0x2545F4914F6CDD1D) - } - - /// Generates a pseudo-random `usize` in the range `[0, end)`. - pub fn gen_range(&mut self, end: usize) -> usize { - if end == 0 { - return 0; - } - (self.next_u64() % (end as u64)) as usize - } - - /// Generates a boolean value with probability `p` of being `true`. - pub fn gen_bool(&mut self, p: f64) -> bool { - self.gen_f64() < p - } - - /// Generates a pseudo-random `f64` in the range `[0, 1)`. - pub fn gen_f64(&mut self) -> f64 { - // 53 bits - let v = self.next_u64() >> 11; - (v as f64) * (1.0 / 9007199254740992.0) - } - - /// Forks the RNG state with a salt, returning an independent generator. - pub fn fork_with(&self, salt: u64) -> Self { - let mixed = Self::splitmix64(self.state ^ salt ^ 0x9E3779B97F4A7C15); - let state = if mixed == 0 { - 0xCAFEBABEDEADBEEF - } else { - mixed - }; - Self { state } - } - - fn splitmix64(mut x: u64) -> u64 { - x = x.wrapping_add(0x9E3779B97F4A7C15); - let mut z = x; - z = (z ^ (z >> 30)).wrapping_mul(0xBF58476D1CE4E5B9); - z = (z ^ (z >> 27)).wrapping_mul(0x94D049BB133111EB); - z ^ (z >> 31) - } -} - -impl Default for RandomGenerator { - fn default() -> Self { - Self::new() - } -} - -/// Encodes a numeric value into its bit representation and appends it to a `SymbolList`. -/// -/// Bits are appended in least-significant-bit first order. -pub fn encode(symlist: &mut SymbolList, value: u64, bits: usize) { - let mut v = value; - for _ in 0..bits { - symlist.push((v & 1) == 1); - v >>= 1; - } -} - -/// Encodes a signed reward value into its bit representation. -pub fn encode_reward(symlist: &mut SymbolList, value: i64, bits: usize) { - let mut v = value as u64; - for _ in 0..bits { - symlist.push((v & 1) == 1); - v >>= 1; - } -} - -/// Encodes a reward after applying an additive `offset`. -pub fn encode_reward_offset(symlist: &mut SymbolList, value: i64, bits: usize, offset: i64) { - let shifted = (value + offset) as u64; - encode(symlist, shifted, bits); -} - -/// Decodes a numeric value from its bit representation. -pub fn decode(symlist: &[Symbol], bits: usize) -> u64 { - if bits == 0 { - return 0; - } - assert!(bits <= symlist.len()); - let mut value = 0u64; - for i in 0..bits { - let sym = symlist[symlist.len() - 1 - i]; - value = (value << 1) + (if sym { 1 } else { 0 }); - } - value -} - -/// Decodes a signed reward value from its bit representation. -pub fn decode_reward(symlist: &[Symbol], bits: usize) -> i64 { - if bits == 0 { - return 0; - } - let v = decode(symlist, bits); - if bits < 64 && (v & (1 << (bits - 1))) != 0 { - // Sign bit set, perform two's complement sign extension - (v | (!0u64 << bits)) as i64 - } else { - v as i64 - } -} - -/// Decodes a reward encoded with [`encode_reward_offset`]. -pub fn decode_reward_offset(symlist: &[Symbol], bits: usize, offset: i64) -> i64 { - if bits == 0 { - return 0; - } - let v = decode(symlist, bits) as i64; - v - offset -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn observation_repr_full_stream_is_identity() { - let obs = vec![1u64, 2u64, 3u64]; - let repr = observation_repr_from_stream(ObservationKeyMode::FullStream, &obs, 8); - assert_eq!(repr, obs); - } - - #[test] - fn observation_key_first_last() { - let obs = vec![10u64, 20u64, 30u64]; - assert_eq!( - observation_key_from_stream(ObservationKeyMode::First, &obs, 8), - 10 - ); - assert_eq!( - observation_key_from_stream(ObservationKeyMode::Last, &obs, 8), - 30 - ); - - let empty: Vec = vec![]; - assert_eq!( - observation_key_from_stream(ObservationKeyMode::First, &empty, 8), - 0 - ); - assert_eq!( - observation_key_from_stream(ObservationKeyMode::Last, &empty, 8), - 0 - ); - } - - #[test] - fn observation_key_stream_hash_masks_and_mix() { - // observation_bits=3 => mask=0b111 - // obs[0]=9 -> 1; h=0.rotate_left(7)^1 = 1 - // obs[1]=2 -> 2; h=1.rotate_left(7)^2 = 128^2 = 130 - let obs = vec![9u64, 2u64]; - let h = observation_key_from_stream(ObservationKeyMode::StreamHash, &obs, 3); - assert_eq!(h, 130); - } - - #[test] - fn observation_key_stream_hash_observation_bits_zero_is_zero() { - let obs = vec![123u64, 456u64, 789u64]; - let h = observation_key_from_stream(ObservationKeyMode::StreamHash, &obs, 0); - assert_eq!(h, 0); - } - - #[test] - fn observation_key_stream_hash_observation_bits_ge_64_uses_full_u64() { - let obs = vec![u64::MAX, 0x0123_4567_89ab_cdef]; - let h1 = observation_key_from_stream(ObservationKeyMode::StreamHash, &obs, 64); - let h2 = observation_key_from_stream(ObservationKeyMode::StreamHash, &obs, 128); - assert_eq!(h1, h2); - } -} diff --git a/src/aixi/environment.rs b/src/aixi/environment.rs deleted file mode 100644 index b5cedae1..00000000 --- a/src/aixi/environment.rs +++ /dev/null @@ -1,838 +0,0 @@ -//! Standard benchmark environments for AIXI. -//! -//! This module provides a set of environments for testing and evaluating -//! AIXI agents. Each environment implements the `Environment` trait, -//! providing a consistent interface for interaction. - -use crate::aixi::common::{Action, PerceptVal, RandomGenerator, Reward}; - -/// Interface for an agent's environment. -/// -/// An environment consumes actions from the agent and produces percepts -/// (observations and rewards) in response. -pub trait Environment { - /// Executes an action in the environment and updates its internal state. - fn perform_action(&mut self, action: Action); - - /// Returns the current observation produced by the environment. - fn get_observation(&self) -> PerceptVal; - - /// Returns a stream of observation symbols produced by the last action. - /// - /// Default behavior is a single observation. - fn drain_observations(&mut self) -> Vec { - vec![self.get_observation()] - } - - /// Returns the current reward produced by the environment. - fn get_reward(&self) -> Reward; - - /// Returns true if the environment has reached a terminal state. - fn is_finished(&self) -> bool; - - /// Returns the number of bits used to encode observations in this environment. - fn get_observation_bits(&self) -> usize; - - /// Returns the number of bits used to encode rewards in this environment. - fn get_reward_bits(&self) -> usize; - - /// Returns the number of bits required to represent all possible actions. - fn get_action_bits(&self) -> usize; - - /// Reseed the environment RNG for deterministic, reproducible runs. - /// - /// Deterministic environments can ignore this. Stochastic environments - /// should reseed and reset any stochastic state so the initial percept - /// sequence is reproducible from `seed`. - fn set_random_seed(&mut self, _seed: u64) {} - - /// Returns the total number of valid actions available. - fn get_num_actions(&self) -> usize { - 1 << self.get_action_bits() - } - - /// Returns the maximum possible reward value in this environment. - fn max_reward(&self) -> Reward { - let bits = self.get_reward_bits(); - if bits == 0 { - return 0; - } - // Prevent overflow for bits >= 64 - if bits >= 64 { - i64::MAX - } else { - (1i64 << (bits - 1)) - 1 - } - } - - /// Returns the minimum possible reward value in this environment. - fn min_reward(&self) -> Reward { - let bits = self.get_reward_bits(); - if bits == 0 { - return 0; - } - // Prevent overflow for bits >= 64 - if bits >= 64 { - i64::MIN - } else { - -(1i64 << (bits - 1)) - } - } -} - -/// A simple biased coin flip environment. -/// -/// The agent predicts the outcome of a coin flip. Correct predictions -/// result in a reward of 1, otherwise 0. -pub struct CoinFlip { - /// Probability of the coin landing heads (1). - p: f64, - /// Current observation (coin face). - obs: PerceptVal, - /// Last reward received. - rew: Reward, - /// Internal RNG. - rng: RandomGenerator, -} - -impl CoinFlip { - /// Creates a new `CoinFlip` environment with bias `p`. - pub fn new(p: f64) -> Self { - Self::new_with_seed(p, None) - } - - /// Creates a new `CoinFlip` environment with optional deterministic seed. - pub fn new_with_seed(p: f64, seed: Option) -> Self { - let mut env = Self { - p, - obs: 0, - rew: 0, - rng: seed.map(RandomGenerator::from_seed).unwrap_or_default(), - }; - // Initial observation - env.gen_next(); - env - } - - fn gen_next(&mut self) { - self.obs = if self.rng.gen_bool(self.p) { 1 } else { 0 }; - } -} - -impl Environment for CoinFlip { - fn perform_action(&mut self, action: Action) { - self.gen_next(); - self.rew = if action == self.obs { 1 } else { 0 }; - } - - fn get_observation(&self) -> PerceptVal { - self.obs - } - fn get_reward(&self) -> Reward { - self.rew - } - fn is_finished(&self) -> bool { - false - } - - fn get_observation_bits(&self) -> usize { - 1 - } - fn get_reward_bits(&self) -> usize { - 1 - } - - fn min_reward(&self) -> Reward { - 0 - } - - fn max_reward(&self) -> Reward { - 1 - } - fn get_action_bits(&self) -> usize { - 1 - } - - fn set_random_seed(&mut self, seed: u64) { - self.rng = RandomGenerator::from_seed(seed); - self.rew = 0; - self.gen_next(); - } -} - -/// A synthetic environment for testing CTW performance. -/// -/// Generates a deterministic sequence designed to be perfectly -/// predictable by a sufficiently deep Context Tree. -pub struct CtwTest { - cycle: usize, - last_action: Action, - obs: PerceptVal, - rew: Reward, -} - -impl CtwTest { - /// Creates a new `CtwTest` environment. - pub fn new() -> Self { - Self { - cycle: 0, - last_action: 0, - obs: 0, - rew: 0, - } - } -} - -impl Default for CtwTest { - fn default() -> Self { - Self::new() - } -} - -impl Environment for CtwTest { - fn perform_action(&mut self, action: Action) { - if self.cycle == 0 { - self.obs = 0; - self.rew = if self.obs == action { 1 } else { 0 }; - } else { - self.obs = (self.last_action + 1) % 2; - self.rew = if self.obs == action { 1 } else { 0 }; - } - self.last_action = action; - self.cycle += 1; - } - - fn get_observation(&self) -> PerceptVal { - self.obs - } - fn get_reward(&self) -> Reward { - self.rew - } - fn is_finished(&self) -> bool { - false - } - - fn get_observation_bits(&self) -> usize { - 1 - } - fn get_reward_bits(&self) -> usize { - 1 - } - - fn min_reward(&self) -> Reward { - 0 - } - - fn max_reward(&self) -> Reward { - 1 - } - fn get_action_bits(&self) -> usize { - 1 - } -} - -/// A Rock-Paper-Scissors environment with a biased opponent. -/// -/// The opponent plays randomly unless it wins a round, in which case -/// it repeats its winning action. -pub struct BiasedRockPaperScissor { - obs: PerceptVal, - rew: Reward, - rng: RandomGenerator, -} - -impl BiasedRockPaperScissor { - /// Creates a new `BiasedRockPaperScissor` environment. - pub fn new() -> Self { - Self::new_with_seed(None) - } - - /// Creates a new `BiasedRockPaperScissor` environment with optional seed. - pub fn new_with_seed(seed: Option) -> Self { - Self { - // Match reference MC-AIXI/PyAIXI initial percept: non-rock. - obs: 1, - rew: 0, - rng: seed.map(RandomGenerator::from_seed).unwrap_or_default(), - } - } -} - -impl Default for BiasedRockPaperScissor { - fn default() -> Self { - Self::new() - } -} - -impl Environment for BiasedRockPaperScissor { - fn perform_action(&mut self, action: Action) { - // action 0: Rock, 1: Paper, 2: Scissors - // Match reference MC-AIXI/PyAIXI bias: repeat rock iff opponent won - // the previous round by playing rock. - let opponent_action = if self.obs == 0 && self.rew == -1 { - 0 - } else { - let r = self.rng.gen_f64(); - if r < 1.0 / 3.0 { - 0 - } else if r < 2.0 / 3.0 { - 1 - } else { - 2 - } - }; - - // Determine Outcome - if opponent_action == action { - self.rew = 0; // Draw - } else if (opponent_action == 0 && action == 1) - || (opponent_action == 1 && action == 2) - || (opponent_action == 2 && action == 0) - { - self.rew = 1; // Win - } else { - self.rew = -1; // Loss - } - self.obs = opponent_action as PerceptVal; - } - - fn get_observation(&self) -> PerceptVal { - self.obs - } - fn get_reward(&self) -> Reward { - self.rew - } - fn is_finished(&self) -> bool { - false - } - - fn get_observation_bits(&self) -> usize { - 2 - } - fn get_reward_bits(&self) -> usize { - 2 - } - - fn min_reward(&self) -> Reward { - -1 - } - - fn max_reward(&self) -> Reward { - 1 - } - fn get_action_bits(&self) -> usize { - 2 - } - fn get_num_actions(&self) -> usize { - 3 - } - - fn set_random_seed(&mut self, seed: u64) { - self.rng = RandomGenerator::from_seed(seed); - // Match reference initial condition after reseed. - self.obs = 1; - self.rew = 0; - } -} - -/// A more complex version of the classic Tiger problem. -/// -/// Includes states for sitting and standing, with different rewards -/// and transition probabilities. -pub struct ExtendedTiger { - state: usize, // 0: sitting, 1: standing - tiger_door: usize, - gold_door: usize, - obs: PerceptVal, - rew: Reward, - rng: RandomGenerator, -} - -impl ExtendedTiger { - /// Creates a new `ExtendedTiger` environment. - pub fn new() -> Self { - let mut rng = RandomGenerator::new(); - let gold_door = if rng.gen_bool(0.5) { 1 } else { 2 }; - let tiger_door = if gold_door == 1 { 2 } else { 3 }; - - Self { - state: 0, - gold_door, - tiger_door, - obs: 0, - rew: 0, - rng, - } - } - - fn reset_doors(&mut self) { - self.gold_door = if self.rng.gen_bool(0.5) { 1 } else { 2 }; - self.tiger_door = if self.gold_door == 1 { 2 } else { 3 }; - } -} - -impl Default for ExtendedTiger { - fn default() -> Self { - Self::new() - } -} - -impl Environment for ExtendedTiger { - fn perform_action(&mut self, action: Action) { - // Actions: 0: Stand, 1: Listen, 2: Open 1, 3: Open 2 - match action { - 0 => { - // Stand - if self.state == 1 { - self.rew = -1; - } else { - self.state = 1; - self.rew = -1; - if self.obs < 4 { - self.obs += 4; - } - } - } - 1 => { - // Listen - if self.state == 1 || self.obs != 0 { - self.rew = -1; - self.obs = 0; - } else { - self.obs = if self.rng.gen_bool(0.85) { - self.tiger_door as PerceptVal - } else { - self.gold_door as PerceptVal - }; - self.rew = -1; - } - } - 2 => { - // Open 1 - if self.state == 0 { - self.rew = -100; - } else { - self.rew = if self.gold_door == 1 { 30 } else { -100 }; - self.obs = 0; - self.state = 0; - self.reset_doors(); - } - } - 3 => { - // Open 2 - if self.state == 0 { - self.rew = -100; - } else { - self.rew = if self.gold_door == 2 { 30 } else { -100 }; - self.obs = 0; - self.state = 0; - self.reset_doors(); - } - } - _ => { - self.rew = -100; - } - } - } - - fn get_observation(&self) -> PerceptVal { - self.obs - } - fn get_reward(&self) -> Reward { - self.rew - } - fn is_finished(&self) -> bool { - false - } - - fn get_observation_bits(&self) -> usize { - 3 - } - fn get_reward_bits(&self) -> usize { - 8 - } - - fn min_reward(&self) -> Reward { - -100 - } - - fn max_reward(&self) -> Reward { - 30 - } - fn get_action_bits(&self) -> usize { - 2 - } - fn get_num_actions(&self) -> usize { - 4 - } - - fn set_random_seed(&mut self, seed: u64) { - self.rng = RandomGenerator::from_seed(seed); - self.state = 0; - self.obs = 0; - self.rew = 0; - self.reset_doors(); - } -} - -/// A standard Tic-Tac-Toe environment against a random opponent. -pub struct TicTacToe { - board: [i8; 9], // 0: empty, 1: agent, -1: opponent. - open_squares: Vec, - state: u64, - obs: PerceptVal, - rew: Reward, - rng: RandomGenerator, -} - -impl TicTacToe { - /// Creates a new `TicTacToe` environment. - pub fn new() -> Self { - Self { - board: [0; 9], - open_squares: (0..9).collect(), - state: 0, - obs: 0, - rew: 0, - rng: RandomGenerator::new(), - } - } - - fn reset_game(&mut self) { - self.board = [0; 9]; - self.open_squares = (0..9).collect(); - self.state = 0; - } - - fn check_win(&self, player: i8) -> bool { - let b = self.board; - let wins = [ - (0, 1, 2), - (3, 4, 5), - (6, 7, 8), // Rows - (0, 3, 6), - (1, 4, 7), - (2, 5, 8), // Cols - (0, 4, 8), - (2, 4, 6), // Diags - ]; - for &(x, y, z) in &wins { - if b[x] == player && b[y] == player && b[z] == player { - return true; - } - } - false - } -} - -impl Default for TicTacToe { - fn default() -> Self { - Self::new() - } -} - -impl Environment for TicTacToe { - fn perform_action(&mut self, action: Action) { - if action >= 9 { - self.rew = -3; - self.obs = self.state as PerceptVal; - return; - } - - if self.board[action as usize] != 0 { - // Illegal move - self.rew = -3; - } else { - // Agent move (1) - self.state += 1 << (2 * action); - self.board[action as usize] = 1; - - // Remove from open - if let Some(pos) = self.open_squares.iter().position(|&x| x == action as usize) { - self.open_squares.remove(pos); - } - - self.rew = 0; - - if self.check_win(1) { - // Agent won - self.reset_game(); - self.rew = 2; - } else if self.open_squares.is_empty() { - // Draw - self.reset_game(); - self.rew = 1; - } else { - // Opponent move (-1, mapped to 2 in base-4) - - // Shuffle open squares - let n = self.open_squares.len(); - if n > 0 { - let idx = self.rng.gen_range(n); - let opponent_move = self.open_squares[idx]; - - self.state += 2 << (2 * opponent_move); - self.board[opponent_move] = -1; - - self.open_squares.remove(idx); - - if self.check_win(-1) { - // Opponent won - self.reset_game(); - self.rew = -2; - } else if self.open_squares.is_empty() { - self.reset_game(); - self.rew = 1; - } - } - } - } - self.obs = self.state as PerceptVal; - } - - fn get_observation(&self) -> PerceptVal { - self.obs - } - fn get_reward(&self) -> Reward { - self.rew - } - fn is_finished(&self) -> bool { - false - } - - fn get_observation_bits(&self) -> usize { - 18 - } // 9 squares * 2 bits - fn get_reward_bits(&self) -> usize { - 3 - } - fn min_reward(&self) -> Reward { - -3 - } - fn max_reward(&self) -> Reward { - 2 - } - fn get_action_bits(&self) -> usize { - 4 - } - fn get_num_actions(&self) -> usize { - 9 - } - - fn set_random_seed(&mut self, seed: u64) { - self.rng = RandomGenerator::from_seed(seed); - self.reset_game(); - self.obs = 0; - self.rew = 0; - } -} - -/// A 2-player imperfect information game: Kuhn Poker. -/// -/// The agent plays against a Nash-optimized opponent in a simplified -/// 3-card poker game. -pub struct KuhnPoker { - opponent_card: usize, // 0:J, 1:Q, 2:K - agent_card: usize, - opponent_action: usize, // 0: bet, 1: pass - obs: PerceptVal, - rew: Reward, - rng: RandomGenerator, -} - -impl KuhnPoker { - /// Creates a new `KuhnPoker` environment. - pub fn new() -> Self { - Self::new_with_seed(None) - } - - /// Creates a new `KuhnPoker` environment with optional deterministic seed. - pub fn new_with_seed(seed: Option) -> Self { - let mut env = Self { - opponent_card: 0, - agent_card: 0, - opponent_action: 0, - obs: 0, - rew: 0, - rng: seed.map(RandomGenerator::from_seed).unwrap_or_default(), - }; - env.reset_game(); - env - } - - #[inline] - fn random_card(&mut self) -> usize { - self.rng.gen_range(3) - } - - fn reset_game(&mut self) { - // Card encoding matches the reference implementations: - // 0=Jack, 1=Queen, 2=King. - self.agent_card = self.random_card(); - self.opponent_card = self.agent_card; - while self.opponent_card == self.agent_card { - self.opponent_card = self.random_card(); - } - - const ACTION_BET: usize = 0; - const ACTION_PASS: usize = 1; - const BET_PROB_KING: f64 = 0.7; - const BET_PROB_JACK: f64 = BET_PROB_KING / 3.0; - - // Opponent first action (reference Nash policy). - self.opponent_action = if self.opponent_card == 0 { - if self.rng.gen_bool(BET_PROB_JACK) { - ACTION_BET - } else { - ACTION_PASS - } - } else if self.opponent_card == 1 { - ACTION_PASS - } else if self.rng.gen_bool(BET_PROB_KING) { - ACTION_BET - } else { - ACTION_PASS - }; - - // Observation encoding matches C++/PyAIXI: - // observation = agent_card + (opponent_pass ? 4 : 0) - let action_code = if self.opponent_action == ACTION_PASS { - 4 - } else { - 0 - }; - let card_code = self.agent_card; - self.obs = (action_code + card_code) as PerceptVal; - } -} - -impl Default for KuhnPoker { - fn default() -> Self { - Self::new() - } -} - -impl Environment for KuhnPoker { - fn perform_action(&mut self, action: Action) { - const ACTION_BET: usize = 0; - const ACTION_PASS: usize = 1; - - // Reference reward levels are encoded as {0,1,3,4}. We emit the - // offset-removed values {-2,-1,1,2} for direct comparability. - const R_BET_LOSS: Reward = -2; - const R_PASS_LOSS: Reward = -1; - const R_PASS_WIN: Reward = 1; - const R_BET_WIN: Reward = 2; - - const BET_PROB_KING: f64 = 0.7; - const BET_PROB_QUEEN: f64 = (1.0 + BET_PROB_KING) / 3.0; - - if action > 1 { - self.rew = R_BET_LOSS; - self.reset_game(); - return; - } - - // If the agent did not call an opponent bet, the agent loses. - if action as usize == ACTION_PASS && self.opponent_action == ACTION_BET { - self.rew = R_PASS_LOSS; - self.reset_game(); - return; - } - - // If opponent passed and agent bet, opponent may reconsider. - if action as usize == ACTION_BET && self.opponent_action == ACTION_PASS { - if self.opponent_card == 1 && self.rng.gen_bool(BET_PROB_QUEEN) { - self.opponent_action = ACTION_BET; - } else if self.opponent_card == 2 { - self.opponent_action = ACTION_BET; - } else { - self.rew = R_PASS_WIN; - self.reset_game(); - return; - } - } - - // Showdown. - let agent_wins = - self.opponent_card == 0 || (self.opponent_card == 1 && self.agent_card == 2); - if agent_wins { - self.rew = if self.opponent_action == ACTION_BET { - R_BET_WIN - } else { - R_PASS_WIN - }; - } else { - self.rew = if action as usize == ACTION_BET { - R_BET_LOSS - } else { - R_PASS_LOSS - }; - } - self.reset_game(); - } - - fn get_observation(&self) -> PerceptVal { - self.obs - } - fn get_reward(&self) -> Reward { - self.rew - } - fn is_finished(&self) -> bool { - false - } - - fn get_observation_bits(&self) -> usize { - 3 - } - fn get_reward_bits(&self) -> usize { - 3 - } - - fn min_reward(&self) -> Reward { - -2 - } - - fn max_reward(&self) -> Reward { - 2 - } - fn get_action_bits(&self) -> usize { - 1 - } // 0 or 1 - fn get_num_actions(&self) -> usize { - 2 - } - - fn set_random_seed(&mut self, seed: u64) { - self.rng = RandomGenerator::from_seed(seed); - self.rew = 0; - self.reset_game(); - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn tictactoe_illegal_move_preserves_state_and_penalizes() { - let mut env = TicTacToe::new(); - env.set_random_seed(7); - - env.perform_action(0); - let occupied_state = env.get_observation(); - assert_ne!(occupied_state, 0, "first move should change the board"); - - env.perform_action(0); - assert_eq!( - env.get_reward(), - -3, - "illegal move should incur the documented penalty" - ); - assert_eq!( - env.get_observation(), - occupied_state, - "illegal move should not mutate the board state", - ); - } -} diff --git a/src/aixi/mcts.rs b/src/aixi/mcts.rs deleted file mode 100644 index 5cd80c9b..00000000 --- a/src/aixi/mcts.rs +++ /dev/null @@ -1,909 +0,0 @@ -//! Monte Carlo Tree Search (MCTS) for AIXI. -//! -//! This module implements the planning component of MC-AIXI. It use an upper -//! confidence bounds applied to trees (UCT) approach to select actions -//! by simulating future interactions with a world model. - -use crate::aixi::common::{ - Action, ObservationKeyMode, PerceptVal, Reward, observation_repr_from_stream, -}; -use rayon::prelude::*; -use std::collections::HashMap; - -/// Hash key for a sampled percept outcome at a chance node. -/// -/// Both the observation representation and the immediate reward are required -/// to identify the correct continuation subtree for generic environments. -/// Some environments can emit the same observation alongside different -/// rewards, so observation-only keys would incorrectly merge distinct -/// successor states during search-tree reuse. -#[derive(Clone, Debug, Eq, PartialEq, Hash)] -struct PerceptOutcome { - /// Observation symbols used for chance-node branching. - observations: Box<[PerceptVal]>, - /// Immediate reward observed on the sampled edge. - reward: Reward, -} - -impl PerceptOutcome { - /// Creates a compact percept key from an observation stream and reward. - fn new(observations: Vec, reward: Reward) -> Self { - Self { - observations: observations.into_boxed_slice(), - reward, - } - } -} - -/// Interface for an agent that can be simulated during MCTS. -/// -/// This trait allows the MCTS algorithm to interact with an agent -/// (like `Agent` in `agent.rs`) to perform "imagined" actions and -/// receive "imagined" percepts during planning. -pub trait AgentSimulator: Send { - /// Returns the number of possible actions the agent can perform. - fn get_num_actions(&self) -> usize; - - /// Returns the bit-width used to encode observations. - fn get_num_observation_bits(&self) -> usize; - - /// Returns the number of observation symbols per action. - fn observation_stream_len(&self) -> usize { - 1 - } - - /// Returns the observation key mode for search-tree branching. - fn observation_key_mode(&self) -> ObservationKeyMode { - ObservationKeyMode::FullStream - } - - /// Returns the observation representation used for tree branching. - fn observation_repr_from_stream(&self, observations: &[PerceptVal]) -> Vec { - observation_repr_from_stream( - self.observation_key_mode(), - observations, - self.get_num_observation_bits(), - ) - } - - /// Returns the bit-width used to encode rewards. - fn get_num_reward_bits(&self) -> usize; - - /// Returns the planning horizon (depth of simulations). - fn horizon(&self) -> usize; - - /// Returns the maximum possible reward value. - fn max_reward(&self) -> Reward; - - /// Returns the minimum possible reward value. - fn min_reward(&self) -> Reward; - - /// Returns the reward offset used to ensure encoded rewards are non-negative. - /// - /// Paper-compatible encoding uses unsigned reward bits and shifts rewards by an offset. - fn reward_offset(&self) -> i64 { - 0 - } - - /// Returns the exploration-exploitation constant (often denoted as C). - fn get_explore_exploit_ratio(&self) -> f64 { - 1.0 - } - - /// Returns the discount factor for future rewards. - fn discount_gamma(&self) -> f64 { - 1.0 - } - - /// Updates the internal model state with a simulated action. - fn model_update_action(&mut self, action: Action); - - /// Generates a simulated percept and updates the model state. - fn gen_percept_and_update(&mut self, bits: usize) -> u64; - - /// Marks the start of a new simulation rollout. - fn begin_simulation(&mut self) {} - - /// Reverts the model state to a previous point in the simulation. - fn model_revert(&mut self, steps: usize); - - /// Generates a random value in `[0, end)`. - fn gen_range(&mut self, end: usize) -> usize; - - /// Generates a random `f64` in `[0, 1)`. - fn gen_f64(&mut self) -> f64; - - /// Creates a boxed clone of this simulator for parallel search. - fn boxed_clone(&self) -> Box { - self.boxed_clone_with_seed(0) - } - - /// Creates a boxed clone of this simulator, re-seeding any RNG state. - fn boxed_clone_with_seed(&self, seed: u64) -> Box; - - /// Normalizes a reward value to [0, 1] based on the agent's range and horizon. - /// - /// For discounted rewards, the cumulative range is `sum_{t=0}^{h-1} gamma^t * (max - min)`. - /// Similarly, the minimum cumulative reward is `sum_{t=0}^{h-1} gamma^t * min`. - fn norm_reward(&self, reward: f64) -> f64 { - let min = self.min_reward() as f64; - let max = self.max_reward() as f64; - let h = self.horizon() as f64; - let gamma = self.discount_gamma().clamp(0.0, 1.0); - - // Discounted sum factor: sum_{t=0}^{h-1} gamma^t = (1 - gamma^h) / (1 - gamma) for gamma != 1 - let discount_sum = if (gamma - 1.0).abs() < 1e-9 { - h - } else { - (1.0 - gamma.powi(h as i32)) / (1.0 - gamma) - }; - - let range = (max - min) * discount_sum; - let min_cumulative = min * discount_sum; - - if range.abs() < 1e-9 { - 0.5 - } else { - (reward - min_cumulative) / range - } - } - - /// Helper to generate a percept stream, update the model, and return a search key + reward. - fn gen_percepts_and_update(&mut self) -> (Vec, Reward) { - let obs_bits = self.get_num_observation_bits(); - let obs_len = self.observation_stream_len().max(1); - let mut observations = Vec::with_capacity(obs_len); - for _ in 0..obs_len { - observations.push(self.gen_percept_and_update(obs_bits)); - } - - let obs_key = self.observation_repr_from_stream(&observations); - let rew_bits = self.get_num_reward_bits(); - let rew_u = self.gen_percept_and_update(rew_bits); - let rew = (rew_u as i64) - self.reward_offset(); - (obs_key, rew) - } -} - -/// A node in the MCTS search tree. -/// -/// Nodes can be either OR-nodes (representing an agent choice) or -/// chance nodes (representing an environment response). -#[derive(Clone)] -pub struct SearchNode { - /// Number of times this node has been visited during search. - visits: u32, - /// The current mean reward estimated for this node. - mean: f64, - /// Whether this is a chance node (observation/reward) rather than an action node. - is_chance_node: bool, - /// Children indexed by action (action nodes only). - action_children: Vec>, - /// Children indexed by percept outcome (chance nodes only). - percept_children: HashMap, -} - -impl SearchNode { - /// Creates a new `SearchNode`. - pub fn new(is_chance_node: bool) -> Self { - Self { - visits: 0, - mean: 0.0, - is_chance_node, - action_children: Vec::new(), - percept_children: HashMap::new(), - } - } - - /// Selects the best action from this node based on accumulated mean rewards. - pub fn best_action(&self, agent: &mut dyn AgentSimulator) -> Action { - let mut best_actions = Vec::new(); - let mut best_mean = -f64::INFINITY; - - for (action, child) in self.action_children.iter().enumerate() { - let Some(child) = child.as_ref() else { - continue; - }; - let mean = child.mean; - if mean > best_mean { - best_mean = mean; - best_actions.clear(); - best_actions.push(action as u64); - } else if (mean - best_mean).abs() < 1e-9 { - best_actions.push(action as u64); - } - } - - if best_actions.is_empty() { - return 0; - } - - let idx = agent.gen_range(best_actions.len()); - best_actions[idx] as Action - } - - fn expectation(&self) -> f64 { - self.mean - } - - fn apply_delta(&mut self, base: &SearchNode, updated: &SearchNode) { - if self.is_chance_node != base.is_chance_node - || self.is_chance_node != updated.is_chance_node - { - return; - } - - let base_visits = base.visits as f64; - let updated_visits = updated.visits as f64; - if updated_visits < base_visits { - return; - } - - let delta_visits = updated.visits - base.visits; - if delta_visits > 0 { - let base_sum = base.mean * base_visits; - let updated_sum = updated.mean * updated_visits; - let delta_sum = updated_sum - base_sum; - let total_visits = self.visits + delta_visits; - let total_sum = self.mean * (self.visits as f64) + delta_sum; - self.visits = total_visits; - self.mean = if total_visits > 0 { - total_sum / (total_visits as f64) - } else { - 0.0 - }; - } - - if self.is_chance_node { - for (key, updated_child) in &updated.percept_children { - if let Some(base_child) = base.percept_children.get(key) { - if let Some(self_child) = self.percept_children.get_mut(key) { - self_child.apply_delta(base_child, updated_child); - } else { - let mut child = SearchNode::new(updated_child.is_chance_node); - child.apply_delta( - &SearchNode::new(updated_child.is_chance_node), - updated_child, - ); - self.percept_children.insert(key.clone(), child); - } - } else if let Some(self_child) = self.percept_children.get_mut(key) { - let empty = SearchNode::new(updated_child.is_chance_node); - self_child.apply_delta(&empty, updated_child); - } else { - let mut child = SearchNode::new(updated_child.is_chance_node); - child.apply_delta( - &SearchNode::new(updated_child.is_chance_node), - updated_child, - ); - self.percept_children.insert(key.clone(), child); - } - } - } else { - let max_len = base - .action_children - .len() - .max(updated.action_children.len()); - if self.action_children.len() < max_len { - self.action_children.resize_with(max_len, || None); - } - for idx in 0..max_len { - let base_child = base.action_children.get(idx).and_then(|c| c.as_ref()); - let updated_child = updated.action_children.get(idx).and_then(|c| c.as_ref()); - let Some(updated_child) = updated_child else { - continue; - }; - match (base_child, self.action_children.get_mut(idx)) { - (Some(base_child), Some(Some(self_child))) => { - self_child.apply_delta(base_child, updated_child); - } - (Some(base_child), Some(slot @ None)) => { - let mut child = SearchNode::new(updated_child.is_chance_node); - child.apply_delta(base_child, updated_child); - *slot = Some(child); - } - (None, Some(Some(self_child))) => { - let empty = SearchNode::new(updated_child.is_chance_node); - self_child.apply_delta(&empty, updated_child); - } - (None, Some(slot @ None)) => { - let mut child = SearchNode::new(updated_child.is_chance_node); - child.apply_delta( - &SearchNode::new(updated_child.is_chance_node), - updated_child, - ); - *slot = Some(child); - } - _ => {} - } - } - } - } - - /// Selects an action to explore, potentially creating a new child node. - fn select_action(&mut self, agent: &mut dyn AgentSimulator) -> (&mut SearchNode, Action) { - let num_actions = agent.get_num_actions(); - - if self.action_children.len() < num_actions { - self.action_children.resize_with(num_actions, || None); - } - - let mut unvisited = Vec::new(); - for a in 0..num_actions { - if self.action_children[a].is_none() { - unvisited.push(a as u64); - } - } - - let action; - if !unvisited.is_empty() { - let idx = agent.gen_range(unvisited.len()); - action = unvisited[idx]; - self.action_children[action as usize] = Some(SearchNode::new(true)); - } else { - // Match reference MC-AIXI UCB scaling: - // priority = E[return] + horizon*max_reward*sqrt(C*log(N)/n) - let c = agent.get_explore_exploit_ratio().max(0.0); - let explore_bias = (agent.horizon() as f64) * (agent.max_reward() as f64).max(0.0); - let mut best_val = -f64::INFINITY; - let mut best_action = None; - let mut num_maximal_actions = 0usize; - let log_visits = (self.visits as f64).ln().max(0.0); - for (a, child) in self.action_children.iter().enumerate() { - let Some(child) = child.as_ref() else { - continue; - }; - let nvisits = child.visits as f64; - let val = child.expectation() + explore_bias * ((c * log_visits) / nvisits).sqrt(); - debug_assert!( - val.is_finite(), - "UCB score must be finite for visited MC-AIXI action children" - ); - match val.total_cmp(&best_val) { - std::cmp::Ordering::Greater => { - best_val = val; - best_action = Some(a as u64); - num_maximal_actions = 1; - } - std::cmp::Ordering::Equal => { - num_maximal_actions += 1; - // Tie-break from "A Monte-Carlo AIXI Approximation": - // choose uniformly among maximal actions. - // Reservoir sampling keeps this O(1) in memory without a tie list. - if agent.gen_range(num_maximal_actions) == 0 { - best_action = Some(a as u64); - } - } - std::cmp::Ordering::Less => {} - } - } - action = best_action.expect("visited MC-AIXI node must have a maximal action"); - } - - agent.model_update_action(action as Action); - ( - self.action_children[action as usize] - .as_mut() - .expect("missing action child"), - action as Action, - ) - } - - /// Performs a single simulation (sample) from this node. - pub fn sample( - &mut self, - agent: &mut dyn AgentSimulator, - horizon: usize, - total_horizon: usize, - ) -> f64 { - if horizon == 0 { - agent.model_revert(total_horizon); - return 0.0; - } - - let reward; - if self.is_chance_node { - let (obs, rew) = agent.gen_percepts_and_update(); - let key = PerceptOutcome::new(obs, rew); - let child = self - .percept_children - .entry(key) - .or_insert_with(|| SearchNode::new(false)); - reward = (rew as f64) - + agent.discount_gamma() * child.sample(agent, horizon - 1, total_horizon); - } else if self.visits == 0 { - reward = Self::playout(agent, horizon, total_horizon); - } else { - let (child, _act) = self.select_action(agent); - reward = child.sample(agent, horizon, total_horizon); - } - - // Update mean logic: - self.mean = (reward + (self.visits as f64) * self.mean) / ((self.visits + 1) as f64); - self.visits += 1; - - reward - } - - /// Performs a randomized simulation until the horizon is reached. - fn playout(agent: &mut dyn AgentSimulator, horizon: usize, total_horizon: usize) -> f64 { - let mut total_rew = 0.0; - let num_actions = agent.get_num_actions(); - let gamma = agent.discount_gamma().clamp(0.0, 1.0); - let mut discount = 1.0; - - for _ in 0..horizon { - let act = agent.gen_range(num_actions); - agent.model_update_action(act as Action); - let (_key, rew) = agent.gen_percepts_and_update(); - total_rew += discount * (rew as f64); - discount *= gamma; - } - - agent.model_revert(total_horizon); - total_rew - } -} - -/// Manages the MCTS tree and provides the `search` entry point. -pub struct SearchTree { - root: Option, -} - -impl SearchTree { - /// Creates a new `SearchTree`. - pub fn new() -> Self { - Self { - root: Some(SearchNode::new(false)), - } - } - - /// Performs several MCTS simulations to find the best next action. - pub fn search( - &mut self, - agent: &mut dyn AgentSimulator, - prev_obs_stream: &[PerceptVal], - prev_rew: Reward, - prev_act: u64, - samples: usize, - ) -> Action { - self.prune_tree(agent, prev_obs_stream, prev_rew, prev_act); - - let root = self.root.as_mut().unwrap(); - let h = agent.horizon(); - let threads = rayon::current_num_threads().max(1); - if samples < 2 || threads < 2 { - for _ in 0..samples { - agent.begin_simulation(); - root.sample(agent, h, h); - } - return root.best_action(agent); - } - - let workers = threads.min(samples); - let base = samples / workers; - let extra = samples % workers; - let snapshot = root.clone(); - - let mut agents = Vec::with_capacity(workers); - for i in 0..workers { - let seed = agent.gen_f64().to_bits() ^ (i as u64); - agents.push(agent.boxed_clone_with_seed(seed)); - } - - let results: Vec = agents - .into_par_iter() - .enumerate() - .map(|(i, mut local_agent)| { - let mut local_root = snapshot.clone(); - let iterations = base + usize::from(i < extra); - for _ in 0..iterations { - local_agent.begin_simulation(); - local_root.sample(local_agent.as_mut(), h, h); - } - local_root - }) - .collect(); - - for local in &results { - root.apply_delta(&snapshot, local); - } - - root.best_action(agent) - } - - /// Prunes the tree, keeping only relevant subtrees based on the previous interaction. - fn prune_tree( - &mut self, - agent: &mut dyn AgentSimulator, - prev_obs_stream: &[PerceptVal], - prev_rew: Reward, - prev_act: u64, - ) { - if self.root.is_none() { - self.root = Some(SearchNode::new(false)); - return; - } - - let mut old_root = self.root.take().unwrap(); - - // Find chance child (prev_act) - let action_child_opt = if old_root.action_children.len() > prev_act as usize { - old_root.action_children[prev_act as usize].take() - } else { - None - }; - - if let Some(mut chance_child) = action_child_opt { - let obs_repr = agent.observation_repr_from_stream(prev_obs_stream); - let key = PerceptOutcome::new(obs_repr, prev_rew); - - if let Some(action_child) = chance_child.percept_children.remove(&key) { - self.root = Some(action_child); - } else { - self.root = Some(SearchNode::new(false)); - } - } else { - self.root = Some(SearchNode::new(false)); - } - } -} - -impl Default for SearchTree { - fn default() -> Self { - Self::new() - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::aixi::common::ObservationKeyMode; - use std::sync::{ - Arc, - atomic::{AtomicUsize, Ordering}, - }; - - #[derive(Clone)] - struct DummyAgent { - obs_bits: usize, - rew_bits: usize, - horizon: usize, - min_reward: Reward, - max_reward: Reward, - key_mode: ObservationKeyMode, - } - - impl DummyAgent { - fn new(obs_bits: usize, key_mode: ObservationKeyMode) -> Self { - Self { - obs_bits, - rew_bits: 8, - horizon: 5, - min_reward: -1, - max_reward: 1, - key_mode, - } - } - } - - impl AgentSimulator for DummyAgent { - fn get_num_actions(&self) -> usize { - 4 - } - - fn get_num_observation_bits(&self) -> usize { - self.obs_bits - } - - fn observation_key_mode(&self) -> ObservationKeyMode { - self.key_mode - } - - fn get_num_reward_bits(&self) -> usize { - self.rew_bits - } - - fn horizon(&self) -> usize { - self.horizon - } - - fn max_reward(&self) -> Reward { - self.max_reward - } - - fn min_reward(&self) -> Reward { - self.min_reward - } - - fn model_update_action(&mut self, _action: Action) {} - - fn gen_percept_and_update(&mut self, _bits: usize) -> u64 { - 0 - } - - fn model_revert(&mut self, _steps: usize) {} - - fn gen_range(&mut self, _end: usize) -> usize { - 0 - } - - fn gen_f64(&mut self) -> f64 { - 0.0 - } - - fn boxed_clone_with_seed(&self, _seed: u64) -> Box { - Box::new(self.clone()) - } - } - - fn build_tree_with_key( - agent: &DummyAgent, - prev_act: u64, - prev_obs_stream: &[PerceptVal], - prev_rew: Reward, - kept_mean: f64, - kept_visits: u32, - ) -> SearchTree { - let mut old_root = SearchNode::new(false); - old_root.action_children.resize(prev_act as usize + 1, None); - - let mut chance_child = SearchNode::new(true); - let mut kept = SearchNode::new(false); - kept.mean = kept_mean; - kept.visits = kept_visits; - - let obs_repr = agent.observation_repr_from_stream(prev_obs_stream); - let key = PerceptOutcome::new(obs_repr, prev_rew); - chance_child.percept_children.insert(key, kept); - - old_root.action_children[prev_act as usize] = Some(chance_child); - SearchTree { - root: Some(old_root), - } - } - - #[test] - fn prune_tree_keeps_matching_subtree() { - let prev_act = 2u64; - let prev_obs_stream = vec![9u64, 2u64, 7u64]; - let prev_rew: Reward = 3; - - let mut agent = DummyAgent::new(3, ObservationKeyMode::FullStream); - let mut tree = build_tree_with_key(&agent, prev_act, &prev_obs_stream, prev_rew, 123.0, 7); - - tree.prune_tree(&mut agent, &prev_obs_stream, prev_rew, prev_act); - - let root = tree.root.as_ref().expect("root should exist"); - assert!(!root.is_chance_node); - assert_eq!(root.mean, 123.0); - assert_eq!(root.visits, 7); - } - - #[test] - fn prune_tree_resets_when_action_missing() { - let prev_act = 10u64; - let prev_obs_stream = vec![1u64]; - let prev_rew: Reward = 0; - - let mut agent = DummyAgent::new(1, ObservationKeyMode::FullStream); - let mut tree = SearchTree::new(); - - tree.prune_tree(&mut agent, &prev_obs_stream, prev_rew, prev_act); - - let root = tree.root.as_ref().unwrap(); - assert!(!root.is_chance_node); - assert_eq!(root.visits, 0); - assert_eq!(root.mean, 0.0); - } - - #[test] - fn prune_tree_resets_when_percept_key_missing() { - let prev_act = 0u64; - let prev_obs_stream = vec![1u64, 2u64]; - let prev_rew: Reward = 1; - - let mut agent = DummyAgent::new(4, ObservationKeyMode::Last); - - // Build tree keyed on a different observation key so pruning misses it. - let mut tree = build_tree_with_key(&agent, prev_act, &[9u64], prev_rew, 9.0, 2); - - tree.prune_tree(&mut agent, &prev_obs_stream, prev_rew, prev_act); - - let root = tree.root.as_ref().unwrap(); - assert!(!root.is_chance_node); - assert_eq!(root.visits, 0); - assert_eq!(root.mean, 0.0); - } - - #[test] - fn prune_tree_resets_when_reward_mismatch_shares_observation_key() { - let prev_act = 1u64; - let prev_obs_stream = vec![4u64, 5u64]; - let kept_rew: Reward = -2; - let requested_rew: Reward = 2; - - let mut agent = DummyAgent::new(6, ObservationKeyMode::FullStream); - let mut tree = build_tree_with_key(&agent, prev_act, &prev_obs_stream, kept_rew, 77.0, 11); - - tree.prune_tree(&mut agent, &prev_obs_stream, requested_rew, prev_act); - - let root = tree.root.as_ref().unwrap(); - assert!(!root.is_chance_node); - assert_eq!(root.visits, 0); - assert_eq!(root.mean, 0.0); - } - - #[derive(Clone)] - struct BeginCountingAgent { - begins: Arc, - } - - impl AgentSimulator for BeginCountingAgent { - fn get_num_actions(&self) -> usize { - 2 - } - - fn get_num_observation_bits(&self) -> usize { - 1 - } - - fn get_num_reward_bits(&self) -> usize { - 1 - } - - fn horizon(&self) -> usize { - 1 - } - - fn max_reward(&self) -> Reward { - 1 - } - - fn min_reward(&self) -> Reward { - 0 - } - - fn model_update_action(&mut self, _action: Action) {} - - fn gen_percept_and_update(&mut self, _bits: usize) -> u64 { - 0 - } - - fn begin_simulation(&mut self) { - self.begins.fetch_add(1, Ordering::Relaxed); - } - - fn model_revert(&mut self, _steps: usize) {} - - fn gen_range(&mut self, _end: usize) -> usize { - 0 - } - - fn gen_f64(&mut self) -> f64 { - 0.0 - } - - fn boxed_clone_with_seed(&self, _seed: u64) -> Box { - Box::new(self.clone()) - } - } - - #[test] - fn search_calls_begin_simulation_for_each_rollout() { - let begins = Arc::new(AtomicUsize::new(0)); - let mut agent = BeginCountingAgent { - begins: begins.clone(), - }; - let mut tree = SearchTree::new(); - - let _ = tree.search(&mut agent, &[0], 0, 0, 5); - assert_eq!(begins.load(Ordering::Relaxed), 5); - } - - #[derive(Clone)] - struct TieBreakAgent { - next_range: Arc, - } - - impl AgentSimulator for TieBreakAgent { - fn get_num_actions(&self) -> usize { - 4 - } - - fn get_num_observation_bits(&self) -> usize { - 1 - } - - fn get_num_reward_bits(&self) -> usize { - 1 - } - - fn horizon(&self) -> usize { - 1 - } - - fn max_reward(&self) -> Reward { - 1 - } - - fn min_reward(&self) -> Reward { - 0 - } - - fn get_explore_exploit_ratio(&self) -> f64 { - 0.0 - } - - fn model_update_action(&mut self, _action: Action) {} - - fn gen_percept_and_update(&mut self, _bits: usize) -> u64 { - 0 - } - - fn model_revert(&mut self, _steps: usize) {} - - fn gen_range(&mut self, end: usize) -> usize { - self.next_range - .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |value| { - Some(value.saturating_sub(1)) - }) - .expect("range source should be initialized") - % end - } - - fn gen_f64(&mut self) -> f64 { - 0.0 - } - - fn boxed_clone_with_seed(&self, _seed: u64) -> Box { - Box::new(self.clone()) - } - } - - #[test] - fn select_action_uses_uniform_tie_break_for_maximal_ucb_actions() { - let mut node = SearchNode::new(false); - node.visits = 16; - node.action_children = vec![ - Some(SearchNode { - visits: 5, - mean: 0.1, - is_chance_node: true, - action_children: Vec::new(), - percept_children: HashMap::new(), - }), - Some(SearchNode { - visits: 5, - mean: 0.9, - is_chance_node: true, - action_children: Vec::new(), - percept_children: HashMap::new(), - }), - Some(SearchNode { - visits: 5, - mean: 0.2, - is_chance_node: true, - action_children: Vec::new(), - percept_children: HashMap::new(), - }), - Some(SearchNode { - visits: 5, - mean: 0.9, - is_chance_node: true, - action_children: Vec::new(), - percept_children: HashMap::new(), - }), - ]; - - let mut agent = TieBreakAgent { - next_range: Arc::new(AtomicUsize::new(0)), - }; - - let (_child, action) = node.select_action(&mut agent); - assert_eq!( - action, 3, - "exactly tied maximal UCB actions should be chosen uniformly; scripted RNG selected the later maximal action" - ); - } -} diff --git a/src/aixi/rate_backend.rs b/src/aixi/rate_backend.rs deleted file mode 100644 index 05164535..00000000 --- a/src/aixi/rate_backend.rs +++ /dev/null @@ -1,61 +0,0 @@ -use crate::{CalibratedSpec, MixtureSpec, RateBackend}; -use std::sync::Arc; - -pub(crate) fn adapt_rate_backend_for_bit_tokens(backend: RateBackend) -> RateBackend { - match backend { - RateBackend::Ctw { depth } => RateBackend::FacCtw { - base_depth: depth, - num_percept_bits: 1, - encoding_bits: 1, - }, - RateBackend::FacCtw { base_depth, .. } => RateBackend::FacCtw { - base_depth, - num_percept_bits: 1, - encoding_bits: 1, - }, - RateBackend::Mixture { spec } => { - let experts = spec - .experts - .iter() - .map(|expert| crate::MixtureExpertSpec { - name: expert.name.clone(), - log_prior: expert.log_prior, - max_order: expert.max_order, - backend: adapt_rate_backend_for_bit_tokens(expert.backend.clone()), - }) - .collect(); - - let mut adapted = MixtureSpec::new(spec.kind, experts) - .with_schedule(spec.schedule) - .with_alpha(spec.alpha); - if let Some(decay) = spec.decay { - adapted = adapted.with_decay(decay); - } - RateBackend::Mixture { - spec: Arc::new(adapted), - } - } - RateBackend::Calibrated { spec } => RateBackend::Calibrated { - spec: Arc::new(CalibratedSpec { - base: adapt_rate_backend_for_bit_tokens(spec.base.clone()), - context: spec.context, - bins: spec.bins, - learning_rate: spec.learning_rate, - bias_clip: spec.bias_clip, - }), - }, - other => other, - } -} - -pub(crate) fn rate_backend_contains_zpaq(backend: &RateBackend) -> bool { - match backend { - RateBackend::Zpaq { .. } => true, - RateBackend::Mixture { spec } => spec - .experts - .iter() - .any(|expert| rate_backend_contains_zpaq(&expert.backend)), - RateBackend::Calibrated { spec } => rate_backend_contains_zpaq(&spec.base), - _ => false, - } -} diff --git a/src/aixi/vm_nyx.rs b/src/aixi/vm_nyx.rs deleted file mode 100644 index 08527025..00000000 --- a/src/aixi/vm_nyx.rs +++ /dev/null @@ -1,2111 +0,0 @@ -//! High-performance VM-backed AIXI environment using nyx-lite (Firecracker). -//! -//! This module provides a VM environment implementation built on top of nyx-lite, -//! enabling high-frequency snapshot-based resets for fast experimentation (hardware and -//! guest behavior dependent). -//! -//! ## Architecture -//! -//! The environment uses Firecracker's KVM-based microVM with nyx-lite's incremental -//! snapshot and reset capabilities. Communication with the guest occurs via: -//! -//! 1. **Shared Memory**: Zero-copy data transfer between host and guest -//! 2. **Hypercalls**: Control plane communication (snapshot, done, etc.) -//! 3. **Serial PTY**: Optional console I/O for simpler protocols -//! -//! ## Design Principles -//! -//! - **Universal**: Not biased towards any specific use case (fuzzing, etc.) -//! - **High Performance**: Leverages incremental snapshots and dirty page tracking -//! - **Configurable**: Pluggable reward policies, action sources, observation modes -//! - **Information-Theoretic**: Built-in support for entropy-based metrics - -use crate::aixi::common::{Action, PerceptVal, RandomGenerator, Reward}; -use crate::aixi::environment::Environment; -#[cfg(feature = "backend-rwkv")] -use crate::coders::softmax_pdf_inplace; -#[cfg(feature = "backend-mamba")] -use crate::mambazip; -#[cfg(feature = "backend-mamba")] -use crate::mambazip::Compressor as MambaCompressor; -use crate::mixture::OnlineBytePredictor; -use crate::rosaplus::RosaPlus; -#[cfg(feature = "backend-rwkv")] -use crate::rwkvzip::Compressor; -use crate::zpaq_rate::ZpaqRateModel; -use crate::{ - RateBackend, cross_entropy_rate_backend, entropy_rate_backend, marginal_entropy_bytes, -}; -use serde_json::Value; -use std::borrow::Cow; -use std::fs::OpenOptions; -use std::io::Write; -use std::path::Path; -use std::sync::Arc; -use std::time::{Duration, Instant}; - -// Re-export nyx-lite types for external use -pub use nyx_lite::mem::SharedMemoryRegion; -pub use nyx_lite::snapshot::NyxSnapshot; -pub use nyx_lite::{ExitReason, NyxVM, SharedMemoryPolicy}; - -// ============================================================================ -// Encoding Types -// ============================================================================ - -/// Payload encoding for wire protocol. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum PayloadEncoding { - /// Treat payloads as UTF-8/text bytes. - Utf8, - /// Treat payloads as hexadecimal text. - Hex, -} - -impl PayloadEncoding { - /// Parse a payload encoding label. - /// - /// Accepted values are `utf8`, `text`, and `hex`. - #[allow(clippy::should_implement_trait)] - pub fn from_str(s: &str) -> Option { - Self::parse(s) - } - - /// Parse a payload encoding label. - /// - /// Accepted values are `utf8`, `text`, and `hex`. - pub fn parse(s: &str) -> Option { - match s { - "utf8" | "text" => Some(Self::Utf8), - "hex" => Some(Self::Hex), - _ => None, - } - } - - /// Decode a wire payload string into raw bytes using this encoding. - pub fn decode(self, s: &str) -> anyhow::Result> { - match self { - Self::Utf8 => Ok(s.as_bytes().to_vec()), - Self::Hex => hex_decode(s), - } - } - - /// Encode raw bytes for transport over the configured wire protocol. - pub fn encode(self, bytes: &[u8]) -> String { - match self { - Self::Utf8 => String::from_utf8_lossy(bytes).to_string(), - Self::Hex => hex_encode(bytes), - } - } -} - -impl std::str::FromStr for PayloadEncoding { - type Err = &'static str; - - fn from_str(s: &str) -> Result { - Self::parse(s).ok_or("unknown payload encoding") - } -} - -fn hex_decode(s: &str) -> anyhow::Result> { - let mut out = Vec::with_capacity(s.len() / 2); - let mut buf = 0u8; - let mut high = true; - for c in s.bytes() { - let v = match c { - b'0'..=b'9' => c - b'0', - b'a'..=b'f' => c - b'a' + 10, - b'A'..=b'F' => c - b'A' + 10, - b' ' | b'\n' | b'\r' | b'\t' => continue, - _ => return Err(anyhow::anyhow!("invalid hex byte: {}", c as char)), - }; - if high { - buf = v << 4; - high = false; - } else { - buf |= v; - out.push(buf); - high = true; - } - } - if !high { - return Err(anyhow::anyhow!("hex string has odd length")); - } - Ok(out) -} - -fn resolve_relative_path(base: &Path, path: &str) -> String { - let p = Path::new(path); - if p.is_absolute() { - path.to_string() - } else { - base.join(p).to_string_lossy().to_string() - } -} - -fn rewrite_firecracker_config_paths(config_path: &str, raw_json: &str) -> anyhow::Result { - let base_dir = Path::new(config_path) - .parent() - .unwrap_or_else(|| Path::new(".")); - let mut v: Value = serde_json::from_str(raw_json)?; - - if let Some(boot) = v.get_mut("boot-source") { - if let Some(path_val) = boot.get_mut("kernel_image_path") - && let Some(path_str) = path_val.as_str() - { - let resolved = resolve_relative_path(base_dir, path_str); - *path_val = Value::String(resolved); - } - if let Some(path_val) = boot.get_mut("initrd_path") - && let Some(path_str) = path_val.as_str() - { - let resolved = resolve_relative_path(base_dir, path_str); - *path_val = Value::String(resolved); - } - } - - if let Some(drives) = v.get_mut("drives").and_then(|d| d.as_array_mut()) { - for drive in drives { - if let Some(path_val) = drive.get_mut("path_on_host") - && let Some(path_str) = path_val.as_str() - { - let resolved = resolve_relative_path(base_dir, path_str); - *path_val = Value::String(resolved); - } - } - } - - Ok(serde_json::to_string(&v)?) -} - -fn hex_encode(bytes: &[u8]) -> String { - let mut s = String::with_capacity(bytes.len() * 2); - for b in bytes { - s.push(hex_digit(b >> 4)); - s.push(hex_digit(b & 0x0F)); - } - s -} - -fn hex_digit(v: u8) -> char { - match v { - 0..=9 => (b'0' + v) as char, - _ => (b'a' + (v - 10)) as char, - } -} - -// ============================================================================ -// Guest Communication Protocol -// ============================================================================ - -/// Hypercall identifiers (must match guest implementation). -/// These are exported for use by custom guest programs. -#[allow(dead_code)] -pub const HYPERCALL_EXECDONE: u64 = 0x656e6f6463657865; // "execdone" -/// Guest requested host-side snapshot operation. -#[allow(dead_code)] -pub const HYPERCALL_SNAPSHOT: u64 = 0x746f687370616e73; // "snapshot" -/// Guest announced nyx-lite protocol/version handshake. -#[allow(dead_code)] -pub const HYPERCALL_NYX_LITE: u64 = 0x6574696c2d78796e; // "nyx-lite" -/// Guest requested shared memory initialization/refresh. -#[allow(dead_code)] -pub const HYPERCALL_SHAREMEM: u64 = 0x6d656d6572616873; // "sharemem" -/// Guest emitted a debug-print hypercall payload. -#[allow(dead_code)] -pub const HYPERCALL_DBGPRINT: u64 = 0x746e697270676264; // "dbgprint" - -const SHARED_ACTION_LEN_OFFSET: u64 = 0; -const SHARED_RESP_LEN_OFFSET: u64 = 8; -const SHARED_PAYLOAD_OFFSET: u64 = 16; - -/// Protocol configuration for structured communication. -#[derive(Clone, Debug)] -pub struct NyxProtocolConfig { - /// Prefix for action messages. - pub action_prefix: String, - /// Suffix for action messages. - pub action_suffix: String, - /// Prefix for observation responses. - pub obs_prefix: String, - /// Prefix for reward responses. - pub rew_prefix: String, - /// Prefix for done indicator. - pub done_prefix: String, - /// Prefix for data payloads. - pub data_prefix: String, - /// Wire encoding for payloads. - pub wire_encoding: PayloadEncoding, -} - -impl Default for NyxProtocolConfig { - fn default() -> Self { - Self { - action_prefix: "ACT ".to_string(), - action_suffix: "\n".to_string(), - obs_prefix: "OBS ".to_string(), - rew_prefix: "REW ".to_string(), - done_prefix: "DONE ".to_string(), - data_prefix: "DATA ".to_string(), - wire_encoding: PayloadEncoding::Hex, - } - } -} - -// ============================================================================ -// Action Configuration -// ============================================================================ - -/// A single action specification. -#[derive(Clone, Debug)] -pub struct NyxActionSpec { - /// Optional human-readable name. - pub name: Option, - /// Raw payload bytes to send. - pub payload: Vec, -} - -/// Fuzzing mutator types. -#[derive(Clone, Debug)] -pub enum FuzzMutator { - /// Flip one random bit. - FlipBit, - /// Flip one full byte. - FlipByte, - /// Insert a random byte at a random position. - InsertByte, - /// Delete one random byte. - DeleteByte, - /// Splice bytes from an existing seed input. - SpliceSeed, - /// Replace the working input with a seed input. - ResetSeed, - /// Apply a short sequence of random mutations. - Havoc, -} - -/// Fuzzing configuration for action generation. -#[derive(Clone, Debug)] -pub struct NyxFuzzConfig { - /// Corpus used for seed/reset/splice operations. - pub seeds: Vec>, - /// Mutator set available for action generation. - pub mutators: Vec, - /// Minimum generated action length. - pub min_len: usize, - /// Maximum generated action length. - pub max_len: usize, - /// Optional dictionary tokens for insertion/splicing. - pub dictionary: Vec>, - /// Deterministic RNG seed for mutation sampling. - pub rng_seed: u64, -} - -/// Source of actions for the environment. -#[derive(Clone, Debug)] -pub enum NyxActionSource { - /// Fixed set of action payloads. - Literal(Vec), - /// Mutation-based action generation. - Fuzz(NyxFuzzConfig), -} - -// ============================================================================ -// Observation Configuration -// ============================================================================ - -/// How observations are derived from guest output. -#[derive(Clone, Copy, Debug)] -pub enum NyxObservationPolicy { - /// Parse structured OBS/REW/DONE messages from guest. - FromGuest, - /// Hash raw output to derive observation. - OutputHash, - /// Use raw output bytes as observation stream. - RawOutput, - /// Use shared memory contents as observation. - SharedMemory, -} - -/// Stream normalization mode. -#[derive(Clone, Copy, Debug)] -pub enum NyxObservationStreamMode { - /// Pad short streams, truncate long ones. - PadTruncate, - /// Only pad short streams. - Pad, - /// Only truncate long streams. - Truncate, -} - -// ============================================================================ -// Reward Configuration -// ============================================================================ - -/// How rewards are computed. -#[derive(Clone)] -pub enum NyxRewardPolicy { - /// Parse reward from guest response. - FromGuest, - /// Pattern matching on output. - Pattern { - /// Substring/pattern tested against guest output. - pattern: String, - /// Reward returned when the pattern does not match. - base_reward: i64, - /// Additional reward added when the pattern matches. - bonus_reward: i64, - }, - /// Custom reward function (callback-based). - Custom(Arc Reward + Send + Sync>), -} - -/// Optional reward shaping (additive to base reward). -#[derive(Clone, Debug)] -pub enum NyxRewardShaping { - /// Entropy reduction vs baseline. - EntropyReduction { - /// Reference bytes used as baseline data distribution. - baseline_bytes: Vec, - /// Max order passed to entropy estimators. - max_order: i64, - /// Scaling factor applied to the shaping term. - scale: f64, - /// Optional additive bonus when guest crashes. - crash_bonus: Option, - /// Optional additive bonus when guest times out. - timeout_bonus: Option, - }, - /// Entropy of trace data (online learning). - TraceEntropy { - /// Max order passed to trace entropy estimation. - max_order: i64, - /// Scaling factor applied to the shaping term. - scale: f64, - /// If true, normalize by trace length. - normalize: bool, - }, -} - -impl std::fmt::Debug for NyxRewardPolicy { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::FromGuest => write!(f, "FromGuest"), - Self::Pattern { - pattern, - base_reward, - bonus_reward, - } => f - .debug_struct("Pattern") - .field("pattern", pattern) - .field("base_reward", base_reward) - .field("bonus_reward", bonus_reward) - .finish(), - Self::Custom(_) => write!(f, "Custom()"), - } - } -} - -// ============================================================================ -// Action Filtering -// ============================================================================ - -/// Information-theoretic action filtering. -#[derive(Clone, Debug)] -pub struct NyxActionFilter { - /// Minimum entropy threshold. - pub min_entropy: Option, - /// Maximum entropy threshold. - pub max_entropy: Option, - /// Minimum intrinsic dependence. - pub min_intrinsic_dependence: Option, - /// Minimum novelty (cross-entropy vs prior). - pub min_novelty: Option, - /// Prior corpus for novelty computation. - pub novelty_prior: Option>, - /// Max order for entropy estimation. - pub max_order: i64, - /// Reward to assign when action is rejected. - pub reject_reward: Option, -} - -// ============================================================================ -// Trace Configuration -// ============================================================================ - -/// Configuration for trace collection and analysis. -#[derive(Clone, Debug)] -pub struct NyxTraceConfig { - /// Shared memory region name for trace data. - pub shared_region_name: Option, - /// Maximum bytes to collect per step. - pub max_bytes: usize, - /// Reset trace model on episode boundary. - pub reset_on_episode: bool, -} - -// ============================================================================ -// Main Configuration -// ============================================================================ - -/// Complete configuration for the nyx-lite VM environment. -#[derive(Clone)] -pub struct NyxVmConfig { - /// Path to Firecracker JSON config. - pub firecracker_config: String, - /// Instance ID for the VM. - pub instance_id: String, - - // Shared memory configuration - /// Name of the shared memory region for communication. - pub shared_region_name: String, - /// Size of the shared memory region. - pub shared_region_size: usize, - /// Shared memory policy (snapshot vs preserve). - pub shared_memory_policy: SharedMemoryPolicy, - - // Timing configuration - /// Timeout for each step. - pub step_timeout: Duration, - /// Timeout for initial boot. - pub boot_timeout: Duration, - - // Episode configuration - /// Number of steps per episode. - pub episode_steps: usize, - /// Cost subtracted from reward each step. - pub step_cost: i64, - - // Observation configuration - /// Observation derivation policy. - pub observation_policy: NyxObservationPolicy, - /// Bits per observation symbol. - pub observation_bits: usize, - /// Number of observation symbols per action. - pub observation_stream_len: usize, - /// Stream normalization mode. - pub observation_stream_mode: NyxObservationStreamMode, - /// Padding byte for short streams. - pub observation_pad_byte: u8, - - // Reward configuration - /// Bits for reward encoding. - pub reward_bits: usize, - /// Reward computation policy. - pub reward_policy: NyxRewardPolicy, - /// Optional reward shaping (additive; non-canonical). - pub reward_shaping: Option, - - // Action configuration - /// Source of actions. - pub action_source: NyxActionSource, - /// Optional action filter. - pub action_filter: Option, - - // Protocol configuration - /// Wire protocol for structured communication. - pub protocol: NyxProtocolConfig, - - // Statistics backend - /// Backend for entropy estimation. - pub stats_backend: RateBackend, - - // Trace configuration - /// Optional trace collection. - pub trace: Option, - - // Debug mode - /// Enable verbose VM/protocol diagnostics. - pub debug_mode: bool, - - // Crash logging - /// Path to log crashes/interesting behaviors (JSONL format). - pub crash_log: Option, -} - -impl Default for NyxVmConfig { - fn default() -> Self { - Self { - firecracker_config: String::new(), - instance_id: "aixi-nyx".to_string(), - shared_region_name: "shared".to_string(), - shared_region_size: 4096, - shared_memory_policy: SharedMemoryPolicy::Snapshot, - step_timeout: Duration::from_millis(100), - boot_timeout: Duration::from_secs(30), - episode_steps: 100, - step_cost: 0, - observation_policy: NyxObservationPolicy::SharedMemory, - observation_bits: 8, - observation_stream_len: 64, - observation_stream_mode: NyxObservationStreamMode::PadTruncate, - observation_pad_byte: 0, - reward_bits: 8, - reward_policy: NyxRewardPolicy::FromGuest, - reward_shaping: None, - action_source: NyxActionSource::Literal(vec![]), - action_filter: None, - protocol: NyxProtocolConfig::default(), - stats_backend: RateBackend::default(), - trace: None, - debug_mode: false, - crash_log: None, - } - } -} - -// ============================================================================ -// Step Result -// ============================================================================ - -/// Result of a single environment step. -#[derive(Clone, Debug)] -pub struct NyxStepResult { - /// Exit reason from the VM. - pub exit_reason: NyxExitKind, - /// Raw output data from guest. - pub output: Vec, - /// Parsed observation (if any). - pub parsed_obs: Option, - /// Parsed reward (if any). - pub parsed_rew: Option, - /// Done flag. - pub done: bool, - /// Trace data (if collected). - pub trace_data: Vec, - /// Shared memory contents snapshot. - pub shared_memory: Vec, -} - -/// Simplified exit reason categories. -#[derive(Clone, Debug)] -pub enum NyxExitKind { - /// Guest terminated normally with an application-defined code. - ExecDone(u64), - /// Step timed out before a terminal signal/response. - Timeout, - /// VM reported a shutdown event. - Shutdown, - /// Raw hypercall event with integer arguments. - Hypercall { - /// Hypercall identifier/magic value. - code: u64, - /// Hypercall argument 1. - arg1: u64, - /// Hypercall argument 2. - arg2: u64, - /// Hypercall argument 3. - arg3: u64, - /// Hypercall argument 4. - arg4: u64, - }, - /// Debug string emitted by guest/host bridge. - DebugPrint(String), - /// Breakpoint/trap-like stop event. - Breakpoint, - /// Uncategorized exit event represented as text. - Other(String), -} - -impl From for NyxExitKind { - fn from(reason: ExitReason) -> Self { - match reason { - ExitReason::ExecDone(code) => Self::ExecDone(code), - ExitReason::Timeout => Self::Timeout, - ExitReason::Shutdown => Self::Shutdown, - ExitReason::Hypercall(r8, r9, r10, r11, r12) => Self::Hypercall { - code: r8, - arg1: r9, - arg2: r10, - arg3: r11, - arg4: r12, - }, - ExitReason::DebugPrint(s) => Self::DebugPrint(s), - ExitReason::Breakpoint => Self::Breakpoint, - ExitReason::RequestSnapshot => Self::Other("RequestSnapshot".to_string()), - ExitReason::SharedMem(name, _, _) => Self::Other(format!("SharedMem({})", name)), - ExitReason::SingleStep => Self::Other("SingleStep".to_string()), - ExitReason::Interrupted => Self::Other("Interrupted".to_string()), - ExitReason::HWBreakpoint(n) => Self::Other(format!("HWBreakpoint({})", n)), - ExitReason::BadMemoryAccess(_) => Self::Other("BadMemoryAccess".to_string()), - } - } -} - -// ============================================================================ -// Trace Model -// ============================================================================ - -/// Predictive model for trace-based reward computation. -enum TraceModel { - Rosa { - model: RosaPlus, - max_order: i64, - }, - Ctw { - tree: crate::ctw::ContextTree, - }, - FacCtw { - tree: crate::ctw::FacContextTree, - bits_per_symbol: usize, - }, - #[cfg(feature = "backend-mamba")] - Mamba { - compressor: MambaCompressor, - primed: bool, - }, - Rwkv7 { - compressor: Compressor, - primed: bool, - }, - Zpaq { - model: ZpaqRateModel, - }, - Mixture { - backend: RateBackend, - model: crate::mixture::RateBackendPredictor, - }, -} - -impl TraceModel { - fn predictor_backed(backend: RateBackend) -> Self { - let mut model = - crate::mixture::RateBackendPredictor::from_backend(backend.clone(), -1, 2f64.powi(-24)); - model - .begin_stream(None) - .unwrap_or_else(|e| panic!("predictor-backed stream init failed: {e}")); - TraceModel::Mixture { backend, model } - } - - fn new(backend: &RateBackend, max_order: i64) -> Self { - match backend { - RateBackend::RosaPlus => { - let mut model = RosaPlus::new(max_order, false, 0, 42); - model.build_lm_full_bytes_no_finalize_endpos(); - TraceModel::Rosa { model, max_order } - } - #[cfg(feature = "backend-mamba")] - RateBackend::Mamba { model } => { - let compressor = MambaCompressor::new_from_model(model.clone()); - TraceModel::Mamba { - compressor, - primed: false, - } - } - #[cfg(feature = "backend-mamba")] - RateBackend::MambaMethod { method } => { - let compressor = MambaCompressor::new_from_method(method) - .unwrap_or_else(|e| panic!("invalid mamba method for vm trace model: {e}")); - TraceModel::Mamba { - compressor, - primed: false, - } - } - RateBackend::Rwkv7 { model } => { - let compressor = Compressor::new_from_model(model.clone()); - TraceModel::Rwkv7 { - compressor, - primed: false, - } - } - RateBackend::Rwkv7Method { method } => { - let compressor = Compressor::new_from_method(method) - .unwrap_or_else(|e| panic!("invalid rwkv7 method for vm trace model: {e}")); - TraceModel::Rwkv7 { - compressor, - primed: false, - } - } - RateBackend::Zpaq { method } => TraceModel::Zpaq { - model: ZpaqRateModel::new(method.clone(), 2f64.powi(-24)), - }, - RateBackend::Mixture { .. } - | RateBackend::Particle { .. } - | RateBackend::Match { .. } - | RateBackend::SparseMatch { .. } - | RateBackend::Ppmd { .. } - | RateBackend::Sequitur { .. } - | RateBackend::Calibrated { .. } => TraceModel::predictor_backed(backend.clone()), - RateBackend::Ctw { depth } => TraceModel::Ctw { - tree: crate::ctw::ContextTree::new(*depth), - }, - RateBackend::FacCtw { - base_depth, - num_percept_bits: _, - encoding_bits, - } => { - let bits_per_symbol = (*encoding_bits).clamp(1, 8); - TraceModel::FacCtw { - tree: crate::ctw::FacContextTree::new(*base_depth, bits_per_symbol), - bits_per_symbol, - } - } - } - } - - fn reset(&mut self) { - match self { - TraceModel::Rosa { model, max_order } => { - let mut fresh = RosaPlus::new(*max_order, false, 0, 42); - fresh.build_lm_full_bytes_no_finalize_endpos(); - *model = fresh; - } - TraceModel::Ctw { tree } => tree.clear(), - TraceModel::FacCtw { tree, .. } => tree.clear(), - #[cfg(feature = "backend-mamba")] - TraceModel::Mamba { compressor, primed } => { - compressor.state.reset(); - *primed = false; - } - TraceModel::Rwkv7 { compressor, primed } => { - compressor.state.reset(); - *primed = false; - } - TraceModel::Zpaq { model } => { - model.reset(); - } - TraceModel::Mixture { backend, model } => { - *model = crate::mixture::RateBackendPredictor::from_backend( - backend.clone(), - -1, - 2f64.powi(-24), - ); - model - .begin_stream(None) - .unwrap_or_else(|e| panic!("mixture stream init failed: {e}")); - } - } - } - - /// Update the model with new data and return the surprise (bits). - fn update_and_score(&mut self, data: &[u8]) -> f64 { - if data.is_empty() { - return 0.0; - } - match self { - TraceModel::Rosa { model, .. } => { - let mut bits = 0.0; - for &b in data { - let p = model.prob_for_last(b as u32).max(1e-12); - bits -= p.log2(); - model.train_byte(b); - } - bits - } - TraceModel::Ctw { tree } => { - let log_before = tree.get_log_block_probability(); - for &b in data { - for i in (0..8).rev() { - tree.update(((b >> i) & 1) == 1); - } - } - let log_after = tree.get_log_block_probability(); - let log_delta = log_after - log_before; - -log_delta / std::f64::consts::LN_2 - } - TraceModel::FacCtw { - tree, - bits_per_symbol, - } => { - let log_before = tree.get_log_block_probability(); - for &b in data { - for i in 0..*bits_per_symbol { - tree.update(((b >> i) & 1) == 1, i); - } - } - let log_after = tree.get_log_block_probability(); - let log_delta = log_after - log_before; - -log_delta / std::f64::consts::LN_2 - } - #[cfg(feature = "backend-mamba")] - TraceModel::Mamba { compressor, primed } => { - if !*primed { - let bias = compressor.online_bias_snapshot(); - let logits = - compressor - .model - .forward(&mut compressor.scratch, 0, &mut compressor.state); - mambazip::Compressor::logits_to_pdf( - logits, - bias.as_deref(), - &mut compressor.pdf_buffer, - ); - *primed = true; - } - let mut bits = 0.0; - for &b in data { - let p = compressor.pdf_buffer[b as usize].max(1e-12); - bits -= p.log2(); - let bias = compressor.online_bias_snapshot(); - let logits = compressor.model.forward( - &mut compressor.scratch, - b as u32, - &mut compressor.state, - ); - mambazip::Compressor::logits_to_pdf( - logits, - bias.as_deref(), - &mut compressor.pdf_buffer, - ); - } - bits - } - TraceModel::Rwkv7 { compressor, primed } => { - if !*primed { - let vocab_size = compressor.vocab_size(); - let logits = - compressor - .model - .forward(&mut compressor.scratch, 0, &mut compressor.state); - softmax_pdf_inplace(logits, vocab_size, &mut compressor.pdf_buffer); - *primed = true; - } - let mut bits = 0.0; - let vocab_size = compressor.vocab_size(); - for &b in data { - let p = compressor.pdf_buffer[b as usize].max(1e-12); - bits -= p.log2(); - let logits = compressor.model.forward( - &mut compressor.scratch, - b as u32, - &mut compressor.state, - ); - softmax_pdf_inplace(logits, vocab_size, &mut compressor.pdf_buffer); - } - bits - } - TraceModel::Zpaq { model } => model.update_and_score(data), - TraceModel::Mixture { model, .. } => { - let mut bits = 0.0; - for &b in data { - let logp = model.log_prob(b); - bits -= logp / std::f64::consts::LN_2; - model.update(b); - } - bits - } - } - } -} - -// ============================================================================ -// Fuzz State -// ============================================================================ - -struct FuzzState { - current: Vec, - rng: RandomGenerator, -} - -// ============================================================================ -// NyxVmEnvironment -// ============================================================================ - -/// High-performance VM environment using nyx-lite. -pub struct NyxVmEnvironment { - /// Configuration. - config: NyxVmConfig, - /// The nyx-lite VM instance. - vm: NyxVM, - /// Base snapshot for episode resets. - base_snapshot: Option>, - /// Shared memory virtual address in guest. - shared_vaddr: Option, - /// CR3 used when shared memory was registered. - shared_cr3: Option, - /// Trace model for entropy-based rewards. - trace_model: Option, - /// Baseline entropy for entropy reduction rewards. - baseline_entropy: Option, - /// Effective reward shaping policy (additive). - reward_shaping: Option, - /// Fuzzing state. - fuzz_state: Option, - - // Current step state - /// Current observation. - obs: PerceptVal, - /// Current reward. - rew: Reward, - /// Current observation stream. - obs_stream: Vec, - /// Step within current episode. - step_in_episode: usize, - /// Whether the environment needs reset. - needs_reset: bool, - /// Whether the VM has been initialized. - initialized: bool, -} - -impl NyxVmEnvironment { - /// Creates a new NyxVmEnvironment with the given configuration. - pub fn new(config: NyxVmConfig) -> anyhow::Result { - // Validate configuration - if config.firecracker_config.is_empty() { - return Err(anyhow::anyhow!("firecracker_config path must be set")); - } - if config.episode_steps == 0 { - return Err(anyhow::anyhow!("episode_steps must be > 0")); - } - if matches!(config.observation_policy, NyxObservationPolicy::RawOutput) - && config.observation_stream_len == 0 - { - return Err(anyhow::anyhow!( - "observation_stream_len must be > 0 for RawOutput policy" - )); - } - - // Load Firecracker config and resolve relative paths - let fc_config_raw = std::fs::read_to_string(&config.firecracker_config) - .map_err(|e| anyhow::anyhow!("Failed to read firecracker config: {}", e))?; - let fc_config = - rewrite_firecracker_config_paths(&config.firecracker_config, &fc_config_raw) - .map_err(|e| anyhow::anyhow!("Failed to parse firecracker config: {}", e))?; - - // Create the VM - let vm = NyxVM::new(config.instance_id.clone(), &fc_config); - - // Initialize reward shaping - let reward_shaping = config.reward_shaping.clone(); - - if matches!(reward_shaping, Some(NyxRewardShaping::TraceEntropy { .. })) - && config.trace.is_none() - { - return Err(anyhow::anyhow!( - "vm_trace must be configured for vm_reward_shaping.mode=trace-entropy" - )); - } - - // Initialize trace model if needed - let trace_model = match &reward_shaping { - Some(NyxRewardShaping::TraceEntropy { max_order, .. }) => { - Some(TraceModel::new(&config.stats_backend, *max_order)) - } - _ => None, - }; - - // Compute baseline entropy if needed - let baseline_entropy = match &reward_shaping { - Some(NyxRewardShaping::EntropyReduction { - baseline_bytes, - max_order, - .. - }) => { - let h = if *max_order == 0 { - marginal_entropy_bytes(baseline_bytes) - } else { - entropy_rate_backend(baseline_bytes, *max_order, &config.stats_backend) - }; - Some(h) - } - _ => None, - }; - - // Initialize fuzz state if needed - let fuzz_state = match &config.action_source { - NyxActionSource::Fuzz(fuzz) => { - if fuzz.seeds.is_empty() { - return Err(anyhow::anyhow!("Fuzz mode requires at least one seed")); - } - if fuzz.mutators.is_empty() { - return Err(anyhow::anyhow!("Fuzz mode requires at least one mutator")); - } - let seed = fuzz.seeds[0].clone(); - Some(FuzzState { - current: seed, - rng: RandomGenerator::new().fork_with(fuzz.rng_seed), - }) - } - NyxActionSource::Literal(actions) => { - if actions.is_empty() { - return Err(anyhow::anyhow!("Literal mode requires at least one action")); - } - None - } - }; - - let mut env = Self { - config, - vm, - base_snapshot: None, - shared_vaddr: None, - shared_cr3: None, - trace_model, - baseline_entropy, - reward_shaping, - fuzz_state, - obs: 0, - rew: 0, - obs_stream: Vec::new(), - step_in_episode: 0, - needs_reset: true, - initialized: false, - }; - - // Boot and initialize - env.initialize()?; - - Ok(env) - } - - /// Initializes the VM by booting to the snapshot point. - fn initialize(&mut self) -> anyhow::Result<()> { - if self.initialized { - return Ok(()); - } - - if self.config.debug_mode { - eprintln!("[NyxVm] Booting VM..."); - } - - // Run until we get the shared memory registration - let start = Instant::now(); - loop { - if start.elapsed() > self.config.boot_timeout { - return Err(anyhow::anyhow!("Boot timeout waiting for shared memory")); - } - - let exit = self.vm.run(Duration::from_secs(1)); - match exit { - ExitReason::SharedMem(name, vaddr, size) => { - if self.config.debug_mode { - eprintln!( - "[NyxVm] Shared memory registered: {} @ {:#x} ({} bytes)", - name, vaddr, size - ); - } - if name.trim_end_matches('\0') == self.config.shared_region_name { - self.shared_vaddr = Some(vaddr); - self.shared_cr3 = Some(self.vm.sregs().cr3); - // Register the shared region with the configured policy - let _ = self.vm.register_shared_region_current( - vaddr, - size, - self.config.shared_memory_policy, - ); - break; - } - } - ExitReason::DebugPrint(msg) => { - if self.config.debug_mode { - eprintln!("[NyxVm] Guest: {}", msg); - } - } - ExitReason::Shutdown => { - return Err(anyhow::anyhow!("VM shut down during boot")); - } - _ => { - if self.config.debug_mode { - eprintln!("[NyxVm] Boot exit: {:?}", exit); - } - // Continue waiting - } - } - } - - // Continue running until snapshot request - loop { - if start.elapsed() > self.config.boot_timeout { - return Err(anyhow::anyhow!("Boot timeout waiting for snapshot request")); - } - - let exit = self.vm.run(Duration::from_secs(1)); - match exit { - ExitReason::RequestSnapshot => { - if self.config.debug_mode { - eprintln!("[NyxVm] Taking base snapshot..."); - } - self.base_snapshot = Some(self.vm.take_base_snapshot()); - break; - } - ExitReason::DebugPrint(msg) => { - if self.config.debug_mode { - eprintln!("[NyxVm] Guest: {}", msg); - } - } - ExitReason::Shutdown => { - return Err(anyhow::anyhow!("VM shut down before snapshot")); - } - _ => { - if self.config.debug_mode { - eprintln!("[NyxVm] Snapshot wait exit: {:?}", exit); - } - // Continue waiting - } - } - } - - if self.config.debug_mode { - eprintln!("[NyxVm] Initialization complete"); - } - - self.initialized = true; - self.needs_reset = false; - Ok(()) - } - - /// Resets to the base snapshot. - pub fn reset(&mut self) -> anyhow::Result<()> { - let snapshot = self - .base_snapshot - .as_ref() - .ok_or_else(|| anyhow::anyhow!("No base snapshot available"))? - .clone(); - - self.vm.apply_snapshot(&snapshot); - - // Reset trace model if configured - if let Some(trace_cfg) = &self.config.trace - && trace_cfg.reset_on_episode - && let Some(model) = &mut self.trace_model - { - model.reset(); - } - - self.step_in_episode = 0; - self.needs_reset = false; - - Ok(()) - } - - /// Writes action data to shared memory. - fn write_action_to_shared_memory(&mut self, payload: &[u8]) -> anyhow::Result<()> { - let vaddr = self - .shared_vaddr - .ok_or_else(|| anyhow::anyhow!("Shared memory not initialized"))?; - let cr3 = self - .shared_cr3 - .ok_or_else(|| anyhow::anyhow!("Shared memory CR3 not initialized"))?; - let process = self.vm.process_memory(cr3); - - // Ensure guest has cleared the previous message length to avoid races. - let wait_start = Instant::now(); - loop { - let cur_len = process - .read_u64(vaddr + SHARED_ACTION_LEN_OFFSET) - .unwrap_or(0); - if cur_len == 0 { - break; - } - if wait_start.elapsed() > self.config.step_timeout { - return Err(anyhow::anyhow!("shared buffer busy (len={cur_len})")); - } - std::thread::yield_now(); - } - - // Write length as first 8 bytes (u64 LE) - let len = payload.len() as u64; - process - .write_u64(vaddr + SHARED_ACTION_LEN_OFFSET, len) - .map_err(|e| anyhow::anyhow!("write len failed: {e}"))?; - let _ = process.write_u64(vaddr + SHARED_RESP_LEN_OFFSET, 0); - - // Write payload starting at offset 8 - let max_len = self - .config - .shared_region_size - .saturating_sub(SHARED_PAYLOAD_OFFSET as usize); - let write_len = payload.len().min(max_len); - if write_len > 0 { - let _ = process - .write_bytes(vaddr + SHARED_PAYLOAD_OFFSET, &payload[..write_len]) - .map_err(|e| anyhow::anyhow!("write payload failed: {e}"))?; - } - - if self.config.debug_mode { - let verify = process - .read_u64(vaddr + SHARED_ACTION_LEN_OFFSET) - .unwrap_or(0) as usize; - eprintln!( - "[NyxVm] Wrote action len={}, verified len={}", - write_len, verify - ); - } - - Ok(()) - } - - /// Reads response from shared memory. - fn read_shared_memory(&self) -> Vec { - let Some(vaddr) = self.shared_vaddr else { - return Vec::new(); - }; - let Some(cr3) = self.shared_cr3 else { - return Vec::new(); - }; - let process = self.vm.process_memory(cr3); - - // Read length from first 8 bytes - let len = process - .read_u64(vaddr + SHARED_RESP_LEN_OFFSET) - .unwrap_or(0) as usize; - let max_len = self - .config - .shared_region_size - .saturating_sub(SHARED_PAYLOAD_OFFSET as usize); - let read_len = len.min(max_len); - - if read_len == 0 { - return Vec::new(); - } - - let mut buf = vec![0u8; read_len]; - let _ = process.read_bytes(vaddr + SHARED_PAYLOAD_OFFSET, &mut buf); - buf - } - - fn clear_shared_length(&self) { - let (Some(vaddr), Some(cr3)) = (self.shared_vaddr, self.shared_cr3) else { - return; - }; - let process = self.vm.process_memory(cr3); - let _ = process.write_u64(vaddr + SHARED_ACTION_LEN_OFFSET, 0); - let _ = process.write_u64(vaddr + SHARED_RESP_LEN_OFFSET, 0); - } - - /// Runs a single step, returning detailed results. - pub fn run_step(&mut self, payload: &[u8]) -> anyhow::Result { - // Write action to shared memory - self.write_action_to_shared_memory(payload)?; - - // Run the VM until we get a meaningful exit - let start = Instant::now(); - let mut output = Vec::new(); - let mut trace_data = Vec::new(); - let mut parsed_obs = None; - let mut parsed_rew = None; - let mut done = false; - let exit_kind; - let collect_output = - matches!( - self.config.observation_policy, - NyxObservationPolicy::OutputHash | NyxObservationPolicy::RawOutput - ) || matches!(self.config.reward_policy, NyxRewardPolicy::Pattern { .. }) - || matches!( - self.reward_shaping, - Some(NyxRewardShaping::EntropyReduction { .. }) - ); - - loop { - let remaining = self - .config - .step_timeout - .checked_sub(start.elapsed()) - .unwrap_or(Duration::ZERO); - - if remaining.is_zero() { - exit_kind = NyxExitKind::Timeout; - break; - } - - let exit = self.vm.run(remaining); - match exit { - ExitReason::ExecDone(code) => { - exit_kind = NyxExitKind::ExecDone(code); - done = true; - break; - } - ExitReason::Timeout => { - if self.config.debug_mode { - eprintln!("[NyxVm] Step timeout"); - } - exit_kind = NyxExitKind::Timeout; - break; - } - ExitReason::Shutdown => { - if self.config.debug_mode { - eprintln!("[NyxVm] VM shutdown during step"); - } - exit_kind = NyxExitKind::Shutdown; - done = true; - break; - } - ExitReason::DebugPrint(msg) => { - if self.config.debug_mode { - eprintln!("[NyxVm] Guest: {}", msg); - } - // Accumulate debug output - if collect_output { - output.extend_from_slice(msg.as_bytes()); - } - // Continue running - } - ExitReason::Hypercall(r8, r9, r10, r11, r12) => { - exit_kind = NyxExitKind::Hypercall { - code: r8, - arg1: r9, - arg2: r10, - arg3: r11, - arg4: r12, - }; - // Attempt to parse structured response - if let Some(obs) = Self::try_parse_u64(r9) { - parsed_obs = Some(obs); - } - if let Some(rew) = Self::try_parse_i64(r10) { - parsed_rew = Some(rew); - } - break; - } - ExitReason::Breakpoint => { - if self.config.debug_mode { - eprintln!("[NyxVm] Breakpoint exit during step"); - } - exit_kind = NyxExitKind::Breakpoint; - break; - } - _ => { - // Continue for other exits - } - } - } - - // Read shared memory contents (only if needed) - let need_shared_memory = matches!( - self.config.observation_policy, - NyxObservationPolicy::SharedMemory - ) || matches!( - self.config.reward_policy, - NyxRewardPolicy::Pattern { .. } - ) || matches!( - self.reward_shaping, - Some(NyxRewardShaping::EntropyReduction { .. }) - ) || self.config.trace.is_some(); - let shared_memory = if need_shared_memory { - self.read_shared_memory() - } else { - Vec::new() - }; - - // Clear shared length to avoid host/guest races on the next step. - self.clear_shared_length(); - - // Collect trace data if configured - if let Some(trace_cfg) = &self.config.trace - && trace_cfg.shared_region_name.is_some() - { - // Read from trace shared memory region (implementation-specific) - // For now, use main shared memory as fallback - trace_data = shared_memory.clone(); - if trace_data.len() > trace_cfg.max_bytes { - trace_data.truncate(trace_cfg.max_bytes); - } - } - - Ok(NyxStepResult { - exit_reason: exit_kind, - output, - parsed_obs, - parsed_rew, - done, - trace_data, - shared_memory, - }) - } - - fn try_parse_u64(val: u64) -> Option { - // Hypercall args are already u64 - Some(val) - } - - fn try_parse_i64(val: u64) -> Option { - Some(val as i64) - } - - /// Gets the action payload for the given action index. - fn get_action_payload(&mut self, action: Action) -> anyhow::Result> { - match &self.config.action_source { - NyxActionSource::Literal(actions) => { - let idx = action as usize; - if idx >= actions.len() { - return Err(anyhow::anyhow!("Action index out of range")); - } - Ok(Cow::Borrowed(actions[idx].payload.as_slice())) - } - NyxActionSource::Fuzz(fuzz) => { - let state = self - .fuzz_state - .as_mut() - .ok_or_else(|| anyhow::anyhow!("Fuzz state missing"))?; - let idx = action as usize % fuzz.mutators.len(); - let mut input = state.current.clone(); - let mutator = &fuzz.mutators[idx]; - apply_mutator(mutator, &mut input, fuzz, &mut state.rng); - if input.len() < fuzz.min_len { - input.resize(fuzz.min_len, 0); - } - if input.len() > fuzz.max_len { - input.truncate(fuzz.max_len); - } - state.current = input.clone(); - Ok(Cow::Owned(input)) - } - } - } - - /// Applies action filtering, returning reject reward if filtered. - fn filter_action(&self, payload: &[u8]) -> Option { - let filter = self.config.action_filter.as_ref()?; - if payload.is_empty() { - return filter.reject_reward; - } - - let (entropy, intrinsic, novelty) = self.compute_filter_metrics(payload, filter); - - if let Some(min_entropy) = filter.min_entropy - && entropy < min_entropy - { - return filter.reject_reward; - } - if let Some(max_entropy) = filter.max_entropy - && entropy > max_entropy - { - return filter.reject_reward; - } - if let Some(min_intrinsic) = filter.min_intrinsic_dependence - && intrinsic < min_intrinsic - { - return filter.reject_reward; - } - if let Some(min_novelty) = filter.min_novelty - && filter.novelty_prior.is_some() - && novelty < min_novelty - { - return filter.reject_reward; - } - None - } - - fn wrap_action_payload(&self, payload: &[u8]) -> Vec { - let p = &self.config.protocol; - let mut wrapped = p.action_prefix.clone().into_bytes(); - wrapped.extend_from_slice(p.wire_encoding.encode(payload).as_bytes()); - wrapped.extend_from_slice(p.action_suffix.as_bytes()); - wrapped - } - - fn compute_filter_metrics(&self, payload: &[u8], filter: &NyxActionFilter) -> (f64, f64, f64) { - let h_marg = marginal_entropy_bytes(payload); - let h_rate = if filter.max_order == 0 { - h_marg - } else { - entropy_rate_backend(payload, filter.max_order, &self.config.stats_backend) - }; - - let intrinsic = if h_marg < 1e-9 { - 0.0 - } else { - ((h_marg - h_rate) / h_marg).clamp(0.0, 1.0) - }; - - let novelty = if let Some(ref prior) = filter.novelty_prior { - cross_entropy_rate_backend(payload, prior, filter.max_order, &self.config.stats_backend) - } else { - 0.0 - }; - - (h_rate, intrinsic, novelty) - } - - /// Computes reward from step result. - fn compute_reward(&mut self, result: &NyxStepResult) -> Reward { - let base_reward = match &self.config.reward_policy { - NyxRewardPolicy::FromGuest => result.parsed_rew.unwrap_or(0), - NyxRewardPolicy::Pattern { - pattern, - base_reward, - bonus_reward, - } => { - let text = String::from_utf8_lossy(&result.output); - let shared_text = String::from_utf8_lossy(&result.shared_memory); - if text.contains(pattern) || shared_text.contains(pattern) { - base_reward + bonus_reward - } else { - *base_reward - } - } - NyxRewardPolicy::Custom(f) => f(result), - }; - - let shaping_reward = if let Some(shaping) = self.reward_shaping.clone() { - self.compute_reward_shaping(&shaping, result) - } else { - 0 - }; - - let mut reward = base_reward.saturating_add(shaping_reward); - - reward = reward.saturating_sub(self.config.step_cost); - let min_reward = self.min_reward(); - let max_reward = self.max_reward(); - reward.clamp(min_reward, max_reward) - } - - fn compute_reward_shaping( - &mut self, - shaping: &NyxRewardShaping, - result: &NyxStepResult, - ) -> Reward { - match shaping { - NyxRewardShaping::EntropyReduction { - max_order, - scale, - crash_bonus, - timeout_bonus, - .. - } => { - let mut base_reward = { - let data = if result.shared_memory.is_empty() { - &result.output - } else { - &result.shared_memory - }; - let h_obs = if *max_order == 0 { - marginal_entropy_bytes(data) - } else { - entropy_rate_backend(data, *max_order, &self.config.stats_backend) - }; - let h_base = self.baseline_entropy.unwrap_or(0.0); - let er = (h_base - h_obs) * scale; - er.round() as i64 - }; - - // Add bonuses for interesting behaviors (bugs/crashes) - match &result.exit_reason { - NyxExitKind::Shutdown | NyxExitKind::Breakpoint => { - if let Some(bonus) = crash_bonus { - base_reward = base_reward.saturating_add(*bonus); - } - } - NyxExitKind::Timeout => { - if let Some(bonus) = timeout_bonus { - base_reward = base_reward.saturating_add(*bonus); - } - } - _ => {} - } - - base_reward - } - NyxRewardShaping::TraceEntropy { - scale, normalize, .. - } => { - let data = &result.trace_data; - let bits = match self.trace_model.as_mut() { - Some(model) => model.update_and_score(data), - None => 0.0, - }; - let bits = if *normalize && !data.is_empty() { - bits / data.len() as f64 - } else { - bits - }; - (bits * scale).round() as i64 - } - } - } - - fn mask_observation(&self, value: u64) -> u64 { - let bits = self.config.observation_bits; - if bits >= 64 { - value - } else if bits == 0 { - 0 - } else { - value & ((1u64 << bits) - 1) - } - } - - fn build_observation_stream(&self, result: &NyxStepResult) -> Vec { - let mut observations = match self.config.observation_policy { - NyxObservationPolicy::FromGuest => { - if let Some(obs) = result.parsed_obs { - vec![self.mask_observation(obs)] - } else { - vec![self.hash_observation(&result.shared_memory)] - } - } - NyxObservationPolicy::OutputHash => { - vec![self.hash_observation(&result.output)] - } - NyxObservationPolicy::RawOutput => { - result.output.iter().map(|b| *b as PerceptVal).collect() - } - NyxObservationPolicy::SharedMemory => result - .shared_memory - .iter() - .map(|b| *b as PerceptVal) - .collect(), - }; - - if observations.is_empty() { - observations.push(0); - } - - self.normalize_observation_stream(&mut observations); - observations - } - - fn hash_observation(&self, data: &[u8]) -> PerceptVal { - let h = robust_hash_bytes(data); - self.mask_observation(h) - } - - fn normalize_observation_stream(&self, observations: &mut Vec) { - let mask = if self.config.observation_bits >= 64 { - u64::MAX - } else if self.config.observation_bits == 0 { - 0 - } else { - (1u64 << self.config.observation_bits) - 1 - }; - - for obs in observations.iter_mut() { - *obs &= mask; - } - - let target = self.config.observation_stream_len; - if target == 0 { - return; - } - - if observations.len() > target { - match self.config.observation_stream_mode { - NyxObservationStreamMode::Truncate | NyxObservationStreamMode::PadTruncate => { - observations.truncate(target); - } - NyxObservationStreamMode::Pad => {} - } - } else if observations.len() < target { - match self.config.observation_stream_mode { - NyxObservationStreamMode::Pad | NyxObservationStreamMode::PadTruncate => { - let pad = self.config.observation_pad_byte as PerceptVal; - observations.resize(target, pad); - } - NyxObservationStreamMode::Truncate => {} - } - } - } - - fn action_count(&self) -> usize { - match &self.config.action_source { - NyxActionSource::Literal(actions) => actions.len(), - NyxActionSource::Fuzz(fuzz) => fuzz.mutators.len(), - } - } - - /// Direct access to the underlying NyxVM for advanced use cases. - pub fn vm(&self) -> &NyxVM { - &self.vm - } - - /// Mutable access to the underlying NyxVM. - pub fn vm_mut(&mut self) -> &mut NyxVM { - &mut self.vm - } - - /// Takes a new snapshot at the current state. - pub fn take_snapshot(&mut self) -> Arc { - self.vm.take_snapshot() - } - - /// Applies a specific snapshot. - pub fn apply_snapshot(&mut self, snapshot: &Arc) { - self.vm.apply_snapshot(snapshot); - } - - /// Resets trace model. - pub fn reset_trace_model(&mut self) { - if let Some(model) = &mut self.trace_model { - model.reset(); - } - } - - /// Logs crashes and interesting behaviors to file. - fn log_crash(&self, action_payload: &[u8], result: &NyxStepResult, reward: i64) { - let Some(log_path) = &self.config.crash_log else { - return; - }; - - // Only log interesting exits - let is_interesting = matches!( - result.exit_reason, - NyxExitKind::Shutdown | NyxExitKind::Breakpoint | NyxExitKind::Timeout - ); - - if !is_interesting { - return; - } - - let log_entry = serde_json::json!({ - "timestamp": std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default() - .as_secs(), - "exit_reason": format!("{:?}", result.exit_reason), - "action_payload": hex_encode(action_payload), - "action_payload_str": String::from_utf8_lossy(action_payload), - "output": String::from_utf8_lossy(&result.output), - "shared_memory": hex_encode(&result.shared_memory), - "reward": reward, - "parsed_obs": result.parsed_obs, - "parsed_rew": result.parsed_rew, - }); - - // Append to JSONL file - if let Ok(mut file) = OpenOptions::new().create(true).append(true).open(log_path) - && let Ok(json_str) = serde_json::to_string(&log_entry) - { - let _ = writeln!(file, "{}", json_str); - } - } -} - -// ============================================================================ -// Environment Trait Implementation -// ============================================================================ - -impl Environment for NyxVmEnvironment { - fn perform_action(&mut self, action: Action) { - if self.needs_reset - && let Err(e) = self.reset() - && self.config.debug_mode - { - eprintln!("[NyxVm] Reset failed: {}", e); - } - - let payload = match self.get_action_payload(action) { - Ok(payload) => payload.into_owned(), - Err(e) => { - if self.config.debug_mode { - eprintln!("[NyxVm] Invalid action: {}", e); - } - self.obs = 0; - self.rew = self.min_reward(); - self.obs_stream.clear(); - self.obs_stream.push(0); - self.step_in_episode = (self.step_in_episode + 1) % self.config.episode_steps; - if self.step_in_episode == 0 { - self.needs_reset = true; - } - return; - } - }; - - // Check action filter - if let Some(reject_reward) = self.filter_action(&payload) { - self.obs = 0; - self.rew = reject_reward.clamp(self.min_reward(), self.max_reward()); - self.obs_stream.clear(); - self.obs_stream.push(0); - self.step_in_episode = (self.step_in_episode + 1) % self.config.episode_steps; - if self.step_in_episode == 0 { - self.needs_reset = true; - } - return; - } - - // Run the step - let wrapped_payload = self.wrap_action_payload(&payload); - let result = match self.run_step(&wrapped_payload) { - Ok(result) => result, - Err(e) => { - if self.config.debug_mode { - eprintln!("[NyxVm] Step failed: {}", e); - } - self.obs = 0; - self.rew = self.min_reward(); - self.obs_stream.clear(); - self.obs_stream.push(0); - self.step_in_episode = (self.step_in_episode + 1) % self.config.episode_steps; - if self.step_in_episode == 0 { - self.needs_reset = true; - } - return; - } - }; - - // Process results - self.obs_stream = self.build_observation_stream(&result); - self.obs = self.obs_stream.first().copied().unwrap_or(0); - self.rew = self.compute_reward(&result); - - // Log crashes and interesting behaviors - self.log_crash(&payload, &result, self.rew); - - if self.config.debug_mode { - eprintln!( - "[NyxVm] Action={} Obs={} Rew={} Done={:?} Exit={:?}", - action, self.obs, self.rew, result.done, result.exit_reason - ); - } - - self.step_in_episode = (self.step_in_episode + 1) % self.config.episode_steps; - if self.step_in_episode == 0 || result.done { - self.needs_reset = true; - } - } - - fn get_observation(&self) -> PerceptVal { - self.obs - } - - fn drain_observations(&mut self) -> Vec { - if self.obs_stream.is_empty() { - vec![self.obs] - } else { - std::mem::take(&mut self.obs_stream) - } - } - - fn get_reward(&self) -> Reward { - self.rew - } - - fn is_finished(&self) -> bool { - false - } - - fn get_observation_bits(&self) -> usize { - self.config.observation_bits - } - - fn get_reward_bits(&self) -> usize { - self.config.reward_bits - } - - fn get_action_bits(&self) -> usize { - let n = self.action_count(); - if n <= 1 { - return 1; - } - (n as f64).log2().ceil() as usize - } - - fn get_num_actions(&self) -> usize { - self.action_count() - } - - fn max_reward(&self) -> Reward { - let bits = self.config.reward_bits; - if bits >= 64 { - i64::MAX - } else if bits == 0 { - 0 - } else { - (1i64 << (bits - 1)) - 1 - } - } - - fn min_reward(&self) -> Reward { - let bits = self.config.reward_bits; - if bits >= 64 { - i64::MIN - } else if bits == 0 { - 0 - } else { - -(1i64 << (bits - 1)) - } - } -} - -// ============================================================================ -// Helper Functions -// ============================================================================ - -fn robust_hash_bytes(data: &[u8]) -> u64 { - let mut h = 0u64; - for &b in data { - h = h.rotate_left(7) ^ (b as u64); - } - h -} - -fn apply_mutator( - mutator: &FuzzMutator, - input: &mut Vec, - fuzz: &NyxFuzzConfig, - rng: &mut RandomGenerator, -) { - match mutator { - FuzzMutator::FlipBit => { - if input.is_empty() { - input.push(0); - } - let idx = rng.gen_range(input.len()); - let bit = rng.gen_range(8); - input[idx] ^= 1u8 << bit; - } - FuzzMutator::FlipByte => { - if input.is_empty() { - input.push(0); - } - let idx = rng.gen_range(input.len()); - input[idx] ^= rng.next_u64() as u8; - } - FuzzMutator::InsertByte => { - let idx = if input.is_empty() { - 0 - } else { - rng.gen_range(input.len() + 1) - }; - let byte = if !fuzz.dictionary.is_empty() { - let d = rng.gen_range(fuzz.dictionary.len()); - let entry = &fuzz.dictionary[d]; - if entry.is_empty() { - 0 - } else { - entry[rng.gen_range(entry.len())] - } - } else { - rng.next_u64() as u8 - }; - input.insert(idx, byte); - } - FuzzMutator::DeleteByte => { - if input.len() > 1 { - let idx = rng.gen_range(input.len()); - input.remove(idx); - } - } - FuzzMutator::SpliceSeed => { - if fuzz.seeds.is_empty() { - return; - } - let seed = &fuzz.seeds[rng.gen_range(fuzz.seeds.len())]; - if input.is_empty() { - input.extend_from_slice(seed); - } else if !seed.is_empty() { - let cut = rng.gen_range(input.len()); - let seed_cut = rng.gen_range(seed.len()); - let mut out = Vec::new(); - out.extend_from_slice(&input[..cut]); - out.extend_from_slice(&seed[seed_cut..]); - *input = out; - } - } - FuzzMutator::ResetSeed => { - if fuzz.seeds.is_empty() { - return; - } - *input = fuzz.seeds[rng.gen_range(fuzz.seeds.len())].clone(); - } - FuzzMutator::Havoc => { - let flips = 1 + rng.gen_range(8); - for _ in 0..flips { - if input.is_empty() { - input.push(0); - } - let idx = rng.gen_range(input.len()); - input[idx] ^= rng.next_u64() as u8; - } - } - } -} - -// ============================================================================ -// Tests -// ============================================================================ - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_hex_encoding() { - let data = b"hello"; - let encoded = hex_encode(data); - assert_eq!(encoded, "68656c6c6f"); - let decoded = hex_decode(&encoded).unwrap(); - assert_eq!(decoded, data); - } - - #[test] - fn test_robust_hash() { - let data1 = b"test data"; - let data2 = b"test data"; - let data3 = b"different"; - - assert_eq!(robust_hash_bytes(data1), robust_hash_bytes(data2)); - assert_ne!(robust_hash_bytes(data1), robust_hash_bytes(data3)); - } - - #[test] - fn test_payload_encoding() { - let utf8 = PayloadEncoding::Utf8; - let hex = PayloadEncoding::Hex; - - let data = b"test"; - assert_eq!(utf8.encode(data), "test"); - assert_eq!(hex.encode(data), "74657374"); - - assert_eq!(utf8.decode("test").unwrap(), data); - assert_eq!(hex.decode("74657374").unwrap(), data); - } - - #[test] - fn trace_model_supports_predictor_backed_backends() { - let backends = vec![ - RateBackend::Match { - hash_bits: 20, - min_len: 4, - max_len: 255, - base_mix: 0.02, - confidence_scale: 1.0, - }, - RateBackend::SparseMatch { - hash_bits: 19, - min_len: 3, - max_len: 64, - gap_min: 1, - gap_max: 2, - base_mix: 0.05, - confidence_scale: 1.0, - }, - RateBackend::Ppmd { - order: 8, - memory_mb: 8, - }, - RateBackend::Calibrated { - spec: Arc::new(crate::CalibratedSpec { - base: RateBackend::Ctw { depth: 8 }, - context: crate::CalibrationContextKind::Text, - bins: 33, - learning_rate: 0.02, - bias_clip: 4.0, - }), - }, - RateBackend::Particle { - spec: Arc::new(crate::ParticleSpec { - num_particles: 4, - num_cells: 4, - cell_dim: 8, - ..crate::ParticleSpec::default() - }), - }, - RateBackend::Mixture { - spec: Arc::new(crate::MixtureSpec::new( - crate::MixtureKind::Bayes, - vec![crate::MixtureExpertSpec { - name: Some("ctw".to_string()), - log_prior: 0.0, - max_order: -1, - backend: RateBackend::Ctw { depth: 8 }, - }], - )), - }, - ]; - - for backend in backends { - let mut model = TraceModel::new(&backend, 4); - let bits = model.update_and_score(b"trace payload"); - assert!(bits.is_finite() && bits >= 0.0, "bits={bits}"); - model.reset(); - let bits_after_reset = model.update_and_score(b"trace payload"); - assert!( - bits_after_reset.is_finite() && bits_after_reset >= 0.0, - "bits_after_reset={bits_after_reset}" - ); - } - } -} diff --git a/src/axioms.rs b/src/axioms.rs deleted file mode 100644 index 37c09727..00000000 --- a/src/axioms.rs +++ /dev/null @@ -1,150 +0,0 @@ -//! # Axioms: Mathematical Property Verifiers -//! -//! This module provides generic functions to verify mathematical properties -//! that should hold for any correct implementation of information-theoretic -//! measures. - -// ============================================================================ -// Metric Axioms -// ============================================================================ - -/// Verify that a distance function d(x,x) is close to 0 (allow for small overhead). -pub fn verify_identity(metric: F, x: &[u8], tolerance: f64) -> bool -where - F: Fn(&[u8], &[u8]) -> f64, -{ - let d = metric(x, x); - d.abs() <= tolerance -} - -/// Verify symmetry: d(x,y) ≈ d(y,x). -pub fn verify_symmetry(metric: F, x: &[u8], y: &[u8], tolerance: f64) -> bool -where - F: Fn(&[u8], &[u8]) -> f64, -{ - let d_xy = metric(x, y); - let d_yx = metric(y, x); - (d_xy - d_yx).abs() <= tolerance -} - -/// Verify triangle inequality: d(x,z) ≤ d(x,y) + d(y,z). -pub fn verify_triangle_inequality( - metric: F, - x: &[u8], - y: &[u8], - z: &[u8], - tolerance: f64, -) -> bool -where - F: Fn(&[u8], &[u8]) -> f64, -{ - let d_xy = metric(x, y); - let d_yz = metric(y, z); - let d_xz = metric(x, z); - d_xz <= (d_xy + d_yz + tolerance) -} - -/// Verify non-negativity: d(x,y) ≥ 0. -pub fn verify_non_negativity(metric: F, x: &[u8], y: &[u8]) -> bool -where - F: Fn(&[u8], &[u8]) -> f64, -{ - // Allow tiny floating point errors slightly below zero - metric(x, y) >= -1e-12 -} - -// ============================================================================ -// Information Inequalities -// ============================================================================ - -/// Verify mutual information non-negativity: I(X;Y) ≥ 0. -pub fn verify_mi_nonnegative(mi: F, x: &[u8], y: &[u8]) -> bool -where - F: Fn(&[u8], &[u8]) -> f64, -{ - mi(x, y) >= -1e-12 -} - -/// Verify subadditivity: H(X,Y) ≤ H(X) + H(Y). -/// -/// This is equivalent to I(X;Y) ≥ 0. -pub fn verify_subadditivity( - joint_entropy: FJoint, - marginal_entropy: FMarg, - x: &[u8], - y: &[u8], - tolerance: f64, -) -> bool -where - FJoint: Fn(&[u8], &[u8]) -> f64, - FMarg: Fn(&[u8]) -> f64, -{ - let h_xy = joint_entropy(x, y); - let h_x = marginal_entropy(x); - let h_y = marginal_entropy(y); - h_xy <= (h_x + h_y + tolerance) -} - -/// Verify conditioning reduces entropy: H(X|Y) ≤ H(X). -pub fn verify_conditioning_reduces_entropy( - conditional_entropy: FCond, - marginal_entropy: FMarg, - x: &[u8], - y: &[u8], - tolerance: f64, -) -> bool -where - FCond: Fn(&[u8], &[u8]) -> f64, - FMarg: Fn(&[u8]) -> f64, -{ - let h_x_given_y = conditional_entropy(x, y); - let h_x = marginal_entropy(x); - h_x_given_y <= (h_x + tolerance) -} - -/// Verify chain rule: H(X,Y) = H(X) + H(Y|X). -pub fn verify_chain_rule( - joint: FJoint, - marginal: FMarg, - conditional: FCond, - x: &[u8], - y: &[u8], - tolerance: f64, -) -> bool -where - FJoint: Fn(&[u8], &[u8]) -> f64, - FMarg: Fn(&[u8]) -> f64, - FCond: Fn(&[u8], &[u8]) -> f64, // H(Y|X) -{ - let h_xy = joint(x, y); - let h_x = marginal(x); - let h_y_given_x = conditional(y, x); - - (h_xy - (h_x + h_y_given_x)).abs() <= tolerance -} - -// ============================================================================ -// Bounds -// ============================================================================ - -/// Verify NCD range: 0 ≤ NCD ≤ 1+epsilon. -/// -/// NCD theoretically can slightly exceed 1 due to compression overhead, so we allow -/// a small margin or just check it's not egregiously large. Usually NCD <= 1.1 is safe. -pub fn verify_ncd_bounds(ncd: F, x: &[u8], y: &[u8]) -> bool -where - F: Fn(&[u8], &[u8]) -> f64, -{ - let val = ncd(x, y); - (-1e-12..=1.1).contains(&val) -} - -/// Verify entropy is bounded by log2(alphabet_size). -/// For bytes, max entropy is 8.0 bits/byte. -pub fn verify_entropy_bounds(entropy: F, data: &[u8]) -> bool -where - F: Fn(&[u8]) -> f64, -{ - let h = entropy(data); - (-1e-12..=8.0 + 1e-12).contains(&h) -} diff --git a/src/backends/mod.rs b/src/backends/mod.rs deleted file mode 100644 index 66331b62..00000000 --- a/src/backends/mod.rs +++ /dev/null @@ -1,298 +0,0 @@ -//! Backend discovery helpers and canonical backend naming. -//! -//! This module provides: -//! - canonical backend name resolution for CLI/Python inputs, -//! - feature-aware availability reporting, -//! - exported lists of enabled backend families. - -/// Online probability calibration wrapper for rate predictors. -pub mod calibration; -pub mod ctw; -/// Shared policy parser/compiler for online LLM backends. -pub mod llm_policy; -/// Mamba-1 based rate/compression backend. -#[cfg(feature = "backend-mamba")] -pub mod mambazip; -/// Contiguous/sparse local match predictor primitives. -pub mod match_model; -/// Particle-latent rate backend. -pub mod particle; -/// Bounded-memory PPMD-style byte model. -pub mod ppmd; -pub mod rosaplus; -/// RWKV7-based rate/compression backend. -#[cfg(feature = "backend-rwkv")] -pub mod rwkvzip; -/// Exact online Sequitur grammar backend with byte-level predictive readout. -pub mod sequitur; -/// Sparse/gapped match predictor that wraps [`match_model`]. -pub mod sparse_match; -/// Text/repeat context feature extraction for adaptive backends. -pub mod text_context; -pub mod zpaq_rate; -use crate::coders::CoderType; - -/// Outcome of resolving a backend alias to a canonical backend name. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum BackendAvailability { - /// Backend is compiled and available. - Enabled(&'static str), - /// Backend alias is recognized, but the required Cargo feature is disabled. - Disabled { - /// Canonical backend name. - canonical: &'static str, - /// Cargo feature needed to enable this backend. - feature: &'static str, - }, -} - -/// Canonical names for rate backends recognized by CLI/API alias resolution. -pub const AVAILABLE_RATE_BACKENDS: &[&str] = &[ - "rosaplus", - "ctw", - "fac-ctw", - "match", - "sparse-match", - "ppmd", - "sequitur", - "calibrated", - #[cfg(feature = "backend-mamba")] - "mamba", - #[cfg(feature = "backend-rwkv")] - "rwkv7", - #[cfg(feature = "backend-zpaq")] - "zpaq", - "mixture", - "particle", -]; - -/// Canonical names for available compression backends in this build. -pub const AVAILABLE_COMPRESSION_BACKENDS: &[&str] = &[ - #[cfg(feature = "backend-zpaq")] - "zpaq", - "rate-ac", - "rate-rans", - #[cfg(feature = "backend-rwkv")] - "rwkv7", -]; - -/// Resolve a user-provided rate backend alias to a canonical backend name. -/// -/// Returns `None` when the alias is unknown, and `BackendAvailability::Disabled` -/// when known but not enabled in the current feature set. -pub fn resolve_rate_backend_name(input: &str) -> Option { - let key = input.trim().to_ascii_lowercase(); - match key.as_str() { - "rosaplus" | "rosa" => Some(BackendAvailability::Enabled("rosaplus")), - "ctw" => Some(BackendAvailability::Enabled("ctw")), - "fac-ctw" | "facctw" => Some(BackendAvailability::Enabled("fac-ctw")), - "match" => Some(BackendAvailability::Enabled("match")), - "sparse-match" | "sparse_match" | "sparsematch" => { - Some(BackendAvailability::Enabled("sparse-match")) - } - "ppmd" | "ppm" => Some(BackendAvailability::Enabled("ppmd")), - "sequitur" => Some(BackendAvailability::Enabled("sequitur")), - "calibrated" | "cal" => Some(BackendAvailability::Enabled("calibrated")), - "zpaq" => { - if cfg!(feature = "backend-zpaq") { - Some(BackendAvailability::Enabled("zpaq")) - } else { - Some(BackendAvailability::Disabled { - canonical: "zpaq", - feature: "backend-zpaq", - }) - } - } - "mixture" | "mix" => Some(BackendAvailability::Enabled("mixture")), - "particle" | "particles" => Some(BackendAvailability::Enabled("particle")), - "mamba" | "mamba1" => { - if cfg!(feature = "backend-mamba") { - Some(BackendAvailability::Enabled("mamba")) - } else { - Some(BackendAvailability::Disabled { - canonical: "mamba", - feature: "backend-mamba", - }) - } - } - "rwkv7" | "rwkv" => { - if cfg!(feature = "backend-rwkv") { - Some(BackendAvailability::Enabled("rwkv7")) - } else { - Some(BackendAvailability::Disabled { - canonical: "rwkv7", - feature: "backend-rwkv", - }) - } - } - _ => None, - } -} - -/// Resolve a user-provided compression backend alias to a canonical backend name. -/// -/// Returns `None` when the alias is unknown, and `BackendAvailability::Disabled` -/// when known but not enabled in the current feature set. -pub fn resolve_compression_backend_name(input: &str) -> Option { - let key = input.trim().to_ascii_lowercase(); - match key.as_str() { - "zpaq" => { - if cfg!(feature = "backend-zpaq") { - Some(BackendAvailability::Enabled("zpaq")) - } else { - Some(BackendAvailability::Disabled { - canonical: "zpaq", - feature: "backend-zpaq", - }) - } - } - "rwkv7" | "rwkv" => { - if cfg!(feature = "backend-rwkv") { - Some(BackendAvailability::Enabled("rwkv7")) - } else { - Some(BackendAvailability::Disabled { - canonical: "rwkv7", - feature: "backend-rwkv", - }) - } - } - "rate-ac" | "rate_ac" | "rateac" => Some(BackendAvailability::Enabled("rate-ac")), - "rate-rans" | "rate_rans" | "raterans" => Some(BackendAvailability::Enabled("rate-rans")), - _ => None, - } -} - -/// Parse a generic entropy coder alias (`"ac"`/`"rans"`). -pub fn parse_rate_coder(v: &str) -> Option { - match v { - "ac" | "AC" => Some(CoderType::AC), - "rans" | "RANS" | "rANS" => Some(CoderType::RANS), - _ => None, - } -} - -/// Parse an RWKV entropy coder alias (`"ac"`/`"rans"`). -#[cfg(feature = "backend-rwkv")] -pub fn parse_rwkv7_coder(v: &str) -> Option { - parse_rate_coder(v) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn resolve_rate_backend_name_canonicalizes_aliases() { - assert_eq!( - resolve_rate_backend_name(" Rosa "), - Some(BackendAvailability::Enabled("rosaplus")) - ); - assert_eq!( - resolve_rate_backend_name("facctw"), - Some(BackendAvailability::Enabled("fac-ctw")) - ); - assert_eq!( - resolve_rate_backend_name("mix"), - Some(BackendAvailability::Enabled("mixture")) - ); - assert_eq!( - resolve_rate_backend_name("sequitur"), - Some(BackendAvailability::Enabled("sequitur")) - ); - assert_eq!(resolve_rate_backend_name("unknown"), None); - } - - #[test] - fn resolve_rate_backend_name_reports_feature_disabled() { - if cfg!(feature = "backend-zpaq") { - assert_eq!( - resolve_rate_backend_name("zpaq"), - Some(BackendAvailability::Enabled("zpaq")) - ); - } else { - assert_eq!( - resolve_rate_backend_name("zpaq"), - Some(BackendAvailability::Disabled { - canonical: "zpaq", - feature: "backend-zpaq", - }) - ); - } - - if cfg!(feature = "backend-rwkv") { - assert_eq!( - resolve_rate_backend_name("rwkv7"), - Some(BackendAvailability::Enabled("rwkv7")) - ); - } else { - assert_eq!( - resolve_rate_backend_name("rwkv"), - Some(BackendAvailability::Disabled { - canonical: "rwkv7", - feature: "backend-rwkv", - }) - ); - } - - if cfg!(feature = "backend-mamba") { - assert_eq!( - resolve_rate_backend_name("mamba1"), - Some(BackendAvailability::Enabled("mamba")) - ); - } else { - assert_eq!( - resolve_rate_backend_name("mamba"), - Some(BackendAvailability::Disabled { - canonical: "mamba", - feature: "backend-mamba", - }) - ); - } - } - - #[test] - fn resolve_compression_backend_name_canonicalizes_aliases() { - assert_eq!(resolve_compression_backend_name("unknown"), None); - - if cfg!(feature = "backend-zpaq") { - assert_eq!( - resolve_compression_backend_name("zpaq"), - Some(BackendAvailability::Enabled("zpaq")) - ); - } else { - assert_eq!( - resolve_compression_backend_name("zpaq"), - Some(BackendAvailability::Disabled { - canonical: "zpaq", - feature: "backend-zpaq", - }) - ); - } - - assert_eq!( - resolve_compression_backend_name("rate_ac"), - Some(BackendAvailability::Enabled("rate-ac")) - ); - assert_eq!( - resolve_compression_backend_name("raterans"), - Some(BackendAvailability::Enabled("rate-rans")) - ); - - if cfg!(feature = "backend-rwkv") { - assert_eq!( - resolve_compression_backend_name("rwkv"), - Some(BackendAvailability::Enabled("rwkv7")) - ); - } - } - - #[cfg(feature = "backend-rwkv")] - #[test] - fn parse_rwkv7_coder_accepts_common_aliases() { - assert_eq!(parse_rwkv7_coder("ac"), Some(CoderType::AC)); - assert_eq!(parse_rwkv7_coder("AC"), Some(CoderType::AC)); - assert_eq!(parse_rwkv7_coder("rans"), Some(CoderType::RANS)); - assert_eq!(parse_rwkv7_coder("RANS"), Some(CoderType::RANS)); - assert_eq!(parse_rwkv7_coder("nope"), None); - } -} diff --git a/src/backends/ppmd.rs b/src/backends/ppmd.rs deleted file mode 100644 index 2388089d..00000000 --- a/src/backends/ppmd.rs +++ /dev/null @@ -1,248 +0,0 @@ -use ahash::AHashMap; -use std::collections::VecDeque; - -const PDF_MIN: f64 = crate::mixture::DEFAULT_MIN_PROB; - -#[derive(Clone, Debug, Default)] -struct ContextStats { - counts: Vec<(u8, u16)>, - total: u32, -} - -impl ContextStats { - fn observe(&mut self, symbol: u8) { - if let Some((_, count)) = self.counts.iter_mut().find(|(s, _)| *s == symbol) { - *count = count.saturating_add(1); - } else { - self.counts.push((symbol, 1)); - } - self.total = self.total.saturating_add(1); - if self.total > 4096 { - self.rescale(); - } - } - - fn rescale(&mut self) { - self.total = 0; - self.counts.retain_mut(|(_, count)| { - *count = (*count).div_ceil(2).max(1); - self.total += *count as u32; - true - }); - } -} - -#[derive(Clone, Debug)] -/// Bounded-memory PPMD-inspired byte model with interpolation across orders. -pub struct PpmdModel { - order: usize, - max_contexts: usize, - contexts: Vec>, - queue: VecDeque<(usize, u64)>, - history: Vec, - pdf: [f64; 256], - cdf: [f64; 257], - valid: bool, - cdf_valid: bool, -} - -impl PpmdModel { - /// Create a model with maximum `order` and approximate memory budget in MiB. - pub fn new(order: usize, memory_mb: usize) -> Self { - let order = order.max(1); - let max_contexts = (memory_mb.max(1) * 1024 * 1024) / 96; - Self { - order, - max_contexts: max_contexts.max(1024), - contexts: (0..=order).map(|_| AHashMap::new()).collect(), - queue: VecDeque::new(), - history: Vec::new(), - pdf: [1.0 / 256.0; 256], - cdf: uniform_cdf(), - valid: false, - cdf_valid: false, - } - } - - /// Fill `out` with the current normalized byte PDF. - pub fn fill_pdf(&mut self, out: &mut [f64; 256]) { - self.ensure_pdf_inner(false); - out.copy_from_slice(&self.pdf); - } - - /// Borrow the current normalized byte PDF. - pub fn pdf(&mut self) -> &[f64; 256] { - self.ensure_pdf_inner(false); - &self.pdf - } - - /// Borrow the cumulative distribution derived from the current PDF. - pub fn cdf(&mut self) -> &[f64; 257] { - self.ensure_pdf_inner(true); - &self.cdf - } - - /// Return `ln(max(P(symbol), min_prob))`. - pub fn log_prob(&mut self, symbol: u8, min_prob: f64) -> f64 { - self.ensure_pdf_inner(false); - self.pdf[symbol as usize].max(min_prob).ln() - } - - /// Observe one symbol and update all active contexts up to model order. - pub fn update(&mut self, symbol: u8) { - let max_order = self.order.min(self.history.len()); - for ord in 0..=max_order { - let key = self.context_key(ord); - let map = &mut self.contexts[ord]; - if !map.contains_key(&key) { - map.insert(key, ContextStats::default()); - self.queue.push_back((ord, key)); - } - if let Some(ctx) = map.get_mut(&key) { - ctx.observe(symbol); - } - } - self.prune(); - self.history.push(symbol); - self.valid = false; - self.cdf_valid = false; - } - - /// Reset only the conditioning history while preserving fitted contexts. - pub fn reset_history(&mut self) { - self.history.clear(); - self.valid = false; - self.cdf_valid = false; - self.pdf.fill(1.0 / 256.0); - self.cdf = uniform_cdf(); - } - - /// Advance conditioning history without updating fitted context counts. - pub fn update_history_only(&mut self, symbol: u8) { - self.history.push(symbol); - self.valid = false; - self.cdf_valid = false; - } - - fn ensure_pdf_inner(&mut self, want_cdf: bool) { - if self.valid { - if want_cdf && !self.cdf_valid { - build_cdf_from_pdf(&self.pdf, &mut self.cdf); - self.cdf_valid = true; - } - return; - } - let mut lower = [1.0 / 256.0; 256]; - let max_order = self.order.min(self.history.len()); - for ord in 0..=max_order { - let key = self.context_key(ord); - if let Some(ctx) = self.contexts[ord].get(&key) { - lower = interpolate_context(ctx, &lower); - } - } - self.pdf.copy_from_slice(&lower); - normalize_pdf_and_maybe_cdf( - &mut self.pdf, - if want_cdf { Some(&mut self.cdf) } else { None }, - ); - self.valid = true; - self.cdf_valid = want_cdf; - } - - fn prune(&mut self) { - let mut total_contexts: usize = self.contexts.iter().map(|m| m.len()).sum(); - while total_contexts > self.max_contexts { - let Some((ord, key)) = self.queue.pop_front() else { - break; - }; - if self.contexts[ord].remove(&key).is_some() { - total_contexts -= 1; - } - } - } - - fn context_key(&self, ord: usize) -> u64 { - if ord == 0 { - return 0; - } - let start = self.history.len() - ord; - hash_bytes(&self.history[start..]) - } -} - -fn interpolate_context(ctx: &ContextStats, lower: &[f64; 256]) -> [f64; 256] { - let distinct = ctx.counts.len() as f64; - let denom = (ctx.total as f64) + distinct + 1.0; - let escape = (distinct + 1.0) / denom; - let mut out = [0.0; 256]; - for i in 0..256 { - out[i] = lower[i] * escape; - } - for &(symbol, count) in &ctx.counts { - out[symbol as usize] += (count as f64) / denom; - } - out -} - -fn normalize_pdf_and_maybe_cdf(pdf: &mut [f64; 256], mut cdf: Option<&mut [f64; 257]>) { - let mut sum = 0.0; - for p in pdf.iter_mut() { - *p = if p.is_finite() { - (*p).max(PDF_MIN) - } else { - PDF_MIN - }; - sum += *p; - } - if !(sum.is_finite()) || sum <= 0.0 { - let u = 1.0 / 256.0; - pdf.fill(u); - if let Some(cdf) = cdf.as_deref_mut() { - *cdf = uniform_cdf(); - } - return; - } - let inv = 1.0 / sum; - if let Some(cdf) = cdf.as_deref_mut() { - cdf[0] = 0.0; - let mut acc = 0.0; - for i in 0..256 { - pdf[i] *= inv; - acc += pdf[i]; - cdf[i + 1] = acc; - } - } else { - for p in pdf.iter_mut() { - *p *= inv; - } - } -} - -#[inline] -fn uniform_cdf() -> [f64; 257] { - let mut cdf = [0.0; 257]; - let inv = 1.0 / 256.0; - for (i, slot) in cdf.iter_mut().enumerate() { - *slot = (i as f64) * inv; - } - cdf -} - -#[inline] -fn build_cdf_from_pdf(pdf: &[f64; 256], cdf: &mut [f64; 257]) { - cdf[0] = 0.0; - let mut acc = 0.0; - for i in 0..256 { - acc += pdf[i]; - cdf[i + 1] = acc; - } -} - -fn hash_bytes(bytes: &[u8]) -> u64 { - let mut h = 0xCBF2_9CE4_8422_2325u64; - for &b in bytes { - h ^= b as u64; - h = h.wrapping_mul(0x1000_0000_01B3); - } - h -} diff --git a/src/backends/zpaq_rate.rs b/src/backends/zpaq_rate.rs deleted file mode 100644 index 7187e79e..00000000 --- a/src/backends/zpaq_rate.rs +++ /dev/null @@ -1,328 +0,0 @@ -//! ZPAQ-backed sequential rate model. -//! -//! This backend estimates `log p(x_t | x_{, - pending_symbol: Option, - pending_bits: f64, - min_prob: f64, - method: String, - } - - impl ZpaqRateModel { - /// Create a new model with the provided streamable ZPAQ `method`. - /// - /// `min_prob` clamps very small probabilities for numerical stability. - pub fn new(method: impl Into, min_prob: f64) -> Self { - let method = method.into(); - let min_prob = if min_prob.is_finite() && min_prob > 0.0 { - min_prob - } else { - DEFAULT_MIN_PROB - }; - - let compressor = StreamingCompressor::new(method.as_str()).unwrap_or_else(|e| { - panic!("ZPAQ rate backend requires a streamable method; got '{method}': {e}") - }); - - Self { - stream: ZpaqStreaming { - compressor, - last_bits: 0.0, - }, - history: Vec::new(), - pending_symbol: None, - pending_bits: 0.0, - min_prob, - method, - } - } - - /// Reset model state and clear any pending prediction cache. - pub fn reset(&mut self) { - let method = self.method.clone(); - let compressor = StreamingCompressor::new(method.as_str()).unwrap_or_else(|e| { - panic!("ZPAQ rate backend requires a streamable method; got '{method}': {e}") - }); - self.stream = ZpaqStreaming { - compressor, - last_bits: 0.0, - }; - self.history.clear(); - self.pending_symbol = None; - self.pending_bits = 0.0; - } - - fn rebuild_stream_from_history(&mut self) { - let method = self.method.clone(); - let compressor = StreamingCompressor::new(method.as_str()).unwrap_or_else(|e| { - panic!("ZPAQ rate backend requires a streamable method; got '{method}': {e}") - }); - self.stream = ZpaqStreaming { - compressor, - last_bits: 0.0, - }; - let history = self.history.clone(); - for b in history { - let _ = self.encode_bits(b); - } - self.pending_symbol = None; - self.pending_bits = 0.0; - } - - fn log_prob_from_history(&self, symbol: u8) -> f64 { - let mut compressor = - StreamingCompressor::new(self.method.as_str()).expect("zpaq streaming new failed"); - for &b in &self.history { - compressor - .push(b) - .expect("zpaq streaming compression failed"); - } - let before = compressor.bits(); - compressor - .push(symbol) - .expect("zpaq streaming compression failed"); - let bits = (compressor.bits() - before).max(0.0); - let logp = -(bits * LN_2); - logp.max(self.min_prob.ln()) - } - - fn encode_bits(&mut self, symbol: u8) -> f64 { - let before = self.stream.last_bits; - self.stream - .compressor - .push(symbol) - .expect("zpaq streaming compression failed"); - let after = self.stream.compressor.bits(); - self.stream.last_bits = after; - (after - before).max(0.0) - } - - /// Return `ln p(symbol | history)` under the current model state. - /// - /// This may cache the encoded-bit result for a matching immediate `update`. - pub fn log_prob(&mut self, symbol: u8) -> f64 { - if let Some(pending) = self.pending_symbol { - if pending == symbol { - let logp = -(self.pending_bits * LN_2); - return logp.max(self.min_prob.ln()); - } - // We cannot rollback `StreamingCompressor`; rebuild to committed history. - self.rebuild_stream_from_history(); - } - - let bits = self.encode_bits(symbol); - self.pending_symbol = Some(symbol); - self.pending_bits = bits; - let logp = -(bits * LN_2); - logp.max(self.min_prob.ln()) - } - - /// Fill 256-way log-probabilities for the current committed history without mutation. - pub fn fill_log_probs(&mut self, out: &mut [f64; 256]) { - // Treat fill as a read-only query of committed history. - self.rebuild_stream_from_history(); - for (sym, slot) in out.iter_mut().enumerate() { - *slot = self.log_prob_from_history(sym as u8); - } - } - - /// Advance model state with one observed symbol. - pub fn update(&mut self, symbol: u8) { - if let Some(pending) = self.pending_symbol - && pending == symbol - { - self.pending_symbol = None; - self.history.push(symbol); - return; - } - if self.pending_symbol.is_some() { - self.rebuild_stream_from_history(); - } - let _ = self.encode_bits(symbol); - self.pending_symbol = None; - self.pending_bits = 0.0; - self.history.push(symbol); - } - - /// Score and consume an entire byte slice, returning total code length in bits. - pub fn update_and_score(&mut self, data: &[u8]) -> f64 { - if data.is_empty() { - return 0.0; - } - self.pending_symbol = None; - self.pending_bits = 0.0; - let mut bits = 0.0; - for &b in data { - bits += self.encode_bits(b); - self.history.push(b); - } - bits - } - } - - impl Clone for ZpaqRateModel { - fn clone(&self) -> Self { - let mut cloned = Self::new(self.method.clone(), self.min_prob); - if !self.history.is_empty() { - let _ = cloned.update_and_score(&self.history); - } - // Preserve speculative pending state so clone() is state-equivalent - // even when called between log_prob() and update(). - if let Some(symbol) = self.pending_symbol { - let bits = cloned.encode_bits(symbol); - cloned.pending_symbol = Some(symbol); - cloned.pending_bits = bits; - } else { - cloned.pending_symbol = None; - cloned.pending_bits = 0.0; - } - cloned - } - } - - /// Validate that `method` is streamable and accepted by the ZPAQ backend. - pub fn validate_zpaq_rate_method(method: &str) -> Result<(), String> { - StreamingCompressor::new(method) - .map(|_| ()) - .map_err(|e| e.to_string()) - } - - #[cfg(test)] - mod tests { - use super::*; - - #[test] - fn zpaq_log_prob_update_matches_update_and_score() { - let data = b"the quick brown fox jumps over the lazy dog"; - let mut model_a = ZpaqRateModel::new("1", 1e-9); - let mut bits_a = 0.0; - for &b in data { - let logp = model_a.log_prob(b); - bits_a += -logp / LN_2; - model_a.update(b); - } - - let mut model_b = ZpaqRateModel::new("1", 1e-9); - let bits_b = model_b.update_and_score(data); - - let diff = (bits_a - bits_b).abs(); - assert!(diff < 1e-6, "bits mismatch: {bits_a} vs {bits_b}"); - } - - #[test] - fn zpaq_fill_log_probs_is_non_mutating() { - let history = b"zpaq fill non mutating"; - let mut model_a = ZpaqRateModel::new("1", 1e-9); - let mut model_b = ZpaqRateModel::new("1", 1e-9); - for &b in history { - model_a.update(b); - model_b.update(b); - } - - let mut row = [0.0f64; 256]; - model_b.fill_log_probs(&mut row); - - let sym = b'x'; - let lp_a = model_a.log_prob(sym); - let lp_b = model_b.log_prob(sym); - assert!((lp_a - lp_b).abs() < 1e-9, "lp_a={lp_a} lp_b={lp_b}"); - assert!((row[sym as usize] - lp_a).abs() < 1e-9); - - model_a.update(sym); - model_b.update(sym); - let next_sym = b'y'; - let lp_a2 = model_a.log_prob(next_sym); - let lp_b2 = model_b.log_prob(next_sym); - assert!((lp_a2 - lp_b2).abs() < 1e-9, "lp_a2={lp_a2} lp_b2={lp_b2}"); - } - - #[test] - fn zpaq_clone_preserves_pending_prediction_state() { - let mut model_a = ZpaqRateModel::new("1", 1e-9); - for &b in b"clone preserves pending state" { - model_a.update(b); - } - - let probe = b'x'; - let lp_a = model_a.log_prob(probe); - let mut model_b = model_a.clone(); - let lp_b = model_b.log_prob(probe); - assert!((lp_a - lp_b).abs() < 1e-9, "lp_a={lp_a} lp_b={lp_b}"); - - model_a.update(probe); - model_b.update(probe); - let next = b'y'; - let lp_a2 = model_a.log_prob(next); - let lp_b2 = model_b.log_prob(next); - assert!((lp_a2 - lp_b2).abs() < 1e-9, "lp_a2={lp_a2} lp_b2={lp_b2}"); - } - } -} - -#[cfg(not(feature = "backend-zpaq"))] -mod imp { - #[derive(Clone)] - pub struct ZpaqRateModel { - min_log_prob: f64, - } - - impl ZpaqRateModel { - pub fn new(_method: impl Into, min_prob: f64) -> Self { - let min_prob = if min_prob.is_finite() && min_prob > 0.0 { - min_prob - } else { - 1e-12 - }; - Self { - min_log_prob: min_prob.ln(), - } - } - - pub fn reset(&mut self) {} - - pub fn log_prob(&mut self, _symbol: u8) -> f64 { - self.min_log_prob - } - - pub fn fill_log_probs(&mut self, out: &mut [f64; 256]) { - out.fill(self.min_log_prob); - } - - pub fn update(&mut self, _symbol: u8) {} - - pub fn update_and_score(&mut self, data: &[u8]) -> f64 { - let bits_per_symbol = -self.min_log_prob / std::f64::consts::LN_2; - bits_per_symbol * (data.len() as f64) - } - } - - pub fn validate_zpaq_rate_method(_method: &str) -> Result<(), String> { - Err("zpaq backend disabled at compile time".to_string()) - } -} - -/// Stateful ZPAQ-based rate estimator. -pub use imp::ZpaqRateModel; -/// Validate that a ZPAQ method string is streamable and usable for rate modeling. -pub use imp::validate_zpaq_rate_method; diff --git a/src/lib.rs b/src/lib.rs deleted file mode 100644 index 99a6e4c2..00000000 --- a/src/lib.rs +++ /dev/null @@ -1,3937 +0,0 @@ -#![allow(unsafe_op_in_unsafe_fn)] - -//! # InfoTheory: Information Theoretic Estimators & Metrics -//! -//! This crate provides a comprehensive suite of information-theoretic primitives for -//! quantifying complexity, dependence, and similarity between data sequences. -//! -//! It implements two primary classes of estimators: -//! 1. **Compression-based (Kolmogorov Complexity)**: Using the ZPAQ compression algorithm to estimate -//! Normalized Compression Distance (NCD). -//! 2. **Entropy-based (Shannon Information)**: Using both exact marginal histograms (for i.i.d. data) -//! and the ROSA (Rapid Online Suffix Automaton) predictive language model (for sequential data) -//! to estimate Entropy, Mutual Information, and related distances. -//! -//! ## Mathematical Primitives -//! -//! The library implements the following core measures. For sequential data, "Rate" variants -//! use the ROSA model to estimate `Ĥ(X)` (entropy rate), while "Marginal" variants -//! treat data as a bag-of-bytes (i.i.d.) and compute `H(X)` from histograms. -//! -//! ### 1. Normalized Compression Distance (NCD) -//! Approximates the Normalized Information Distance (NID) using a compressor `C`. -//! -//! `NCD(x,y) = (C(xy) - min(C(x), C(y))) / max(C(x), C(y))` -//! -//! ### 2. Normalized Entropy Distance (NED) -//! An entropic analogue to NCD, defined using Shannon entropy `H`. -//! -//! `NED(X,Y) = (H(X,Y) - min(H(X), H(Y))) / max(H(X), H(Y))` -//! -//! ### 3. Normalized Transform Effort (NTE) -//! Based on the Variation of Information (VI), normalized by the maximum entropy. -//! -//! `NTE(X,Y) = (H(X|Y) + H(Y|X)) / max(H(X), H(Y)) = (2H(X,Y) - H(X) - H(Y)) / max(H(X), H(Y))` -//! -//! ### 4. Mutual Information (MI) -//! Measures the amount of information obtained about one random variable by observing another. -//! -//! `I(X;Y) = H(X) + H(Y) - H(X,Y)` -//! -//! ### 5. Divergences & Distances -//! * **Total Variation Distance (TVD)**: `δ(P,Q) = 0.5 * Σ |P(x) - Q(x)|` -//! * **Normalized Hellinger Distance (NHD)**: `sqrt(1 - Σ sqrt(P(x)Q(x)))` -//! * **Kullback-Leibler Divergence (KL)**: `D_KL(P||Q) = Σ P(x) log(P(x)/Q(x))` -//! * **Jensen-Shannon Divergence (JSD)**: Symmetrized and smoothed KL divergence. -//! -//! ### 6. Intrinsic Dependence (ID) -//! Measures the redundancy within a sequence, comparing marginal entropy to entropy rate. -//! -//! `ID(X) = (H_marginal(X) - H_rate(X)) / H_marginal(X)` -//! -//! ### 7. Resistance to Transformation -//! Quantifies how much information is preserved after a transformation `T` is applied. -//! -//! `R(X, T) = I(X; T(X)) / H(X)` -//! -//! ## Usage -//! -//! ```rust,no_run -//! use infotheory::{ncd_vitanyi, mutual_information_bytes, NcdVariant}; -//! -//! let x = b"some data sequence"; -//! let y = b"another data sequence"; -//! -//! // Compression-based distance -//! let ncd = ncd_vitanyi("file1.txt", "file2.txt", "5"); -//! -//! // Entropy-based mutual information (Marginal / i.i.d.) -//! let mi_marg = mutual_information_bytes(x, y, 0); -//! -//! // Entropy-based mutual information (Rate / Sequential, max_order=8) -//! let mi_rate = mutual_information_bytes(x, y, 8); -//! ``` - -/// AIXI planning components, environments, and model abstractions. -pub mod aixi; -/// Core information-theoretic axioms and validation helpers. -pub mod axioms; -/// Entropy/compression backend implementations and backend discovery. -pub mod backends; -/// Entropy coder implementations (AC and rANS). -pub mod coders; -/// Rate-coded compression helpers built on generic rate backends. -pub mod compression; -/// Synthetic data generators for information-theory experiments. -pub mod datagen; -/// Diagnostic tooling for exact AC/log-loss mixture tracing. -pub mod diagnostics; -/// Online Bayesian/switching/MDL mixture predictors. -pub mod mixture; -pub(crate) mod neural_mix; -/// Information-theoretic code search pipeline (3-stage: prefilter, filter, KMI rerank). -pub mod search; -pub(crate) mod simd_math; -/// CTW and FAC-CTW backend types. -pub use backends::ctw; -#[cfg(feature = "backend-mamba")] -/// Mamba backend types and compressor. -pub use backends::mambazip; -/// Match-based repeat predictor. -pub use backends::match_model; -/// Particle-latent filter ensemble rate backend. -pub use backends::particle; -/// PPMD-style byte model. -pub use backends::ppmd; -/// ROSA+ backend types. -pub use backends::rosaplus; -#[cfg(feature = "backend-rwkv")] -/// RWKV backend types and compressor. -pub use backends::rwkvzip; -/// Exact online Sequitur backend types. -pub use backends::sequitur; -/// Sparse/gapped match predictor. -pub use backends::sparse_match; -/// ZPAQ rate-model adapter. -pub use backends::zpaq_rate; - -use rayon::prelude::*; - -use crate::coders::CoderType; -use std::cell::RefCell; -#[cfg(any(feature = "backend-rwkv", feature = "backend-mamba"))] -use std::collections::HashMap; -use std::sync::Arc; -use std::sync::OnceLock; - -/// How generated symbols should update the model state. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum GenerationUpdateMode { - /// Keep adapting/fitting on generated bytes. - Adaptive, - /// Freeze fitted parameters/statistics and only advance conditioning state. - Frozen, -} - -/// How to pick the next byte from the model distribution. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum GenerationStrategy { - /// Deterministic argmax over the next-byte distribution. - Greedy, - /// Seeded sampling from the next-byte distribution. - Sample, -} - -/// Generation options shared by the library API and CLI. -#[derive(Clone, Copy, Debug)] -pub struct GenerationConfig { - /// Byte-selection strategy. - pub strategy: GenerationStrategy, - /// Whether generated bytes should keep adapting the model. - pub update_mode: GenerationUpdateMode, - /// RNG seed used by [`GenerationStrategy::Sample`]. - pub seed: u64, - /// Softmax temperature for sampling. `<= 0` behaves like greedy. - pub temperature: f64, - /// Optional top-k truncation. `0` disables it. - pub top_k: usize, - /// Optional nucleus truncation. Values `>= 1.0` disable it. - pub top_p: f64, -} - -impl Default for GenerationConfig { - fn default() -> Self { - Self::sampled_frozen(42) - } -} - -impl GenerationConfig { - /// Deterministic frozen continuation. - pub const fn greedy_frozen() -> Self { - Self { - strategy: GenerationStrategy::Greedy, - update_mode: GenerationUpdateMode::Frozen, - seed: 0xD00D_F00D_CAFE_BABEu64, - temperature: 1.0, - top_k: 0, - top_p: 1.0, - } - } - - /// Seeded frozen sampling from the model distribution. - pub const fn sampled_frozen(seed: u64) -> Self { - Self { - strategy: GenerationStrategy::Sample, - update_mode: GenerationUpdateMode::Frozen, - seed, - temperature: 1.0, - top_k: 0, - top_p: 1.0, - } - } -} - -struct GenerationRng { - state: u64, -} - -impl GenerationRng { - fn new(seed: u64) -> Self { - Self { - state: if seed == 0 { - 0xD00D_F00D_CAFE_BABEu64 - } else { - seed - }, - } - } - - fn next_u64(&mut self) -> u64 { - let mut x = self.state; - x ^= x << 13; - x ^= x >> 7; - x ^= x << 17; - self.state = x; - x - } - - fn next_f64(&mut self) -> f64 { - (self.next_u64() as f64) / (u64::MAX as f64) - } -} - -static NUM_THREADS: OnceLock = OnceLock::new(); - -thread_local! { - #[cfg(feature = "backend-mamba")] - static MAMBA_TLS: RefCell> = RefCell::new(HashMap::new()); - #[cfg(feature = "backend-mamba")] - static MAMBA_RATE_TLS: RefCell> = RefCell::new(HashMap::new()); - #[cfg(feature = "backend-mamba")] - static MAMBA_METHOD_TLS: RefCell> = RefCell::new(HashMap::new()); - #[cfg(feature = "backend-rwkv")] - static RWKV_TLS: RefCell> = RefCell::new(HashMap::new()); - #[cfg(feature = "backend-rwkv")] - static RWKV_RATE_TLS: RefCell> = RefCell::new(HashMap::new()); - #[cfg(feature = "backend-rwkv")] - static RWKV_METHOD_TLS: RefCell> = RefCell::new(HashMap::new()); -} - -#[cfg(feature = "backend-zpaq")] -impl Default for CompressionBackend { - fn default() -> Self { - CompressionBackend::Zpaq { - method: "5".to_string(), - } - } -} - -#[cfg(not(feature = "backend-zpaq"))] -impl Default for CompressionBackend { - fn default() -> Self { - CompressionBackend::Rate { - rate_backend: RateBackend::default(), - coder: CoderType::AC, - framing: compression::FramingMode::Raw, - } - } -} - -thread_local! { - static DEFAULT_CTX: RefCell = RefCell::new(InfotheoryCtx::default()); -} - -/// Returns the current default information theory context for the thread. -pub fn get_default_ctx() -> InfotheoryCtx { - DEFAULT_CTX.with(|ctx| ctx.borrow().clone()) -} - -/// Sets the default information theory context for the thread. -pub fn set_default_ctx(ctx: InfotheoryCtx) { - DEFAULT_CTX.with(|c| *c.borrow_mut() = ctx); -} - -#[inline(always)] -fn with_default_ctx(f: impl FnOnce(&InfotheoryCtx) -> R) -> R { - DEFAULT_CTX.with(|ctx| f(&ctx.borrow())) -} - -/// Mutual information rate estimate under an explicit `backend`. -/// -/// Inputs are aligned to the shared prefix length. -pub fn mutual_information_rate_backend( - x: &[u8], - y: &[u8], - max_order: i64, - backend: &RateBackend, -) -> f64 { - let (x, y) = aligned_prefix(x, y); - if x.is_empty() { - return 0.0; - } - // For CTW, we might want a special aligned implementation? - // Using standard formula for now. - let h_x = entropy_rate_backend(x, max_order, backend); - let h_y = entropy_rate_backend(y, max_order, backend); - let h_xy = joint_entropy_rate_backend(x, y, max_order, backend); - (h_x + h_y - h_xy).max(0.0) -} - -/// Normalized entropy distance under an explicit `backend`. -/// -/// Returns a value in `[0, 1]` after clamping. -pub fn ned_rate_backend(x: &[u8], y: &[u8], max_order: i64, backend: &RateBackend) -> f64 { - let (x, y) = aligned_prefix(x, y); - if x.is_empty() { - return 0.0; - } - let h_x = entropy_rate_backend(x, max_order, backend); - let h_y = entropy_rate_backend(y, max_order, backend); - let h_xy = joint_entropy_rate_backend(x, y, max_order, backend); - let min_h = h_x.min(h_y); - let max_h = h_x.max(h_y); - if max_h == 0.0 { - 0.0 - } else { - ((h_xy - min_h) / max_h).clamp(0.0, 1.0) - } -} - -/// Normalized transform effort (variation-of-information form) under an explicit `backend`. -/// -/// Returns a value in `[0, 2]` after clamping. -pub fn nte_rate_backend(x: &[u8], y: &[u8], max_order: i64, backend: &RateBackend) -> f64 { - let (x, y) = aligned_prefix(x, y); - if x.is_empty() { - return 0.0; - } - let h_x = entropy_rate_backend(x, max_order, backend); - let h_y = entropy_rate_backend(y, max_order, backend); - let h_xy = joint_entropy_rate_backend(x, y, max_order, backend); - let max_h = h_x.max(h_y); - if max_h == 0.0 { - 0.0 - } else { - // VI = H(X|Y) + H(Y|X) can be as large as H(X) + H(Y) ≈ 2*max(H) - // for independent sequences, so NTE ∈ [0, 2] - let vi = (h_xy - h_x).max(0.0) + (h_xy - h_y).max(0.0); - (vi / max_h).clamp(0.0, 2.0) - } -} - -/// Core predictive model class used by the library. -/// -/// `RateBackend` is the shared model class behind entropy-rate estimation, -/// rate-coded compression, generation, and the world-model interface used by -/// MC-AIXI/AIQI planners. -#[derive(Clone)] -pub enum RateBackend { - /// ROSA+ suffix-automaton estimator. - RosaPlus, - /// Local contiguous match predictor. - Match { - /// Number of retained hash bits for suffix lookup. - hash_bits: usize, - /// Minimum repeat length required before predicting. - min_len: usize, - /// Maximum repeat length used for confidence scaling. - max_len: usize, - /// Residual probability mass left for non-match symbols. - base_mix: f64, - /// Confidence multiplier applied to short-match tapering. - confidence_scale: f64, - }, - /// Sparse/gapped local match predictor. - SparseMatch { - /// Number of retained hash bits for spaced-suffix lookup. - hash_bits: usize, - /// Minimum spaced repeat length required before predicting. - min_len: usize, - /// Maximum spaced repeat length used for confidence scaling. - max_len: usize, - /// Minimum gap between matched bytes. - gap_min: usize, - /// Maximum gap between matched bytes. - gap_max: usize, - /// Residual probability mass left for non-match symbols. - base_mix: f64, - /// Confidence multiplier applied to short-match tapering. - confidence_scale: f64, - }, - /// Pure-Rust bounded-memory PPMD-style model. - Ppmd { - /// Maximum context order. - order: usize, - /// Approximate memory budget in MiB. - memory_mb: usize, - }, - /// Exact online Sequitur grammar backend with byte-level predictive readout. - Sequitur { - /// Maximum number of terminal bytes retained per grammar-derived context. - context_bytes: usize, - }, - #[cfg(feature = "backend-mamba")] - /// Mamba model loaded from explicit weights. - Mamba { - /// Loaded Mamba model. - model: Arc, - }, - #[cfg(feature = "backend-mamba")] - /// Mamba method string (e.g. `file:...` or `cfg:...[;policy:...]`) resolved lazily. - MambaMethod { - /// Mamba method string. - method: String, - }, - #[cfg(feature = "backend-rwkv")] - /// RWKV7 model loaded from explicit weights. - Rwkv7 { - /// Loaded RWKV7 model. - model: Arc, - }, - #[cfg(feature = "backend-rwkv")] - /// RWKV7 method string (e.g. `file:...` or `cfg:...[;policy:...]`) resolved lazily. - Rwkv7Method { - /// RWKV7 method string. - method: String, - }, - /// ZPAQ compression-based rate model (streamable methods only). - Zpaq { - /// ZPAQ method string (streamable modes only for rate estimation). - method: String, - }, - /// Online mixture over `RateBackend` experts. - /// - /// `Bayes`, `Switching`, and `Convex` follow - /// "On Ensemble Techniques for AIXI Approximation"; `FadingBayes`, - /// `Mdl`, and `Neural` are repository extensions. - Mixture { - /// Mixture expert/runtime specification. - spec: Arc, - }, - /// Particle-latent filter ensemble. - Particle { - /// Particle filter specification. - spec: Arc, - }, - /// Calibrated wrapper over another bytewise backend. - Calibrated { - /// Calibration specification. - spec: Arc, - }, - /// Action-Conditional CTW (single context tree). - Ctw { - /// Context tree depth. - depth: usize, - }, - /// Factorized Action-Conditional CTW (k trees for k-bit percepts). - FacCtw { - /// Base context depth. - base_depth: usize, - /// Number of percept bits. - num_percept_bits: usize, - /// Encoding width in bits. - encoding_bits: usize, - }, -} - -#[allow(clippy::derivable_impls)] -impl Default for RateBackend { - fn default() -> Self { - #[cfg(feature = "backend-rosa")] - { - RateBackend::RosaPlus - } - #[cfg(all(not(feature = "backend-rosa"), feature = "backend-zpaq"))] - { - RateBackend::Zpaq { - method: "1".to_string(), - } - } - #[cfg(all(not(feature = "backend-rosa"), not(feature = "backend-zpaq")))] - { - RateBackend::Ctw { depth: 16 } - } - } -} - -/// Compression backend used by NCD/compression-size operations. -#[derive(Clone)] -pub enum CompressionBackend { - /// ZPAQ compressor with explicit method string. - Zpaq { - /// ZPAQ method (for example `"1"` or `"5"`). - method: String, - }, - #[cfg(feature = "backend-rwkv")] - /// RWKV7 model as an entropy-coded compressor. - Rwkv7 { - /// Loaded RWKV7 model. - model: Arc, - /// Entropy coder used for coding model PDFs. - coder: CoderType, - }, - /// Generic rate-coded compressor wrapping an arbitrary rate backend. - Rate { - /// Predictive rate backend. - rate_backend: RateBackend, - /// Entropy coder used for coding model PDFs. - coder: CoderType, - /// Framing mode for output payloads. - framing: compression::FramingMode, - }, -} - -/// Shared maximum nesting depth for recursive mixture specifications. -pub const MAX_MIXTURE_NESTING: usize = 8; - -/// Mixture policy kind for rate-backend mixtures. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum MixtureKind { - /// Standard Bayesian mixture with fixed expert weights. - Bayes, - /// Bayesian mixture with exponential weight decay. - FadingBayes, - /// Switching mixture using the fixed-share update from - /// "On Ensemble Techniques for AIXI Approximation". - Switching, - /// Online convex mixture with projected-simplex weight updates. - Convex, - /// MDL-style best-expert selector. - Mdl, - /// Bytewise neural logistic mixer (fx2-cmix style adaptation). - Neural, -} - -/// Adaptive schedule family for switching and convex mixtures. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum MixtureScheduleMode { - /// Use the implementation's default exposed parameterization. - /// - /// - `Switching`: constant switch rate `alpha` - /// - `Convex`: step size `alpha / sqrt(t)` - Default, - /// Use the theorem schedule from - /// "On Ensemble Techniques for AIXI Approximation". - /// - /// - `Switching`: `alpha_t = 1 / t` - /// - `Convex`: `eta_t = epsilon / sqrt(t)` under this implementation's - /// natural-log gradient, matching the bit-loss schedule analyzed in - /// "On Ensemble Techniques for AIXI Approximation" after - /// accounting for the `1 / ln(2)` factor in the base-2 gradient - /// - /// This preserves configured expert priors; exact theorem hypotheses for - /// switching still additionally require uniform priors. - Theorem, -} - -impl Default for MixtureScheduleMode { - fn default() -> Self { - Self::Default - } -} - -/// Parse a mixture kind name with the shared alias table used across CLI, Python, and WASM. -pub fn parse_mixture_kind_name(kind: &str) -> Result { - match kind.trim().to_ascii_lowercase().as_str() { - "bayes" | "bayes-mix" | "bayes_mix" => Ok(MixtureKind::Bayes), - "fading" | "fading-bayes" | "fading_bayes" => Ok(MixtureKind::FadingBayes), - "switch" | "switching" | "switch-mix" | "switch_mix" => Ok(MixtureKind::Switching), - "convex" | "convex-mix" | "convex_mix" => Ok(MixtureKind::Convex), - "mdl" | "selector" | "mdr" => Ok(MixtureKind::Mdl), - "neural" | "neural-mix" | "neural_mix" | "mix" | "mixture" | "fx2" | "fx2-cmix" - | "fx2_cmix" => Ok(MixtureKind::Neural), - other => Err(format!("unknown mixture kind '{other}'")), - } -} - -/// Parse a mixture schedule mode with the shared alias table used across CLI, Python, and WASM. -pub fn parse_mixture_schedule_name(schedule: &str) -> Result { - match schedule.trim().to_ascii_lowercase().as_str() { - "" | "default" | "constant" | "const" => Ok(MixtureScheduleMode::Default), - "theorem" | "paper" | "paper-theorem" | "paper_theorem" => Ok(MixtureScheduleMode::Theorem), - other => Err(format!("unknown mixture schedule '{other}'")), - } -} - -/// Fixed context families for calibrated PDF wrappers. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum CalibrationContextKind { - /// Single global calibration row. - Global, - /// Previous-byte class only. - ByteClass, - /// Text-structure-aware context hash. - Text, - /// Repeat-aware context hash. - Repeat, - /// Joint text/repeat-aware context hash. - TextRepeat, -} - -/// Configuration for a calibrated wrapper rate backend. -#[derive(Clone)] -pub struct CalibratedSpec { - /// Base backend whose PDF is calibrated. - pub base: RateBackend, - /// Context family controlling table row selection. - pub context: CalibrationContextKind, - /// Number of probability bins per row. - pub bins: usize, - /// Online learning rate for observed-symbol updates. - pub learning_rate: f64, - /// Symmetric clip applied to calibration weights. - pub bias_clip: f64, -} - -/// Expert specification for mixture backends. -#[derive(Clone)] -pub struct MixtureExpertSpec { - /// Optional expert display name. - pub name: Option, - /// Log prior weight (natural log). Uniform priors can be `0.0`. - pub log_prior: f64, - /// Max order for ROSA experts (ignored for other backends). - pub max_order: i64, - /// Underlying backend for this expert. - pub backend: RateBackend, -} - -/// Mixture specification for rate-backend mixtures. -#[derive(Clone)] -pub struct MixtureSpec { - /// Mixture policy. - pub kind: MixtureKind, - /// Adaptive schedule family for supported mixture kinds. - pub schedule: MixtureScheduleMode, - /// Shared scalar parameter: switch rate for `Switching`, step-size scale for `Convex`, - /// learning rate for `Neural`, and generic alpha for the remaining families. - /// - /// In theorem mode for `Switching` and `Convex`, this field is retained for API - /// compatibility but is not used by the update schedule. - pub alpha: f64, - /// Decay factor for fading Bayes mixtures. - pub decay: Option, - /// Expert list. - pub experts: Vec, -} - -impl MixtureSpec { - /// Build a mixture specification from kind and expert list. - pub fn new(kind: MixtureKind, experts: Vec) -> Self { - Self { - kind, - schedule: MixtureScheduleMode::Default, - alpha: 0.01, - decay: None, - experts, - } - } - - /// Set the schedule family. - pub fn with_schedule(mut self, schedule: MixtureScheduleMode) -> Self { - self.schedule = schedule; - self - } - - /// Set the family-specific alpha parameter. - pub fn with_alpha(mut self, alpha: f64) -> Self { - self.alpha = alpha; - self - } - - /// Set fading decay factor. - pub fn with_decay(mut self, decay: f64) -> Self { - self.decay = Some(decay); - self - } - - /// Validate the mixture configuration before building runtime state. - pub fn validate(&self) -> Result<(), String> { - validate_mixture_spec_with_depth(self, MAX_MIXTURE_NESTING) - } - - /// Convert to executable expert configs for runtime mixture evaluation. - pub fn build_experts(&self) -> Vec { - self.experts - .iter() - .map(|spec| { - crate::mixture::ExpertConfig::from_rate_backend( - spec.name.clone(), - spec.log_prior, - spec.backend.clone(), - spec.max_order, - ) - }) - .collect() - } -} - -fn validate_mixture_spec_with_depth(spec: &MixtureSpec, depth: usize) -> Result<(), String> { - if depth == 0 { - return Err("mixture spec nesting too deep".to_string()); - } - validate_mixture_spec_shallow(spec)?; - for (index, expert) in spec.experts.iter().enumerate() { - validate_rate_backend_with_depth(&expert.backend, depth - 1).map_err(|err| { - if let Some(name) = expert.name.as_deref() { - format!("mixture expert '{name}' invalid: {err}") - } else { - format!("mixture expert #{} invalid: {err}", index + 1) - } - })?; - } - Ok(()) -} - -fn validate_mixture_spec_shallow(spec: &MixtureSpec) -> Result<(), String> { - if spec.experts.is_empty() { - return Err("mixture spec must include at least one expert".to_string()); - } - if !spec.alpha.is_finite() { - return Err("mixture alpha must be finite".to_string()); - } - if spec - .experts - .iter() - .any(|expert| !expert.log_prior.is_finite()) - { - return Err("mixture expert log_prior must be finite".to_string()); - } - if let Some(decay) = spec.decay { - if !decay.is_finite() || !(0.0..1.0).contains(&decay) { - return Err("mixture decay must be in (0, 1)".to_string()); - } - } - if matches!(spec.kind, MixtureKind::FadingBayes) && spec.decay.is_none() { - return Err("fading Bayes mixture requires decay".to_string()); - } - if spec.schedule != MixtureScheduleMode::Default - && !matches!(spec.kind, MixtureKind::Switching | MixtureKind::Convex) - { - return Err( - "mixture schedule is only supported for switching and convex mixtures".to_string(), - ); - } - match (spec.kind, spec.schedule) { - (MixtureKind::Switching, MixtureScheduleMode::Default) => { - if !(0.0..=1.0).contains(&spec.alpha) { - return Err("switching mixture alpha must be in [0, 1]".to_string()); - } - } - (MixtureKind::Convex, MixtureScheduleMode::Default) - | (MixtureKind::Neural, MixtureScheduleMode::Default) => { - if spec.alpha <= 0.0 { - return Err("mixture alpha must be > 0".to_string()); - } - } - (MixtureKind::Neural, MixtureScheduleMode::Theorem) => unreachable!(), - _ => {} - } - Ok(()) -} - -fn validate_rate_backend_with_depth(backend: &RateBackend, depth: usize) -> Result<(), String> { - match backend { - RateBackend::Sequitur { context_bytes } => { - if *context_bytes < 2 { - Err("sequitur context_bytes must be >= 2".to_string()) - } else { - Ok(()) - } - } - RateBackend::Mixture { spec } => validate_mixture_spec_with_depth(spec.as_ref(), depth), - RateBackend::Particle { spec } => spec.validate(), - RateBackend::Calibrated { spec } => { - if depth == 0 { - return Err("calibrated spec nesting too deep".to_string()); - } - validate_rate_backend_with_depth(&spec.base, depth - 1) - .map_err(|err| format!("calibrated base invalid: {err}")) - } - _ => Ok(()), - } -} - -/// Validate a rate backend, including nested mixture/calibrated subgraphs. -pub fn validate_rate_backend(backend: &RateBackend) -> Result<(), String> { - validate_rate_backend_with_depth(backend, MAX_MIXTURE_NESTING) -} - -/// Configuration for a particle-latent filter ensemble rate backend. -#[derive(Clone, Debug)] -pub struct ParticleSpec { - /// Number of particles in the ensemble. - pub num_particles: usize, - /// Context window length for rolling byte context. - pub context_window: usize, - /// Number of latent update unroll steps per byte. - pub unroll_steps: usize, - /// Number of latent cells per particle. - pub num_cells: usize, - /// Dimensionality of each latent cell. - pub cell_dim: usize, - /// Number of discrete rules for soft routing. - pub num_rules: usize, - /// Hidden dimension for the selector MLP. - pub selector_hidden: usize, - /// Hidden dimension for each rule MLP. - pub rule_hidden: usize, - /// Dimension of per-rule noise input (ignored when deterministic). - pub noise_dim: usize, - /// Whether to use fully deterministic execution (no RNG). - pub deterministic: bool, - /// Whether to inject noise into rule inputs (ignored when deterministic). - pub enable_noise: bool, - /// Base scale for deterministic hash-noise injected into rule inputs. - pub noise_scale: f64, - /// Number of steps over which injected noise linearly anneals to zero. - pub noise_anneal_steps: usize, - /// Learning rate for readout layer SGD. - pub learning_rate_readout: f64, - /// Learning rate for selector MLP SGD. - pub learning_rate_selector: f64, - /// Learning rate for rule MLP SGD. - pub learning_rate_rule: f64, - /// Truncated backpropagation-through-time depth (number of recent steps). - pub bptt_depth: usize, - /// Momentum coefficient for selector/rule online updates (in [0, 1)). - pub optimizer_momentum: f64, - /// Gradient clipping threshold (max abs value per element). - pub grad_clip: f64, - /// Latent cell state clipping threshold (max abs value per element). - pub state_clip: f64, - /// Forgetting factor for particle log-weights (0 = no forgetting). - pub forget_lambda: f64, - /// Effective sample size ratio threshold for resampling (in (0, 1]). - pub resample_threshold: f64, - /// Fraction of particles to mutate after resampling (in [0, 1]). - pub mutate_fraction: f64, - /// Scale of hash-noise perturbation applied during mutation. - pub mutate_scale: f64, - /// Whether mutation also perturbs model parameters (state is always mutated). - pub mutate_model_params: bool, - /// Diagnostics print interval in steps (0 disables particle diagnostics logs). - pub diagnostics_interval: usize, - /// Minimum probability floor for numerical stability. - pub min_prob: f64, - /// Master seed for deterministic initialization and mutation. - pub seed: u64, -} - -impl Default for ParticleSpec { - fn default() -> Self { - Self { - num_particles: 16, - context_window: 32, - unroll_steps: 2, - num_cells: 8, - cell_dim: 32, - num_rules: 4, - selector_hidden: 64, - rule_hidden: 64, - noise_dim: 8, - deterministic: true, - enable_noise: false, - noise_scale: 0.10, - noise_anneal_steps: 8192, - learning_rate_readout: 0.01, - learning_rate_selector: 1e-4, - learning_rate_rule: 3e-4, - bptt_depth: 3, - optimizer_momentum: 0.05, - grad_clip: 1.0, - state_clip: 8.0, - forget_lambda: 0.0, - resample_threshold: 0.5, - mutate_fraction: 0.1, - mutate_scale: 0.01, - mutate_model_params: false, - diagnostics_interval: 0, - min_prob: 2f64.powi(-24), - seed: 42, - } - } -} - -impl ParticleSpec { - /// Validate all fields, returning an error message on failure. - pub fn validate(&self) -> Result<(), String> { - if self.num_particles == 0 { - return Err("num_particles must be > 0".into()); - } - if self.context_window == 0 { - return Err("context_window must be > 0".into()); - } - if self.unroll_steps == 0 { - return Err("unroll_steps must be > 0".into()); - } - if self.num_cells == 0 { - return Err("num_cells must be > 0".into()); - } - if self.cell_dim == 0 { - return Err("cell_dim must be > 0".into()); - } - if self.num_rules == 0 { - return Err("num_rules must be > 0".into()); - } - if self.selector_hidden == 0 { - return Err("selector_hidden must be > 0".into()); - } - if self.rule_hidden == 0 { - return Err("rule_hidden must be > 0".into()); - } - if !self.learning_rate_readout.is_finite() || self.learning_rate_readout < 0.0 { - return Err("learning_rate_readout must be finite and non-negative".into()); - } - if !self.learning_rate_selector.is_finite() || self.learning_rate_selector < 0.0 { - return Err("learning_rate_selector must be finite and non-negative".into()); - } - if !self.learning_rate_rule.is_finite() || self.learning_rate_rule < 0.0 { - return Err("learning_rate_rule must be finite and non-negative".into()); - } - if !self.noise_scale.is_finite() || self.noise_scale < 0.0 { - return Err("noise_scale must be finite and non-negative".into()); - } - if !self.optimizer_momentum.is_finite() - || self.optimizer_momentum < 0.0 - || self.optimizer_momentum >= 1.0 - { - return Err("optimizer_momentum must be finite and in [0, 1)".into()); - } - if self.bptt_depth == 0 { - return Err("bptt_depth must be > 0".into()); - } - if !(self.resample_threshold > 0.0 && self.resample_threshold <= 1.0) { - return Err("resample_threshold must be in (0, 1]".into()); - } - if !(self.mutate_fraction >= 0.0 && self.mutate_fraction <= 1.0) { - return Err("mutate_fraction must be in [0, 1]".into()); - } - if !(self.min_prob > 0.0 && self.min_prob < 0.5) { - return Err("min_prob must be in (0, 0.5)".into()); - } - Ok(()) - } -} - -/// Reusable execution context holding default rate and compression backends. -#[derive(Clone, Default)] -pub struct InfotheoryCtx { - /// Default rate backend for entropy/rate metrics. - pub rate_backend: RateBackend, - /// Default compression backend for NCD/compression primitives. - pub compression_backend: CompressionBackend, -} - -/// Stateful rate-backend session for fitting, conditioning, and continuation. -pub struct RateBackendSession { - predictor: crate::mixture::RateBackendPredictor, -} - -impl RateBackendSession { - /// Create a session from an explicit backend. - pub fn from_backend( - backend: RateBackend, - max_order: i64, - total_symbols: Option, - ) -> Result { - use crate::mixture::OnlineBytePredictor; - - validate_rate_backend(&backend)?; - let mut predictor = crate::mixture::RateBackendPredictor::from_backend( - backend, - max_order, - crate::mixture::DEFAULT_MIN_PROB, - ); - predictor.begin_stream(total_symbols)?; - Ok(Self { predictor }) - } - - /// Observe bytes while adapting/fitting the model. - pub fn observe(&mut self, data: &[u8]) { - use crate::mixture::OnlineBytePredictor; - - for &byte in data { - self.predictor.update(byte); - } - } - - /// Advance conditioning state without changing fitted parameters/statistics. - pub fn condition(&mut self, data: &[u8]) { - use crate::mixture::OnlineBytePredictor; - - for &byte in data { - self.predictor.update_frozen(byte); - } - } - - /// Reset dynamic conditioning state while preserving fitted parameters/statistics. - pub fn reset_frozen(&mut self, total_symbols: Option) -> Result<(), String> { - use crate::mixture::OnlineBytePredictor; - - self.predictor.reset_frozen(total_symbols) - } - - /// Fill the 256-way next-byte log-probabilities. - pub fn fill_log_probs(&mut self, out: &mut [f64; 256]) { - use crate::mixture::OnlineBytePredictor; - - self.predictor.fill_log_probs(out); - } - - /// Generate continuation bytes from the current state. - pub fn generate_bytes(&mut self, bytes: usize, config: GenerationConfig) -> Vec { - use crate::mixture::OnlineBytePredictor; - - if bytes == 0 { - return Vec::new(); - } - - let mut out = Vec::with_capacity(bytes); - let mut logps = [0.0f64; 256]; - let mut rng = GenerationRng::new(config.seed); - - for _ in 0..bytes { - match &mut self.predictor { - // ROSA's scalar path remains the reference for continuation generation. - crate::mixture::RateBackendPredictor::Rosa { .. } => { - for (sym, slot) in logps.iter_mut().enumerate() { - *slot = self.predictor.log_prob(sym as u8); - } - } - _ => self.predictor.fill_log_probs(&mut logps), - } - let byte = pick_generated_byte(&logps, config, &mut rng); - match config.update_mode { - GenerationUpdateMode::Adaptive => self.predictor.update(byte), - GenerationUpdateMode::Frozen => self.predictor.update_frozen(byte), - } - out.push(byte); - } - - out - } - - /// Finalize the underlying stream if the backend needs it. - pub fn finish(&mut self) -> Result<(), String> { - use crate::mixture::OnlineBytePredictor; - - self.predictor.finish_stream() - } -} - -impl InfotheoryCtx { - /// Create a context from explicit rate and compression backends. - pub fn new(rate_backend: RateBackend, compression_backend: CompressionBackend) -> Self { - Self { - rate_backend, - compression_backend, - } - } - - /// Create a context with ROSA+ rate backend and ZPAQ compression backend. - pub fn with_zpaq(method: impl Into) -> Self { - Self { - rate_backend: RateBackend::RosaPlus, - compression_backend: CompressionBackend::Zpaq { - method: method.into(), - }, - } - } - - /// Compressed length of one byte slice under this context's compressor. - pub fn compress_size(&self, data: &[u8]) -> u64 { - compress_size_backend(data, &self.compression_backend) - } - - /// Compressed length of chained slices under one stream. - pub fn compress_size_chain(&self, parts: &[&[u8]]) -> u64 { - compress_size_chain_backend(parts, &self.compression_backend) - } - - /// Create a stateful session for the active rate backend. - pub fn rate_backend_session( - &self, - max_order: i64, - total_symbols: Option, - ) -> Result { - RateBackendSession::from_backend(self.rate_backend.clone(), max_order, total_symbols) - } - - /// Entropy-rate estimate for `data` under this context's rate backend. - pub fn entropy_rate_bytes(&self, data: &[u8], max_order: i64) -> f64 { - entropy_rate_backend(data, max_order, &self.rate_backend) - } - - /// Biased entropy-rate estimate (plugin variant) for `data`. - pub fn biased_entropy_rate_bytes(&self, data: &[u8], max_order: i64) -> f64 { - biased_entropy_rate_backend(data, max_order, &self.rate_backend) - } - - /// Cross entropy of `test_data` under model trained on `train_data`. - pub fn cross_entropy_rate_bytes( - &self, - test_data: &[u8], - train_data: &[u8], - max_order: i64, - ) -> f64 { - cross_entropy_rate_backend(test_data, train_data, max_order, &self.rate_backend) - } - - /// Cross entropy with order-0 fast-path fallback when `max_order == 0`. - pub fn cross_entropy_bytes(&self, test_data: &[u8], train_data: &[u8], max_order: i64) -> f64 { - if max_order == 0 { - if test_data.is_empty() { - return 0.0; - } - let p_x = byte_histogram(test_data); - let p_y = byte_histogram(train_data); - let mut h = 0.0f64; - for i in 0..256 { - if p_x[i] > 0.0 { - let q_y = p_y[i].max(1e-12); - h -= p_x[i] * q_y.log2(); - } - } - h - } else { - self.cross_entropy_rate_bytes(test_data, train_data, max_order) - } - } - - /// Joint entropy-rate estimate `H(X,Y)` under aligned-prefix semantics. - pub fn joint_entropy_rate_bytes(&self, x: &[u8], y: &[u8], max_order: i64) -> f64 { - let (x, y) = aligned_prefix(x, y); - if x.is_empty() { - return 0.0; - } - joint_entropy_rate_backend(x, y, max_order, &self.rate_backend) - } - - /// Conditional entropy-rate estimate `H(X|Y)`. - pub fn conditional_entropy_rate_bytes(&self, x: &[u8], y: &[u8], max_order: i64) -> f64 { - let (x, y) = aligned_prefix(x, y); - if x.is_empty() { - return 0.0; - } - let h_xy = self.joint_entropy_rate_bytes(x, y, max_order); - let h_y = self.entropy_rate_bytes(y, max_order); - (h_xy - h_y).max(0.0) - } - - /// Compute `H(data | prefix_parts)` by conditioning the active rate backend - /// on an explicit prefix chain. - pub fn cross_entropy_conditional_chain(&self, prefix_parts: &[&[u8]], data: &[u8]) -> f64 { - match &self.rate_backend { - RateBackend::RosaPlus => { - let mut prefix = Vec::new(); - let total: usize = prefix_parts.iter().map(|p| p.len()).sum(); - prefix.reserve(total); - for p in prefix_parts { - prefix.extend_from_slice(p); - } - cross_entropy_rate_backend(data, &prefix, -1, &RateBackend::RosaPlus) - } - RateBackend::Match { .. } - | RateBackend::SparseMatch { .. } - | RateBackend::Ppmd { .. } - | RateBackend::Sequitur { .. } - | RateBackend::Calibrated { .. } => { - prequential_rate_backend(data, prefix_parts, -1, &self.rate_backend) - } - #[cfg(feature = "backend-rwkv")] - RateBackend::Rwkv7 { model } => with_rwkv_tls(model, |c| { - c.cross_entropy_conditional_chain(prefix_parts, data) - .unwrap_or_else(|e| panic!("rwkv conditional-chain scoring failed: {e:#}")) - }), - #[cfg(feature = "backend-rwkv")] - RateBackend::Rwkv7Method { method } => with_rwkv_method_tls(method, |c| { - c.cross_entropy_conditional_chain(prefix_parts, data) - .unwrap_or_else(|e| { - panic!("rwkv method conditional-chain scoring failed: {e:#}") - }) - }), - #[cfg(feature = "backend-mamba")] - RateBackend::Mamba { model } => with_mamba_tls(model, |c| { - c.cross_entropy_conditional_chain(prefix_parts, data) - .unwrap_or_else(|e| panic!("mamba conditional-chain scoring failed: {e:#}")) - }), - #[cfg(feature = "backend-mamba")] - RateBackend::MambaMethod { method } => with_mamba_method_tls(method, |c| { - c.cross_entropy_conditional_chain(prefix_parts, data) - .unwrap_or_else(|e| { - panic!("mamba method conditional-chain scoring failed: {e:#}") - }) - }), - RateBackend::Ctw { depth } => { - if data.is_empty() { - return 0.0; - } - let mut tree = crate::ctw::ContextTree::new(*depth); - for &part in prefix_parts { - for &b in part { - for i in (0..8).rev() { - tree.update(((b >> i) & 1) == 1); - } - } - } - let log_p_prefix = tree.get_log_block_probability(); - for &b in data { - for i in (0..8).rev() { - tree.update(((b >> i) & 1) == 1); - } - } - let log_p_joint = tree.get_log_block_probability(); - let log_p_cond = log_p_joint - log_p_prefix; - let bits = -log_p_cond / std::f64::consts::LN_2; - bits / (data.len() as f64) - } - RateBackend::Zpaq { method } => { - if data.is_empty() { - return 0.0; - } - let mut model = - crate::zpaq_rate::ZpaqRateModel::new(method.clone(), 2f64.powi(-24)); - for &part in prefix_parts { - model.update_and_score(part); - } - let bits = model.update_and_score(data); - bits / (data.len() as f64) - } - RateBackend::Mixture { spec } => { - if data.is_empty() { - return 0.0; - } - let experts = spec.build_experts(); - let mut mix = crate::mixture::build_mixture_runtime(spec.as_ref(), &experts) - .unwrap_or_else(|e| panic!("MixtureSpec invalid: {e}")); - let total = prefix_parts - .iter() - .map(|p| p.len() as u64) - .sum::() - .saturating_add(data.len() as u64); - mix.begin_stream(Some(total)) - .unwrap_or_else(|e| panic!("Mixture stream init failed: {e}")); - for &part in prefix_parts { - for &b in part { - mix.step(b); - } - } - let mut bits = 0.0; - for &b in data { - bits -= mix.step(b) / std::f64::consts::LN_2; - } - bits / (data.len() as f64) - } - RateBackend::Particle { spec } => { - if data.is_empty() { - return 0.0; - } - let mut runtime = crate::particle::ParticleRuntime::new(spec.as_ref()); - for &part in prefix_parts { - for &b in part { - runtime.step(b); - } - } - let mut bits = 0.0; - for &b in data { - bits -= runtime.step(b) / std::f64::consts::LN_2; - } - bits / (data.len() as f64) - } - RateBackend::FacCtw { - base_depth, - num_percept_bits: _, - encoding_bits, - } => { - if data.is_empty() { - return 0.0; - } - let bits_per_byte = (*encoding_bits).clamp(1, 8); - let mut fac = crate::ctw::FacContextTree::new(*base_depth, bits_per_byte); - for &part in prefix_parts { - for &b in part { - // Fix Issue 1: LSB-first - for i in 0..bits_per_byte { - let bit_idx = i; - // b >> i gets the i-th bit (0 is LSB) - fac.update(((b >> i) & 1) == 1, bit_idx); - } - } - } - let log_p_prefix = fac.get_log_block_probability(); - for &b in data { - for i in 0..bits_per_byte { - let bit_idx = i; - fac.update(((b >> i) & 1) == 1, bit_idx); - } - } - let log_p_joint = fac.get_log_block_probability(); - let log_p_cond = log_p_joint - log_p_prefix; - let bits = -log_p_cond / std::f64::consts::LN_2; - bits / (data.len() as f64) - } - } - } - - /// Generate a continuation from `prompt` with [`GenerationConfig::default()`]. - /// - /// The default is deterministic frozen sampling with seed `42`. - pub fn generate_bytes(&self, prompt: &[u8], bytes: usize, max_order: i64) -> Vec { - self.generate_bytes_with_config(prompt, bytes, max_order, GenerationConfig::default()) - } - - /// Generate a continuation from `prompt` using an explicit generation config. - pub fn generate_bytes_with_config( - &self, - prompt: &[u8], - bytes: usize, - max_order: i64, - config: GenerationConfig, - ) -> Vec { - generate_rate_backend_chain(&[prompt], bytes, max_order, &self.rate_backend, config) - } - - /// Generate a continuation after conditioning on an explicit chain of prefix parts. - pub fn generate_bytes_conditional_chain( - &self, - prefix_parts: &[&[u8]], - bytes: usize, - max_order: i64, - ) -> Vec { - self.generate_bytes_conditional_chain_with_config( - prefix_parts, - bytes, - max_order, - GenerationConfig::default(), - ) - } - - /// Generate a continuation after conditioning on an explicit chain of prefix parts. - pub fn generate_bytes_conditional_chain_with_config( - &self, - prefix_parts: &[&[u8]], - bytes: usize, - max_order: i64, - config: GenerationConfig, - ) -> Vec { - generate_rate_backend_chain(prefix_parts, bytes, max_order, &self.rate_backend, config) - } - - /// NCD between byte slices using this context's compression backend. - pub fn ncd_bytes(&self, x: &[u8], y: &[u8], variant: NcdVariant) -> f64 { - ncd_bytes_backend(x, y, &self.compression_backend, variant) - } - - /// Rate-backend mutual information estimate. - pub fn mutual_information_rate_bytes(&self, x: &[u8], y: &[u8], max_order: i64) -> f64 { - mutual_information_rate_backend(x, y, max_order, &self.rate_backend) - } - - /// Mutual information with `max_order == 0` marginal fast-path. - pub fn mutual_information_bytes(&self, x: &[u8], y: &[u8], max_order: i64) -> f64 { - if max_order == 0 { - mutual_information_marg_bytes(x, y) - } else { - self.mutual_information_rate_bytes(x, y, max_order) - } - } - - /// Conditional entropy with aligned-prefix semantics. - pub fn conditional_entropy_bytes(&self, x: &[u8], y: &[u8], max_order: i64) -> f64 { - let (x, y) = aligned_prefix(x, y); - if max_order == 0 { - let h_xy = joint_marginal_entropy_bytes(x, y); - let h_y = marginal_entropy_bytes(y); - (h_xy - h_y).max(0.0) - } else { - let h_xy = self.joint_entropy_rate_bytes(x, y, max_order); - let h_y = self.entropy_rate_bytes(y, max_order); - (h_xy - h_y).max(0.0) - } - } - - /// Normalized entropy distance (NED) under this context. - pub fn ned_bytes(&self, x: &[u8], y: &[u8], max_order: i64) -> f64 { - if max_order == 0 { - ned_marg_bytes(x, y) - } else { - ned_rate_backend(x, y, max_order, &self.rate_backend) - } - } - - /// Conservative NED normalization variant. - pub fn ned_cons_bytes(&self, x: &[u8], y: &[u8], max_order: i64) -> f64 { - let (x, y) = aligned_prefix(x, y); - let (h_x, h_y, h_xy) = if max_order == 0 { - ( - marginal_entropy_bytes(x), - marginal_entropy_bytes(y), - joint_marginal_entropy_bytes(x, y), - ) - } else { - ( - self.entropy_rate_bytes(x, max_order), - self.entropy_rate_bytes(y, max_order), - self.joint_entropy_rate_bytes(x, y, max_order), - ) - }; - let min_h = h_x.min(h_y); - if h_xy == 0.0 { - 0.0 - } else { - ((h_xy - min_h) / h_xy).clamp(0.0, 1.0) - } - } - - /// Normalized transform effort (NTE) under this context. - pub fn nte_bytes(&self, x: &[u8], y: &[u8], max_order: i64) -> f64 { - if max_order == 0 { - nte_marg_bytes(x, y) - } else { - nte_rate_backend(x, y, max_order, &self.rate_backend) - } - } - - /// Intrinsic dependence score in `[0,1]`. - pub fn intrinsic_dependence_bytes(&self, data: &[u8], max_order: i64) -> f64 { - let h_marginal = marginal_entropy_bytes(data); - if h_marginal < 1e-9 { - return 0.0; - } - let h_rate = self.entropy_rate_bytes(data, max_order); - ((h_marginal - h_rate) / h_marginal).clamp(0.0, 1.0) - } - - /// Resistance-to-transformation ratio `I(X;T(X))/H(X)` in `[0,1]`. - pub fn resistance_to_transformation_bytes(&self, x: &[u8], tx: &[u8], max_order: i64) -> f64 { - let (x, tx) = aligned_prefix(x, tx); - let h_x = if max_order == 0 { - marginal_entropy_bytes(x) - } else { - self.entropy_rate_bytes(x, max_order) - }; - if h_x < 1e-9 { - return 0.0; - } - let mi = self.mutual_information_bytes(x, tx, max_order); - (mi / h_x).clamp(0.0, 1.0) - } -} - -#[cfg(feature = "backend-rwkv")] -/// Load an RWKV7 model from `.safetensors` path. -pub fn load_rwkv7_model_from_path(path: &str) -> Arc { - rwkvzip::Compressor::load_model(path).expect("failed to load RWKV7 model") -} - -#[cfg(feature = "backend-mamba")] -/// Load a Mamba-1 model from `.safetensors` path. -pub fn load_mamba_model_from_path(path: &str) -> Arc { - mambazip::Compressor::load_model(path).expect("failed to load Mamba model") -} - -#[inline(always)] -fn aligned_prefix<'a>(x: &'a [u8], y: &'a [u8]) -> (&'a [u8], &'a [u8]) { - let n = x.len().min(y.len()); - (&x[..n], &y[..n]) -} - -#[cfg(feature = "backend-zpaq")] -#[inline(always)] -fn zpaq_compress_size_bytes(data: &[u8], method: &str) -> u64 { - zpaq_rs::compress_size(data, method).unwrap_or(0) -} - -#[cfg(not(feature = "backend-zpaq"))] -#[inline(always)] -fn zpaq_compress_size_bytes(_data: &[u8], _method: &str) -> u64 { - panic!("CompressionBackend::Zpaq is unavailable: build with feature 'backend-zpaq'") -} - -#[cfg(feature = "backend-zpaq")] -#[inline(always)] -fn zpaq_compress_size_parallel_bytes(data: &[u8], method: &str, threads: usize) -> u64 { - zpaq_rs::compress_size_parallel(data, method, threads).unwrap_or(0) -} - -#[cfg(not(feature = "backend-zpaq"))] -#[inline(always)] -fn zpaq_compress_size_parallel_bytes(_data: &[u8], _method: &str, _threads: usize) -> u64 { - panic!("CompressionBackend::Zpaq is unavailable: build with feature 'backend-zpaq'") -} - -#[cfg(feature = "backend-zpaq")] -#[inline(always)] -fn zpaq_compress_size_stream(reader: R, method: &str) -> u64 { - zpaq_rs::compress_size_stream(reader, method, None, None).unwrap_or(0) -} - -#[cfg(not(feature = "backend-zpaq"))] -#[inline(always)] -fn zpaq_compress_size_stream(_reader: R, _method: &str) -> u64 { - panic!("CompressionBackend::Zpaq is unavailable: build with feature 'backend-zpaq'") -} - -#[cfg(feature = "backend-zpaq")] -#[inline(always)] -fn zpaq_compress_to_vec(data: &[u8], method: &str) -> anyhow::Result> { - Ok(zpaq_rs::compress_to_vec(data, method)?) -} - -#[cfg(not(feature = "backend-zpaq"))] -#[inline(always)] -fn zpaq_compress_to_vec(_data: &[u8], _method: &str) -> anyhow::Result> { - anyhow::bail!("zpaq backend disabled at compile time (enable feature 'backend-zpaq')") -} - -#[cfg(feature = "backend-zpaq")] -#[inline(always)] -fn zpaq_decompress_to_vec(data: &[u8]) -> anyhow::Result> { - Ok(zpaq_rs::decompress_to_vec(data)?) -} - -#[cfg(not(feature = "backend-zpaq"))] -#[inline(always)] -fn zpaq_decompress_to_vec(_data: &[u8]) -> anyhow::Result> { - anyhow::bail!("zpaq backend disabled at compile time (enable feature 'backend-zpaq')") -} - -/// ------- Base Compression Functions ------- -#[inline(always)] -pub fn get_compressed_size(path: &str, method: &str) -> u64 { - // Convert Input file to Vec, and reference that (compress_size only takes &[u8] input), and pass method. - // Will panic if file does not exist, so it must be prevalidated. - zpaq_compress_size_bytes(&std::fs::read(path).unwrap(), method) -} - -/// Validate that a ZPAQ method string is supported for rate estimation. -pub fn validate_zpaq_rate_method(method: &str) -> Result<(), String> { - #[cfg(feature = "backend-zpaq")] - { - zpaq_rate::validate_zpaq_rate_method(method) - } - #[cfg(not(feature = "backend-zpaq"))] - { - let _ = method; - Err("zpaq backend disabled at compile time".to_string()) - } -} - -#[cfg(feature = "backend-rwkv")] -fn with_rwkv_tls( - model: &Arc, - f: impl FnOnce(&mut rwkvzip::Compressor) -> R, -) -> R { - let key = Arc::as_ptr(model) as usize; - RWKV_TLS.with(|cell| { - let mut map = cell.borrow_mut(); - let comp = map - .entry(key) - .or_insert_with(|| rwkvzip::Compressor::new_from_model(model.clone())); - f(comp) - }) -} - -#[cfg(feature = "backend-rwkv")] -fn with_rwkv_method_tls(method: &str, f: impl FnOnce(&mut rwkvzip::Compressor) -> R) -> R { - RWKV_METHOD_TLS.with(|cell| { - let mut map = cell.borrow_mut(); - // Keep a per-method template compressor for fast cloning while ensuring - // each call gets isolated mutable runtime state (no cross-call leakage). - let mut comp = if let Some(template) = map.get(method) { - template.clone() - } else { - let template = rwkvzip::Compressor::new_from_method(method).unwrap_or_else(|e| { - panic!("invalid rwkv method '{method}': {e:#}"); - }); - map.insert(method.to_string(), template.clone()); - template - }; - drop(map); - f(&mut comp) - }) -} - -#[cfg(feature = "backend-rwkv")] -fn with_rwkv_rate_tls( - model: &Arc, - f: impl FnOnce(&mut rwkvzip::Compressor) -> R, -) -> R { - let key = Arc::as_ptr(model) as usize; - RWKV_RATE_TLS.with(|cell| { - let mut map = cell.borrow_mut(); - let mut comp = if let Some(template) = map.get(&key) { - template.clone() - } else { - let template = rwkvzip::Compressor::new_from_model(model.clone()); - map.insert(key, template.clone()); - template - }; - drop(map); - f(&mut comp) - }) -} - -#[cfg(feature = "backend-mamba")] -fn with_mamba_tls( - model: &Arc, - f: impl FnOnce(&mut mambazip::Compressor) -> R, -) -> R { - let key = Arc::as_ptr(model) as usize; - MAMBA_TLS.with(|cell| { - let mut map = cell.borrow_mut(); - let comp = map - .entry(key) - .or_insert_with(|| mambazip::Compressor::new_from_model(model.clone())); - f(comp) - }) -} - -#[cfg(feature = "backend-mamba")] -fn with_mamba_rate_tls( - model: &Arc, - f: impl FnOnce(&mut mambazip::Compressor) -> R, -) -> R { - let key = Arc::as_ptr(model) as usize; - MAMBA_RATE_TLS.with(|cell| { - let mut map = cell.borrow_mut(); - let mut comp = if let Some(template) = map.get(&key) { - template.clone() - } else { - let template = mambazip::Compressor::new_from_model(model.clone()); - map.insert(key, template.clone()); - template - }; - drop(map); - f(&mut comp) - }) -} - -#[cfg(feature = "backend-mamba")] -fn with_mamba_method_tls(method: &str, f: impl FnOnce(&mut mambazip::Compressor) -> R) -> R { - MAMBA_METHOD_TLS.with(|cell| { - let mut map = cell.borrow_mut(); - let mut comp = if let Some(template) = map.get(method) { - template.clone() - } else { - let template = mambazip::Compressor::new_from_method(method).unwrap_or_else(|e| { - panic!("invalid mamba method '{method}': {e:#}"); - }); - map.insert(method.to_string(), template.clone()); - template - }; - drop(map); - f(&mut comp) - }) -} - -struct SliceChainReader<'a> { - parts: &'a [&'a [u8]], - i: usize, - off: usize, -} - -impl<'a> SliceChainReader<'a> { - fn new(parts: &'a [&'a [u8]]) -> Self { - Self { - parts, - i: 0, - off: 0, - } - } -} - -impl<'a> std::io::Read for SliceChainReader<'a> { - fn read(&mut self, mut buf: &mut [u8]) -> std::io::Result { - let mut total = 0; - if buf.is_empty() { - return Ok(0); - } - while self.i < self.parts.len() { - let p = self.parts[self.i]; - if self.off >= p.len() { - self.i += 1; - self.off = 0; - continue; - } - let n = (p.len() - self.off).min(buf.len()); - // Safe copy slice - buf[..n].copy_from_slice(&p[self.off..self.off + n]); - - // Advance state - self.off += n; - total += n; - - // Re-slice buf to fill remainder - let tmp = buf; - buf = &mut tmp[n..]; - - if buf.is_empty() { - break; - } - } - Ok(total) - } -} - -/// Compute compressed size of a chain of byte slices with a selected compression backend. -pub fn compress_size_chain_backend(parts: &[&[u8]], backend: &CompressionBackend) -> u64 { - match backend { - CompressionBackend::Zpaq { method } => { - let r = SliceChainReader::new(parts); - zpaq_compress_size_stream(r, method.as_str()) - } - #[cfg(feature = "backend-rwkv")] - CompressionBackend::Rwkv7 { model, coder } => { - with_rwkv_tls(model, |c| c.compress_size_chain(parts, *coder).unwrap_or(0)) - } - CompressionBackend::Rate { - rate_backend, - coder, - framing, - } => { - crate::compression::compress_rate_size_chain(parts, rate_backend, -1, *coder, *framing) - .unwrap_or(0) - } - } -} - -/// Compute compressed size of a single byte slice with a selected compression backend. -pub fn compress_size_backend(data: &[u8], backend: &CompressionBackend) -> u64 { - match backend { - CompressionBackend::Zpaq { method } => zpaq_compress_size_bytes(data, method.as_str()), - #[cfg(feature = "backend-rwkv")] - CompressionBackend::Rwkv7 { model, coder } => { - with_rwkv_tls(model, |c| c.compress_size(data, *coder).unwrap_or(0)) - } - CompressionBackend::Rate { - rate_backend, - coder, - framing, - } => crate::compression::compress_rate_size(data, rate_backend, -1, *coder, *framing) - .unwrap_or(0), - } -} - -/// Compress bytes with a selected compression backend. -pub fn compress_bytes_backend( - data: &[u8], - backend: &CompressionBackend, -) -> anyhow::Result> { - match backend { - CompressionBackend::Zpaq { method } => zpaq_compress_to_vec(data, method), - #[cfg(feature = "backend-rwkv")] - CompressionBackend::Rwkv7 { model, coder } => { - with_rwkv_tls(model, |c| c.compress(data, *coder)) - } - CompressionBackend::Rate { - rate_backend, - coder, - framing, - } => crate::compression::compress_rate_bytes(data, rate_backend, -1, *coder, *framing), - } -} - -/// Decompress bytes with a selected compression backend. -pub fn decompress_bytes_backend( - input: &[u8], - backend: &CompressionBackend, -) -> anyhow::Result> { - match backend { - CompressionBackend::Zpaq { .. } => zpaq_decompress_to_vec(input), - #[cfg(feature = "backend-rwkv")] - CompressionBackend::Rwkv7 { model, .. } => with_rwkv_tls(model, |c| c.decompress(input)), - CompressionBackend::Rate { - rate_backend, - coder, - framing, - } => crate::compression::decompress_rate_bytes(input, rate_backend, -1, *coder, *framing), - } -} - -fn prequential_rate_backend( - data: &[u8], - prefix_parts: &[&[u8]], - max_order: i64, - backend: &RateBackend, -) -> f64 { - use crate::mixture::OnlineBytePredictor; - - if data.is_empty() { - return 0.0; - } - let total = prefix_parts - .iter() - .map(|p| p.len() as u64) - .sum::() - .saturating_add(data.len() as u64); - let mut predictor = crate::mixture::RateBackendPredictor::from_backend( - backend.clone(), - max_order, - crate::mixture::DEFAULT_MIN_PROB, - ); - predictor - .begin_stream(Some(total)) - .unwrap_or_else(|e| panic!("rate backend stream init failed: {e}")); - for prefix in prefix_parts { - for &b in *prefix { - predictor.update(b); - } - } - let mut bits = 0.0; - for &b in data { - bits -= predictor.log_prob(b) / std::f64::consts::LN_2; - predictor.update(b); - } - predictor - .finish_stream() - .unwrap_or_else(|e| panic!("rate backend stream finalize failed: {e}")); - bits / (data.len() as f64) -} - -fn frozen_plugin_rate_backend( - score_data: &[u8], - fit_parts: &[&[u8]], - max_order: i64, - backend: &RateBackend, -) -> f64 { - if score_data.is_empty() { - return 0.0; - } - if matches!(backend, RateBackend::RosaPlus) { - let mut model = rosaplus::RosaPlus::new(max_order, false, 0, 42); - for part in fit_parts { - model.train_example(part); - } - model.build_lm(); - return model.cross_entropy(score_data); - } - #[cfg(feature = "backend-rwkv")] - match backend { - RateBackend::Rwkv7 { model } => { - return with_rwkv_rate_tls(model, |c| { - c.cross_entropy_frozen_plugin_chain(fit_parts, score_data) - .unwrap_or_else(|e| panic!("rwkv frozen-plugin scoring failed: {e:#}")) - }); - } - RateBackend::Rwkv7Method { method } => { - return with_rwkv_method_tls(method, |c| { - c.cross_entropy_frozen_plugin_chain(fit_parts, score_data) - .unwrap_or_else(|e| panic!("rwkv method frozen-plugin scoring failed: {e:#}")) - }); - } - _ => {} - } - #[cfg(feature = "backend-mamba")] - match backend { - RateBackend::Mamba { model } => { - return with_mamba_rate_tls(model, |c| { - c.cross_entropy_frozen_plugin_chain(fit_parts, score_data) - .unwrap_or_else(|e| panic!("mamba frozen-plugin scoring failed: {e:#}")) - }); - } - RateBackend::MambaMethod { method } => { - return with_mamba_method_tls(method, |c| { - c.cross_entropy_frozen_plugin_chain(fit_parts, score_data) - .unwrap_or_else(|e| panic!("mamba method frozen-plugin scoring failed: {e:#}")) - }); - } - _ => {} - } - - use crate::mixture::OnlineBytePredictor; - - let fit_total = fit_parts.iter().map(|part| part.len() as u64).sum::(); - let mut predictor = crate::mixture::RateBackendPredictor::from_backend( - backend.clone(), - max_order, - crate::mixture::DEFAULT_MIN_PROB, - ); - predictor - .begin_stream(Some(fit_total)) - .unwrap_or_else(|e| panic!("rate backend fit-pass init failed: {e}")); - for part in fit_parts { - for &byte in *part { - predictor.update(byte); - } - } - predictor - .finish_stream() - .unwrap_or_else(|e| panic!("rate backend fit-pass finalize failed: {e}")); - predictor - .reset_frozen(Some(score_data.len() as u64)) - .unwrap_or_else(|e| panic!("rate backend frozen-score reset failed: {e}")); - let mut bits = 0.0; - for &byte in score_data { - bits -= predictor.log_prob(byte) / std::f64::consts::LN_2; - predictor.update_frozen(byte); - } - predictor - .finish_stream() - .unwrap_or_else(|e| panic!("rate backend frozen-score finalize failed: {e}")); - bits / (score_data.len() as f64) -} - -#[inline(always)] -fn argmax_log_prob_byte(logps: &[f64; 256]) -> u8 { - let mut best_idx = 0usize; - let mut best = f64::NEG_INFINITY; - for (idx, &logp) in logps.iter().enumerate() { - let score = if logp.is_finite() { - logp - } else { - f64::NEG_INFINITY - }; - if score > best { - best = score; - best_idx = idx; - } - } - best_idx as u8 -} - -fn pick_generated_byte( - logps: &[f64; 256], - config: GenerationConfig, - rng: &mut GenerationRng, -) -> u8 { - if matches!(config.strategy, GenerationStrategy::Greedy) - || !config.temperature.is_finite() - || config.temperature <= 0.0 - { - return argmax_log_prob_byte(logps); - } - - let mut entries = [(0u8, f64::NEG_INFINITY); 256]; - for (idx, &logp) in logps.iter().enumerate() { - let scaled = if logp.is_finite() { - logp / config.temperature - } else { - f64::NEG_INFINITY - }; - entries[idx] = (idx as u8, scaled); - } - entries.sort_by(|a, b| b.1.total_cmp(&a.1)); - - let keep_k = if config.top_k == 0 { - entries.len() - } else { - config.top_k.min(entries.len()) - }; - - let top_p = if config.top_p.is_finite() { - config.top_p.clamp(0.0, 1.0) - } else { - 1.0 - }; - - let mut max_logp = f64::NEG_INFINITY; - for &(_, logp) in entries.iter().take(keep_k) { - if logp.is_finite() { - max_logp = max_logp.max(logp); - } - } - if !max_logp.is_finite() { - return argmax_log_prob_byte(logps); - } - - let mut weights = [(0u8, 0.0f64); 256]; - let mut total = 0.0; - for (idx, &(byte, logp)) in entries.iter().take(keep_k).enumerate() { - let w = if logp.is_finite() { - (logp - max_logp).exp() - } else { - 0.0 - }; - weights[idx] = (byte, w); - total += w; - } - if !(total.is_finite()) || total <= 0.0 { - return argmax_log_prob_byte(logps); - } - - let cutoff_count = if top_p >= 1.0 { - keep_k - } else { - let mut cumulative = 0.0; - let mut keep = 0usize; - for &(_, w) in weights.iter().take(keep_k) { - cumulative += w / total; - keep += 1; - if cumulative >= top_p { - break; - } - } - keep.max(1) - }; - - let mut truncated_total = 0.0; - for &(_, w) in weights.iter().take(cutoff_count) { - truncated_total += w; - } - if !(truncated_total.is_finite()) || truncated_total <= 0.0 { - return argmax_log_prob_byte(logps); - } - - let target = rng.next_f64() * truncated_total; - let mut cumulative = 0.0; - let mut picked = weights[0].0; - for &(byte, weight) in weights.iter().take(cutoff_count) { - cumulative += weight; - if cumulative >= target { - picked = byte; - break; - } - } - picked -} - -fn generate_rate_backend_chain( - prefix_parts: &[&[u8]], - bytes: usize, - max_order: i64, - backend: &RateBackend, - config: GenerationConfig, -) -> Vec { - if bytes == 0 { - return Vec::new(); - } - - let total = prefix_parts - .iter() - .map(|p| p.len() as u64) - .sum::() - .saturating_add(bytes as u64); - let mut session = RateBackendSession::from_backend(backend.clone(), max_order, Some(total)) - .unwrap_or_else(|e| panic!("rate backend generation init failed: {e}")); - for &part in prefix_parts { - session.observe(part); - } - let out = session.generate_bytes(bytes, config); - session - .finish() - .unwrap_or_else(|e| panic!("rate backend generation finalize failed: {e}")); - out -} - -/// Estimate entropy rate of `data` using the explicit rate `backend`. -pub fn entropy_rate_backend(data: &[u8], max_order: i64, backend: &RateBackend) -> f64 { - match backend { - RateBackend::RosaPlus => { - let mut m = rosaplus::RosaPlus::new(max_order, false, 0, 42); - m.predictive_entropy_rate(data) - } - RateBackend::Match { .. } - | RateBackend::SparseMatch { .. } - | RateBackend::Ppmd { .. } - | RateBackend::Sequitur { .. } - | RateBackend::Calibrated { .. } => prequential_rate_backend(data, &[], max_order, backend), - #[cfg(feature = "backend-rwkv")] - RateBackend::Rwkv7 { model } => with_rwkv_tls(model, |c| { - c.cross_entropy(data) - .unwrap_or_else(|e| panic!("rwkv entropy scoring failed: {e:#}")) - }), - #[cfg(feature = "backend-rwkv")] - RateBackend::Rwkv7Method { method } => with_rwkv_method_tls(method, |c| { - c.cross_entropy(data) - .unwrap_or_else(|e| panic!("rwkv method entropy scoring failed: {e:#}")) - }), - #[cfg(feature = "backend-mamba")] - RateBackend::Mamba { model } => with_mamba_tls(model, |c| { - c.cross_entropy(data) - .unwrap_or_else(|e| panic!("mamba entropy scoring failed: {e:#}")) - }), - #[cfg(feature = "backend-mamba")] - RateBackend::MambaMethod { method } => with_mamba_method_tls(method, |c| { - c.cross_entropy(data) - .unwrap_or_else(|e| panic!("mamba method entropy scoring failed: {e:#}")) - }), - RateBackend::Zpaq { method } => { - if data.is_empty() { - return 0.0; - } - let mut model = crate::zpaq_rate::ZpaqRateModel::new(method.clone(), 2f64.powi(-24)); - let bits = model.update_and_score(data); - bits / (data.len() as f64) - } - RateBackend::Mixture { spec } => { - if data.is_empty() { - return 0.0; - } - let experts = spec.build_experts(); - let mut mix = crate::mixture::build_mixture_runtime(spec.as_ref(), &experts) - .unwrap_or_else(|e| panic!("MixtureSpec invalid: {e}")); - mix.begin_stream(Some(data.len() as u64)) - .unwrap_or_else(|e| panic!("Mixture stream init failed: {e}")); - let mut bits = 0.0; - for &b in data { - bits -= mix.step(b) / std::f64::consts::LN_2; - } - mix.finish_stream() - .unwrap_or_else(|e| panic!("Mixture stream finalize failed: {e}")); - bits / (data.len() as f64) - } - RateBackend::Particle { spec } => { - if data.is_empty() { - return 0.0; - } - let mut runtime = crate::particle::ParticleRuntime::new(spec.as_ref()); - let mut bits = 0.0; - for &b in data { - bits -= runtime.step(b) / std::f64::consts::LN_2; - } - bits / (data.len() as f64) - } - RateBackend::Ctw { depth } => { - if data.is_empty() { - return 0.0; - } - // Byte-wise CTW: factorize by bit position so deterministic bits don't leak entropy. - let mut fac = crate::ctw::FacContextTree::new(*depth, 8); - fac.reserve_for_symbols(data.len()); - for &b in data { - fac.update_byte_msb(b); - } - let ln_p = fac.get_log_block_probability(); - let bits = -ln_p / std::f64::consts::LN_2; - bits / (data.len() as f64) - } - RateBackend::FacCtw { - base_depth, - num_percept_bits: _, - encoding_bits, - } => { - if data.is_empty() { - return 0.0; - } - let bits_per_byte = (*encoding_bits).clamp(1, 8); - let mut fac = crate::ctw::FacContextTree::new(*base_depth, bits_per_byte); - fac.reserve_for_symbols(data.len()); - for &b in data { - fac.update_byte_lsb(b); - } - let ln_p = fac.get_log_block_probability(); - let bits = -ln_p / std::f64::consts::LN_2; - bits / (data.len() as f64) - } - } -} - -/// Estimate biased/plugin entropy rate of `data` using the explicit rate `backend`. -pub fn biased_entropy_rate_backend(data: &[u8], max_order: i64, backend: &RateBackend) -> f64 { - match backend { - RateBackend::Zpaq { .. } => { - panic!("biased/plugin entropy is not supported for zpaq rate backends in 1.1.1") - } - _ => frozen_plugin_rate_backend(data, &[data], max_order, backend), - } -} - -/// Cross-entropy H_{train}(test) - score test_data under model trained on train_data. -pub fn cross_entropy_rate_backend( - test_data: &[u8], - train_data: &[u8], - max_order: i64, - backend: &RateBackend, -) -> f64 { - match backend { - RateBackend::Zpaq { method } => { - if test_data.is_empty() { - return 0.0; - } - let mut model = crate::zpaq_rate::ZpaqRateModel::new(method.clone(), 2f64.powi(-24)); - model.update_and_score(train_data); - let bits = model.update_and_score(test_data); - bits / (test_data.len() as f64) - } - _ => frozen_plugin_rate_backend(test_data, &[train_data], max_order, backend), - } -} - -/// Estimate joint entropy rate `H(X,Y)` using an explicit `backend`. -pub fn joint_entropy_rate_backend( - x: &[u8], - y: &[u8], - max_order: i64, - backend: &RateBackend, -) -> f64 { - let (x, y) = aligned_prefix(x, y); - if x.is_empty() { - return 0.0; - } - match backend { - RateBackend::RosaPlus => { - let joint_symbols: Vec = (0..x.len()) - .map(|i| (x[i] as u32) * 256 + (y[i] as u32)) - .collect(); - let mut m = rosaplus::RosaPlus::new(max_order, false, 0, 42); - m.entropy_rate_cps(&joint_symbols) - } - RateBackend::Match { .. } - | RateBackend::SparseMatch { .. } - | RateBackend::Ppmd { .. } - | RateBackend::Sequitur { .. } - | RateBackend::Calibrated { .. } => { - let mut joint = Vec::with_capacity(x.len() * 2); - for (&xb, &yb) in x.iter().zip(y.iter()) { - joint.push(xb); - joint.push(yb); - } - entropy_rate_backend(&joint, max_order, backend) * 2.0 - } - #[cfg(feature = "backend-rwkv")] - RateBackend::Rwkv7 { model } => with_rwkv_tls(model, |c| { - c.joint_cross_entropy_aligned_min(x, y) - .unwrap_or_else(|e| panic!("rwkv joint-entropy scoring failed: {e:#}")) - }), - #[cfg(feature = "backend-rwkv")] - RateBackend::Rwkv7Method { method } => with_rwkv_method_tls(method, |c| { - c.joint_cross_entropy_aligned_min(x, y) - .unwrap_or_else(|e| panic!("rwkv method joint-entropy scoring failed: {e:#}")) - }), - #[cfg(feature = "backend-mamba")] - RateBackend::Mamba { model } => with_mamba_tls(model, |c| { - c.joint_cross_entropy_aligned_min(x, y) - .unwrap_or_else(|e| panic!("mamba joint-entropy scoring failed: {e:#}")) - }), - #[cfg(feature = "backend-mamba")] - RateBackend::MambaMethod { method } => with_mamba_method_tls(method, |c| { - c.joint_cross_entropy_aligned_min(x, y) - .unwrap_or_else(|e| panic!("mamba method joint-entropy scoring failed: {e:#}")) - }), - RateBackend::Zpaq { method } => { - let mut joint = Vec::with_capacity(x.len() * 2); - for (&xb, &yb) in x.iter().zip(y.iter()) { - joint.push(xb); - joint.push(yb); - } - let mut model = crate::zpaq_rate::ZpaqRateModel::new(method.clone(), 2f64.powi(-24)); - let bits = model.update_and_score(&joint); - bits / (x.len() as f64) - } - RateBackend::Mixture { spec } => { - let mut joint = Vec::with_capacity(x.len() * 2); - for (&xb, &yb) in x.iter().zip(y.iter()) { - joint.push(xb); - joint.push(yb); - } - let experts = spec.build_experts(); - let mut mix = crate::mixture::build_mixture_runtime(spec.as_ref(), &experts) - .unwrap_or_else(|e| panic!("MixtureSpec invalid: {e}")); - mix.begin_stream(Some(joint.len() as u64)) - .unwrap_or_else(|e| panic!("Mixture stream init failed: {e}")); - let mut bits = 0.0; - for &b in &joint { - bits -= mix.step(b) / std::f64::consts::LN_2; - } - mix.finish_stream() - .unwrap_or_else(|e| panic!("Mixture stream finalize failed: {e}")); - bits / (x.len() as f64) - } - RateBackend::Particle { spec } => { - let mut joint = Vec::with_capacity(x.len() * 2); - for (&xb, &yb) in x.iter().zip(y.iter()) { - joint.push(xb); - joint.push(yb); - } - let mut runtime = crate::particle::ParticleRuntime::new(spec.as_ref()); - let mut bits = 0.0; - for &b in &joint { - bits -= runtime.step(b) / std::f64::consts::LN_2; - } - bits / (x.len() as f64) - } - RateBackend::Ctw { depth } => { - // NOTE: CTW interleaves bits: x_0, y_0, x_1, y_1... - // This estimates the joint entropy H(X,Y) by modeling the sequence - // of alternating bits. This is a fine-grained joint model but - // theoretically consistent for estimating joint entropy rate. - // ROSA uses 16-bit joint symbols (x << 8 | y). Both are valid. - let mut fac = crate::ctw::FacContextTree::new(*depth, 16); - for k in 0..x.len() { - let bx = x[k]; - let by = y[k]; - for bit_idx in 0..8 { - let bit_x = ((bx >> (7 - bit_idx)) & 1) == 1; - let bit_y = ((by >> (7 - bit_idx)) & 1) == 1; - fac.update(bit_x, bit_idx); - fac.update(bit_y, bit_idx + 8); - } - } - let ln_p = fac.get_log_block_probability(); - let bits = -ln_p / std::f64::consts::LN_2; - bits / (x.len() as f64) - } - RateBackend::FacCtw { - base_depth, - num_percept_bits: _, - encoding_bits, - } => { - // Joint: interleave x and y bits, use 2*encoding_bits trees - let bits_per_byte = (*encoding_bits).clamp(1, 8); - let mut fac = crate::ctw::FacContextTree::new(*base_depth, bits_per_byte * 2); - for k in 0..x.len() { - let bx = x[k]; - let by = y[k]; - for i in 0..bits_per_byte { - // Tree structure: - // bits_per_byte trees for X, bits_per_byte trees for Y. - // But we interleave them in the "joint" sense. - // Here we map bit i of X to tree 2*i, bit i of Y to tree 2*i + 1 - let bit_idx_x = i * 2; - let bit_idx_y = bit_idx_x + 1; - fac.update(((bx >> i) & 1) == 1, bit_idx_x); - fac.update(((by >> i) & 1) == 1, bit_idx_y); - } - } - let ln_p = fac.get_log_block_probability(); - let bits = -ln_p / std::f64::consts::LN_2; - bits / (x.len() as f64) - } - } -} -#[inline(always)] -/// Compute compressed size for a file path with an explicit ZPAQ thread count. -pub fn get_compressed_size_parallel(path: &str, method: &str, threads: usize) -> u64 { - // Convert Input file to Vec, and reference that (compress_size only takes &[u8] input), and pass method. - // Will panic if file does not exist, so it must be prevalidated. - zpaq_compress_size_parallel_bytes(&std::fs::read(path).unwrap(), method, threads) -} - -#[inline(always)] -/// Read all files in `paths` in parallel and return their byte contents. -pub fn get_bytes_from_paths(paths: &[&str]) -> Vec> { - paths - .par_iter() - .map(|path| std::fs::read(*path).expect("failed to read file")) - .collect() -} - -/// ------- Bulk File Compression Functions ------- -#[inline(always)] -pub fn get_sequential_compressed_sizes_from_sequential_paths( - paths: &[&str], - method: &str, -) -> Vec { - // This will, in parallel load all files into memory, THEN in parallel compress each one, each with one thread. - // Use when File IO is the bottleneck - // Only uses ONE ZPAQ THREAD. - // For VERY large n (relative to threads) with small files (relative to memory) this may be useful. - get_bytes_from_paths(paths) - .par_iter() - .map(|data| zpaq_compress_size_bytes(data, method)) - .collect() -} - -#[inline(always)] -/// Compress all paths after preloading bytes, using per-file parallel ZPAQ compression. -pub fn get_parallel_compressed_sizes_from_sequential_paths( - paths: &[&str], - method: &str, - threads: usize, -) -> Vec { - // This will, in parallel load all files into memory, THEN in parallel compress each one, with THREADS. (for each file, the thread count is THREADS) - // Use when File IO is the bottleneck. - // Balanced parallelization between RAYON_NUM_THREADS and ZPAQ `THREADS` const. For when total dataset will fit in memory. - get_bytes_from_paths(paths) - .par_iter() - .map(|data| zpaq_compress_size_parallel_bytes(data, method, threads)) - .collect() -} - -#[inline(always)] -/// Compress all paths directly from disk using single-thread ZPAQ per file. -pub fn get_sequential_compressed_sizes_from_parallel_paths( - paths: &[&str], - method: &str, -) -> Vec { - // This will, in parallel, for each file, read it from disk and compress it with one thread. (one file, one thread) - // Use when File IO is not the bottleneck. Lower memory usage. (does not preload dataset) - // Only uses ONE ZPAQ THREAD. For VERY large n(relative to threads) with large files(relative to memory) this may be useful. - paths - .par_iter() - .map(|path| get_compressed_size(path, method)) - .collect() -} - -#[inline(always)] -/// Compress all paths directly from disk using per-file multi-thread ZPAQ. -pub fn get_parallel_compressed_sizes_from_parallel_paths( - paths: &[&str], - method: &str, - threads: usize, -) -> Vec { - // This will, in parallel, for each file, read it from disk and compress it with THREADS. (for each file, the thread count is THREADS) - // Use when File IO is not the bottleneck. Lower memory usage. (does not preload dataset) - // For large n(relative to threads) with VERY large files(relative to memory) this may be useful. - // This will reflect RAYON_NUM_THREADS and THREAD const values. - paths - .par_iter() - .map(|path| get_compressed_size_parallel(path, method, threads)) - .collect() -} - -/// Optimizes parallelization -#[inline(always)] -pub fn get_compressed_sizes_from_paths(paths: &[&str], method: &str) -> Vec { - let n: usize = paths.len(); - let num_threads: usize = *NUM_THREADS.get_or_init(num_cpus::get); - if n < num_threads { - get_parallel_compressed_sizes_from_parallel_paths(paths, method, num_threads.div_ceil(n)) - } else { - get_sequential_compressed_sizes_from_parallel_paths(paths, method) - } -} - -/// ------- NCD (Normalized Compression Distance) ------ -/// -/// NCD is a parameter-free similarity metric based on Kolmogorov complexity. -/// Since Kolmogorov complexity `K(x)` is uncomputable, we approximate it using -/// the compressed size `C(x)` provided by a real-world compressor (here, ZPAQ). -/// -/// The general form is: -/// `NCD(x,y) = (C(xy) - min(C(x), C(y))) / max(C(x), C(y))` -/// -/// Different variants handle normalization and symmetry differently. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum NcdVariant { - /// Standard Vitanyi NCD: - /// `NCD(x,y) = (C(xy) - min(C(x), C(y))) / max(C(x), C(y))` - /// Note: `C(xy)` denotes compressing the concatenation of x and y. - Vitanyi, - /// Symmetric Vitanyi NCD: - /// `NCD_sym(x,y) = (min(C(xy), C(yx)) - min(C(x), C(y))) / max(C(x), C(y))` - /// Takes the best compression of `xy` or `yx` to ensure symmetry even if the compressor is not symmetric. - SymVitanyi, - /// Conservative NCD: - /// `NCD_cons(x,y) = (C(xy) - min(C(x), C(y))) / C(xy)` - /// Normalizes by the joint compressed size instead of the max marginal. - Cons, - /// Symmetric Conservative NCD: - /// `NCD_sym_cons(x,y) = (min(C(xy), C(yx)) - min(C(x), C(y))) / min(C(xy), C(yx))` - SymCons, -} - -#[inline(always)] -fn compress_size_bytes(data: &[u8], method: &str) -> u64 { - zpaq_compress_size_bytes(data, method) -} - -#[inline(always)] -fn ncd_from_sizes(cx: u64, cy: u64, cxy: u64, cyx: Option, variant: NcdVariant) -> f64 { - let min_c = cx.min(cy) as f64; - let max_c = cx.max(cy) as f64; - - match variant { - NcdVariant::Vitanyi => { - if max_c == 0.0 { - 0.0 - } else { - (cxy as f64 - min_c) / max_c - } - } - NcdVariant::SymVitanyi => { - let m = cxy.min(cyx.expect("cyx required for SymVitanyi")) as f64; - if max_c == 0.0 { - 0.0 - } else { - (m - min_c) / max_c - } - } - NcdVariant::Cons => { - let denom = cxy as f64; - if denom == 0.0 { - 0.0 - } else { - (cxy as f64 - min_c) / denom - } - } - NcdVariant::SymCons => { - let m = cxy.min(cyx.expect("cyx required for SymCons")) as f64; - if m == 0.0 { 0.0 } else { (m - min_c) / m } - } - } -} - -#[inline(always)] -/// Compute NCD for in-memory byte slices using the given ZPAQ `method` and `variant`. -pub fn ncd_bytes(x: &[u8], y: &[u8], method: &str, variant: NcdVariant) -> f64 { - let backend = CompressionBackend::Zpaq { - method: method.to_string(), - }; - ncd_bytes_backend(x, y, &backend, variant) -} - -/// NCD with bytes using the default context. -#[inline(always)] -pub fn ncd_bytes_default(x: &[u8], y: &[u8], variant: NcdVariant) -> f64 { - with_default_ctx(|ctx| ctx.ncd_bytes(x, y, variant)) -} - -/// Compute NCD for in-memory byte slices using an explicit compression `backend`. -pub fn ncd_bytes_backend( - x: &[u8], - y: &[u8], - backend: &CompressionBackend, - variant: NcdVariant, -) -> f64 { - let (cx, cy) = rayon::join( - || compress_size_backend(x, backend), - || compress_size_backend(y, backend), - ); - - let cxy = compress_size_chain_backend(&[x, y], backend); - - let cyx = match variant { - NcdVariant::SymVitanyi | NcdVariant::SymCons => { - Some(compress_size_chain_backend(&[y, x], backend)) - } - _ => None, - }; - - ncd_from_sizes(cx, cy, cxy, cyx, variant) -} - -#[inline(always)] -/// Compute NCD for two file paths using a ZPAQ `method` and `variant`. -pub fn ncd_paths(x: &str, y: &str, method: &str, variant: NcdVariant) -> f64 { - let (bx, by) = rayon::join( - || std::fs::read(x).expect("failed to read x"), - || std::fs::read(y).expect("failed to read y"), - ); - ncd_bytes(&bx, &by, method, variant) -} - -/// Compute NCD for two file paths using an explicit compression `backend`. -pub fn ncd_paths_backend( - x: &str, - y: &str, - backend: &CompressionBackend, - variant: NcdVariant, -) -> f64 { - let (bx, by) = rayon::join( - || std::fs::read(x).expect("failed to read x"), - || std::fs::read(y).expect("failed to read y"), - ); - ncd_bytes_backend(&bx, &by, backend, variant) -} - -/// Back-compat convenience wrappers (operate on file paths). -#[inline(always)] -pub fn ncd_vitanyi(x: &str, y: &str, method: &str) -> f64 { - ncd_paths(x, y, method, NcdVariant::Vitanyi) -} -#[inline(always)] -/// Convenience wrapper for symmetric-Vitanyi NCD on file paths. -pub fn ncd_sym_vitanyi(x: &str, y: &str, method: &str) -> f64 { - ncd_paths(x, y, method, NcdVariant::SymVitanyi) -} -#[inline(always)] -/// Convenience wrapper for conservative NCD on file paths. -pub fn ncd_cons(x: &str, y: &str, method: &str) -> f64 { - ncd_paths(x, y, method, NcdVariant::Cons) -} -#[inline(always)] -/// Convenience wrapper for symmetric-conservative NCD on file paths. -pub fn ncd_sym_cons(x: &str, y: &str, method: &str) -> f64 { - ncd_paths(x, y, method, NcdVariant::SymCons) -} - -/// Computes an NCD matrix (row-major, len = n*n) for in-memory byte blobs. -/// -/// Note: For symmetric variants, this computes each unordered pair once and writes both (i,j) and (j,i). -pub fn ncd_matrix_bytes(datas: &[Vec], method: &str, variant: NcdVariant) -> Vec { - let n = datas.len(); - let cx: Vec = datas - .par_iter() - .map(|d| compress_size_bytes(d, method)) - .collect(); - - let mut out = vec![0.0f64; n * n]; - let out_ptr = std::sync::atomic::AtomicPtr::new(out.as_mut_ptr()); - - match variant { - NcdVariant::SymVitanyi | NcdVariant::SymCons => { - (0..n) - .into_par_iter() - .flat_map_iter(|i| (i + 1..n).map(move |j| (i, j))) - .for_each_init(Vec::::new, |buf, (i, j)| { - let x = &datas[i]; - let y = &datas[j]; - - buf.clear(); - buf.reserve(x.len() + y.len()); - buf.extend_from_slice(x); - buf.extend_from_slice(y); - let cxy = compress_size_bytes(buf, method); - - buf.clear(); - buf.reserve(x.len() + y.len()); - buf.extend_from_slice(y); - buf.extend_from_slice(x); - let cyx = compress_size_bytes(buf, method); - - let d = ncd_from_sizes(cx[i], cx[j], cxy, Some(cyx), variant); - - // Safety: each (i,j) cell is written exactly once across all iterations. - let p = out_ptr.load(std::sync::atomic::Ordering::Relaxed); - unsafe { - *p.add(i * n + j) = d; - *p.add(j * n + i) = d; - } - }); - } - NcdVariant::Vitanyi | NcdVariant::Cons => { - (0..n) - .into_par_iter() - .for_each_init(Vec::::new, |buf, i| { - let x = &datas[i]; - for j in 0..n { - let d = if i == j { - 0.0 - } else { - let y = &datas[j]; - buf.clear(); - buf.reserve(x.len() + y.len()); - buf.extend_from_slice(x); - buf.extend_from_slice(y); - let cxy = compress_size_bytes(buf, method); - ncd_from_sizes(cx[i], cx[j], cxy, None, variant) - }; - - let p = out_ptr.load(std::sync::atomic::Ordering::Relaxed); - unsafe { - *p.add(i * n + j) = d; - } - } - }); - } - } - - out -} - -/// Computes an NCD matrix (row-major, len = n*n) for files (preloads all files into memory once). -pub fn ncd_matrix_paths(paths: &[&str], method: &str, variant: NcdVariant) -> Vec { - let datas = get_bytes_from_paths(paths); - ncd_matrix_bytes(&datas, method, variant) -} - -// ============================================================ -// Entropy-Based Distance Primitives (via ROSA) -// ============================================================ -// -// These use ROSA's Witten-Bell language model to estimate entropy -// and compute information-theoretic distances. - -/// Compute marginal (Shannon) entropy H(X) = −Σ p(x) log₂ p(x) in bits/symbol. -/// -/// This is the simple first-order entropy from the byte histogram, -/// NOT the context-conditional entropy rate from a language model. -#[inline(always)] -pub fn marginal_entropy_bytes(data: &[u8]) -> f64 { - if data.is_empty() { - return 0.0; - } - - let mut counts = [0u64; 256]; - for &b in data { - counts[b as usize] += 1; - } - - let n = data.len() as f64; - let mut h = 0.0f64; - for &count in &counts { - if count > 0 { - let p = count as f64 / n; - h -= p * p.log2(); - } - } - h -} - -/// Compute entropy rate `Ĥ(X)` in bits/symbol using ROSA LM. -/// -/// This uses ROSA's context-conditional Witten-Bell model to estimate -/// the entropy rate, which accounts for sequential dependencies. -/// -/// The estimator is **prequential** (predictive sequential): it sums the negative log-probability -/// of each symbol `x_t` given its past context `x_{ f64 { - with_default_ctx(|ctx| ctx.entropy_rate_bytes(data, max_order)) -} - -/// Compute biased entropy rate Ĥ_biased(X) bits per symbol. -/// -/// This uses the full plugin estimator (training on the whole text, then scoring the same text). -/// While biased as a source entropy estimate, it is mathematically consistent for -/// similarity metrics like Mutual Information and NED. -#[inline(always)] -pub fn biased_entropy_rate_bytes(data: &[u8], max_order: i64) -> f64 { - with_default_ctx(|ctx| ctx.biased_entropy_rate_bytes(data, max_order)) -} - -/// Compute joint marginal entropy H(X,Y) = −Σ p(x,y) log₂ p(x,y) in bits/symbol-pair. -/// -/// Uses a direct histogram of (x_i, y_i) pairs. This is the exact first-order -/// joint entropy, matching the spec.md definition. -#[inline(always)] -pub fn joint_marginal_entropy_bytes(x: &[u8], y: &[u8]) -> f64 { - let (x, y) = aligned_prefix(x, y); - let n = x.len(); - if n == 0 { - return 0.0; - } - - // Count pair occurrences using a HashMap for (x, y) pairs - // There are up to 65536 possible pairs, so we can use a flat array - let mut counts = vec![0u64; 256 * 256]; - for i in 0..n { - let pair_idx = (x[i] as usize) * 256 + (y[i] as usize); - counts[pair_idx] += 1; - } - - let n_f64 = n as f64; - let mut h = 0.0f64; - for &c in &counts { - if c > 0 { - let p = c as f64 / n_f64; - h -= p * p.log2(); - } - } - h -} - -/// Compute joint entropy rate `Ĥ(X,Y)`. -/// -/// Dispatches based on `max_order`: -/// - `max_order == 0`: Strictly aligned pair-symbol mapping (Marginal Joint Entropy). -/// Treats `(x_i, y_i)` as a single symbol in a product alphabet `Σ_X × Σ_Y`. -/// - `max_order != 0`: Shift-invariant algorithmic joint entropy approximated via ROSA. -/// Constructs a sequence of pair-symbols and estimates the entropy rate of that sequence. -/// -/// **Note**: This is an *aligned* joint entropy-rate estimate over time-indexed pairs -/// `(x_i, y_i)`. All joint-based quantities (`H(X)`, `H(Y)`, `H(X,Y)`, `I`, NED, NTE, etc.) -/// should be computed over the same aligned sample. -#[inline(always)] -pub fn joint_entropy_rate_bytes(x: &[u8], y: &[u8], max_order: i64) -> f64 { - with_default_ctx(|ctx| ctx.joint_entropy_rate_bytes(x, y, max_order)) -} - -/// Compute conditional entropy rate `Ĥ(X|Y)`. -/// -/// Dispatches based on `max_order`: -/// - `max_order == 0`: Strictly aligned `H(X,Y) - H(Y)` using marginals. -/// - `max_order != 0`: Chain rule definition `Ĥ(X|Y) = Ĥ(X,Y) - Ĥ(Y)`. -/// -/// Note: This relies on the identity `H(X|Y) = H(X,Y) - H(Y)`. -#[inline(always)] -pub fn conditional_entropy_rate_bytes(x: &[u8], y: &[u8], max_order: i64) -> f64 { - with_default_ctx(|ctx| ctx.conditional_entropy_rate_bytes(x, y, max_order)) -} - -/// Compute conditional entropy H(X|Y) = H(X,Y) − H(Y) -/// -/// Dispatches based on `max_order`. -#[inline(always)] -pub fn conditional_entropy_bytes(x: &[u8], y: &[u8], max_order: i64) -> f64 { - with_default_ctx(|ctx| ctx.conditional_entropy_bytes(x, y, max_order)) -} - -/// Compute mutual information `I(X;Y) = H(X) + H(Y) - H(X,Y)`. -/// -/// Dispatches based on `max_order`. If 0, uses marginals; else uses rates. -/// -/// `I(X;Y) = Σ p(x,y) log(p(x,y) / (p(x)p(y)))` -#[inline(always)] -pub fn mutual_information_bytes(x: &[u8], y: &[u8], max_order: i64) -> f64 { - with_default_ctx(|ctx| ctx.mutual_information_bytes(x, y, max_order)) -} - -/// Marginal Mutual Information (exact/histogram) -pub fn mutual_information_marg_bytes(x: &[u8], y: &[u8]) -> f64 { - let (x, y) = aligned_prefix(x, y); - let h_x = marginal_entropy_bytes(x); - let h_y = marginal_entropy_bytes(y); - let h_xy = joint_marginal_entropy_bytes(x, y); - (h_x + h_y - h_xy).max(0.0) -} - -/// Entropy Rate Mutual Information (ROSA predictive) -#[inline(always)] -pub fn mutual_information_rate_bytes(x: &[u8], y: &[u8], max_order: i64) -> f64 { - with_default_ctx(|ctx| ctx.mutual_information_rate_bytes(x, y, max_order)) -} - -// ====== NED: Normalized Entropy Distance ====== -// -// A metric distance based on the overlap of information between two variables. - -/// NED(X,Y) = (H(X,Y) - min(H(X), H(Y))) / max(H(X), H(Y)) -/// -/// Dispatches based on `max_order`. If 0, uses marginals; else uses rates. -/// -/// Range: [0, 1]. -/// * 0: Identity (X determines Y and Y determines X). -/// * 1: Independence (X and Y share no information). -#[inline(always)] -pub fn ned_bytes(x: &[u8], y: &[u8], max_order: i64) -> f64 { - with_default_ctx(|ctx| ctx.ned_bytes(x, y, max_order)) -} - -/// Marginal NED (exact/histogram) -pub fn ned_marg_bytes(x: &[u8], y: &[u8]) -> f64 { - let (x, y) = aligned_prefix(x, y); - let h_x = marginal_entropy_bytes(x); - let h_y = marginal_entropy_bytes(y); - let h_xy = joint_marginal_entropy_bytes(x, y); - let min_h = h_x.min(h_y); - let max_h = h_x.max(h_y); - if max_h == 0.0 { - 0.0 - } else { - ((h_xy - min_h) / max_h).clamp(0.0, 1.0) - } -} - -/// Normalized Entropy Distance (Rate-based) -#[inline(always)] -pub fn ned_rate_bytes(x: &[u8], y: &[u8], max_order: i64) -> f64 { - with_default_ctx(|ctx| ctx.ned_bytes(x, y, max_order)) -} - -/// NED_cons(X,Y) = (H(X,Y) - min(H(X), H(Y))) / H(X,Y) -/// -/// Conservative variant. Dispatches based on `max_order`. -#[inline(always)] -pub fn ned_cons_bytes(x: &[u8], y: &[u8], max_order: i64) -> f64 { - with_default_ctx(|ctx| ctx.ned_cons_bytes(x, y, max_order)) -} - -/// Conservative marginal NED using histogram entropy estimates. -pub fn ned_cons_marg_bytes(x: &[u8], y: &[u8]) -> f64 { - let h_x = marginal_entropy_bytes(x); - let h_y = marginal_entropy_bytes(y); - let h_xy = joint_marginal_entropy_bytes(x, y); - let min_h = h_x.min(h_y); - if h_xy == 0.0 { - 0.0 - } else { - ((h_xy - min_h) / h_xy).clamp(0.0, 1.0) - } -} - -#[inline(always)] -/// Conservative rate NED using the current default context backend. -pub fn ned_cons_rate_bytes(x: &[u8], y: &[u8], max_order: i64) -> f64 { - with_default_ctx(|ctx| ctx.ned_cons_bytes(x, y, max_order)) -} - -// ====== NTE: Normalized Transform Effort (Variation of Information) ====== - -/// NTE(X,Y) = VI(X,Y) / max(H(X), H(Y)) -/// where `VI(X,Y) = H(X|Y) + H(Y|X) = 2H(X,Y) - H(X) - H(Y)`. -/// -/// Represents the "effort" required to transform X into Y (and vice versa) relative -/// to their complexity. -/// -/// Note: VI can be as large as `H(X) + H(Y)`. If `H(X) ≈ H(Y)`, then VI can be `≈ 2 max(H(X), H(Y))`. -/// Thus, NTE is in [0, 2]. -/// * Values near 0 indicate near-identity. -/// * Values near 1+ indicate substantial effort/transform cost (e.g. independence). -/// -/// Dispatches based on `max_order`. -#[inline(always)] -pub fn nte_bytes(x: &[u8], y: &[u8], max_order: i64) -> f64 { - with_default_ctx(|ctx| ctx.nte_bytes(x, y, max_order)) -} - -/// Marginal NTE using histogram entropy estimates. -pub fn nte_marg_bytes(x: &[u8], y: &[u8]) -> f64 { - let (x, y) = aligned_prefix(x, y); - let h_x = marginal_entropy_bytes(x); - let h_y = marginal_entropy_bytes(y); - let h_xy = joint_marginal_entropy_bytes(x, y); - let vi = 2.0 * h_xy - h_x - h_y; - let max_h = h_x.max(h_y); - if max_h == 0.0 { - 0.0 - } else { - (vi / max_h).clamp(0.0, 2.0) - } -} - -#[inline(always)] -/// Rate NTE using the current default context backend. -pub fn nte_rate_bytes(x: &[u8], y: &[u8], max_order: i64) -> f64 { - with_default_ctx(|ctx| ctx.nte_bytes(x, y, max_order)) -} - -// ====== TVD: Total Variation Distance ====== - -/// Compute marginal byte histogram p(i) = count(i) / N for i ∈ [0, 255] -#[inline(always)] -fn byte_histogram(data: &[u8]) -> [f64; 256] { - let mut counts = [0u64; 256]; - for &b in data { - counts[b as usize] += 1; - } - let n = data.len() as f64; - let mut probs = [0.0f64; 256]; - if n == 0.0 { - return probs; - } - for i in 0..256 { - probs[i] = counts[i] as f64 / n; - } - probs -} - -/// TVD_marg(X,Y) = (1/2) Σᵢ |p_X(i) - p_Y(i)| -/// -/// Total Variation Distance over marginal byte distributions. -/// True metric on probability space. Range: [0, 1]. -/// 0 = identical distributions, 1 = completely disjoint support. -#[inline(always)] -pub fn tvd_bytes(x: &[u8], y: &[u8], _max_order: i64) -> f64 { - if x.is_empty() || y.is_empty() { - return 0.0; - } - let p_x = byte_histogram(x); - let p_y = byte_histogram(y); - - let mut sum = 0.0f64; - for i in 0..256 { - sum += (p_x[i] - p_y[i]).abs(); - } - - (sum / 2.0).clamp(0.0, 1.0) -} - -// ====== NHD: Normalized Hellinger Distance ====== - -/// NHD(X,Y) = sqrt(1 - BC(X,Y)) where BC = Σᵢ sqrt(p_X(i) · p_Y(i)) -/// -/// Normalized Hellinger Distance over marginal byte distributions. -/// True metric. Range: [0, 1]. 0 = identical, 1 = disjoint support. -#[inline(always)] -pub fn nhd_bytes(x: &[u8], y: &[u8], _max_order: i64) -> f64 { - if x.is_empty() || y.is_empty() { - return 0.0; - } - let p_x = byte_histogram(x); - let p_y = byte_histogram(y); - - // Bhattacharyya coefficient: BC = Σᵢ sqrt(p_X(i) · p_Y(i)) - let mut bc = 0.0f64; - for i in 0..256 { - bc += (p_x[i] * p_y[i]).sqrt(); - } - - // NHD = sqrt(1 - BC) - (1.0 - bc).max(0.0).sqrt() -} - -// ====== Other Information-Theoretic Measures ====== - -/// Compute cross-entropy H_{train}(test) - score test_data under model trained on train_data. -/// -/// Dispatches based on `max_order`. -#[inline(always)] -pub fn cross_entropy_bytes(test_data: &[u8], train_data: &[u8], max_order: i64) -> f64 { - with_default_ctx(|ctx| ctx.cross_entropy_bytes(test_data, train_data, max_order)) -} - -/// Compute cross-entropy rate using ROSA/CTW/RWKV. -/// Training model on `train_data` and evaluating probability of `test_data`. -#[inline(always)] -pub fn cross_entropy_rate_bytes(test_data: &[u8], train_data: &[u8], max_order: i64) -> f64 { - with_default_ctx(|ctx| ctx.cross_entropy_rate_bytes(test_data, train_data, max_order)) -} - -/// Generate a continuation from `prompt` -/// using the current default context and [`GenerationConfig::default()`]. -/// -/// The default is deterministic frozen sampling with seed `42`. -#[inline(always)] -pub fn generate_bytes(prompt: &[u8], bytes: usize, max_order: i64) -> Vec { - with_default_ctx(|ctx| ctx.generate_bytes(prompt, bytes, max_order)) -} - -/// Generate a continuation from `prompt` using the current default context. -#[inline(always)] -pub fn generate_bytes_with_config( - prompt: &[u8], - bytes: usize, - max_order: i64, - config: GenerationConfig, -) -> Vec { - with_default_ctx(|ctx| ctx.generate_bytes_with_config(prompt, bytes, max_order, config)) -} - -/// Generate a continuation after conditioning on an explicit chain of prefix parts -/// using the current default context and [`GenerationConfig::default()`]. -#[inline(always)] -pub fn generate_bytes_conditional_chain( - prefix_parts: &[&[u8]], - bytes: usize, - max_order: i64, -) -> Vec { - with_default_ctx(|ctx| ctx.generate_bytes_conditional_chain(prefix_parts, bytes, max_order)) -} - -/// Generate a continuation after conditioning on an explicit chain of prefix parts -/// using the current default context. -#[inline(always)] -pub fn generate_bytes_conditional_chain_with_config( - prefix_parts: &[&[u8]], - bytes: usize, - max_order: i64, - config: GenerationConfig, -) -> Vec { - with_default_ctx(|ctx| { - ctx.generate_bytes_conditional_chain_with_config(prefix_parts, bytes, max_order, config) - }) -} - -/// Kullback-Leibler Divergence D_KL(P || Q) = Σ p(x) log(p(x) / q(x)) -/// -/// Marginal only. Measure of how one probability distribution is different from a second. -pub fn d_kl_bytes(x: &[u8], y: &[u8]) -> f64 { - if x.is_empty() || y.is_empty() { - return 0.0; - } - let p_x = byte_histogram(x); - let p_y = byte_histogram(y); - let mut d_kl = 0.0f64; - for i in 0..256 { - if p_x[i] > 0.0 { - let q_y = p_y[i].max(1e-12); - d_kl += p_x[i] * (p_x[i] / q_y).log2(); - } - } - d_kl.max(0.0) -} - -/// Jensen-Shannon Divergence JSD(P || Q) = 1/2 D_KL(P || M) + 1/2 D_KL(Q || M) -/// where M = 1/2 (P + Q) -/// -/// Marginal only. Symmetrized and smoothed version of KL divergence. Range `[0,1]`. -pub fn js_div_bytes(x: &[u8], y: &[u8]) -> f64 { - if x.is_empty() || y.is_empty() { - return 0.0; - } - let p_x = byte_histogram(x); - let p_y = byte_histogram(y); - let mut m = [0.0f64; 256]; - for i in 0..256 { - m[i] = 0.5 * (p_x[i] + p_y[i]); - } - - let mut kl_pm = 0.0f64; - let mut kl_qm = 0.0f64; - for i in 0..256 { - if p_x[i] > 0.0 { - kl_pm += p_x[i] * (p_x[i] / m[i]).log2(); - } - if p_y[i] > 0.0 { - kl_qm += p_y[i] * (p_y[i] / m[i]).log2(); - } - } - (0.5 * kl_pm + 0.5 * kl_qm).max(0.0) -} - -// ====== Path-based convenience wrappers ====== - -/// NED for files. -pub fn ned_paths(x: &str, y: &str, max_order: i64) -> f64 { - let (bx, by) = rayon::join( - || std::fs::read(x).expect("failed to read x"), - || std::fs::read(y).expect("failed to read y"), - ); - ned_bytes(&bx, &by, max_order) -} - -/// NTE for files. -pub fn nte_paths(x: &str, y: &str, max_order: i64) -> f64 { - let (bx, by) = rayon::join( - || std::fs::read(x).expect("failed to read x"), - || std::fs::read(y).expect("failed to read y"), - ); - nte_bytes(&bx, &by, max_order) -} - -/// TVD for files. -pub fn tvd_paths(x: &str, y: &str, max_order: i64) -> f64 { - let (bx, by) = rayon::join( - || std::fs::read(x).expect("failed to read x"), - || std::fs::read(y).expect("failed to read y"), - ); - tvd_bytes(&bx, &by, max_order) -} - -/// NHD for files. -pub fn nhd_paths(x: &str, y: &str, max_order: i64) -> f64 { - let (bx, by) = rayon::join( - || std::fs::read(x).expect("failed to read x"), - || std::fs::read(y).expect("failed to read y"), - ); - nhd_bytes(&bx, &by, max_order) -} - -/// Mutual Information for files. -pub fn mutual_information_paths(x: &str, y: &str, max_order: i64) -> f64 { - let (bx, by) = rayon::join( - || std::fs::read(x).expect("failed to read x"), - || std::fs::read(y).expect("failed to read y"), - ); - mutual_information_bytes(&bx, &by, max_order) -} - -/// Conditional Entropy for files. -pub fn conditional_entropy_paths(x: &str, y: &str, max_order: i64) -> f64 { - let (bx, by) = rayon::join( - || std::fs::read(x).expect("failed to read x"), - || std::fs::read(y).expect("failed to read y"), - ); - conditional_entropy_bytes(&bx, &by, max_order) -} - -/// Cross-Entropy for files. -pub fn cross_entropy_paths(x: &str, y: &str, max_order: i64) -> f64 { - let (bx, by) = rayon::join( - || std::fs::read(x).expect("failed to read x"), - || std::fs::read(y).expect("failed to read y"), - ); - cross_entropy_bytes(&bx, &by, max_order) -} - -/// KL Divergence for files. -pub fn kl_divergence_paths(x: &str, y: &str) -> f64 { - let (bx, by) = rayon::join( - || std::fs::read(x).expect("failed to read x"), - || std::fs::read(y).expect("failed to read y"), - ); - d_kl_bytes(&bx, &by) -} - -/// Jensen-Shannon Divergence for files. -pub fn js_divergence_paths(x: &str, y: &str) -> f64 { - let (bx, by) = rayon::join( - || std::fs::read(x).expect("failed to read x"), - || std::fs::read(y).expect("failed to read y"), - ); - js_div_bytes(&bx, &by) -} - -// ====== Primitives 6 & 7 ====== - -/// Primitive 6: Intrinsic Dependence (Redundancy Ratio). -/// -/// Measures how much structure is intrinsic to the sample, relative to its -/// own marginal entropy baseline. -/// -/// `R = (H_marginal - H_rate) / H_marginal` -/// -/// Clamped to `[0,1]`. -/// -/// Interpretation: -/// - `R → 0`: Data is close to i.i.d./max-entropy (little intrinsic structure; highly extrinsically explainable by priors). -/// - `R → 1`: Data is highly predictable from its own past (strong intrinsic dependence; e.g., periodic strings like 010101...). -#[inline(always)] -pub fn intrinsic_dependence_bytes(data: &[u8], max_order: i64) -> f64 { - with_default_ctx(|ctx| ctx.intrinsic_dependence_bytes(data, max_order)) -} - -/// Primitive 7: Resistance under Allowed Transformations. -/// -/// Measures how much information is preserved after a transformation `T` is applied to `X`. -/// -/// `Resistance(X, T) = I(X; T(X)) / H(X)` -/// -/// Range `[0,1]` (with guard for `H(X)=0`). -/// * 1 means perfectly resistant (identity transformation). -/// * 0 means the transformation destroyed all information (e.g. mapping everything to a constant). -/// -/// Assumes X and T(X) are aligned. -#[inline(always)] -pub fn resistance_to_transformation_bytes(x: &[u8], tx: &[u8], max_order: i64) -> f64 { - with_default_ctx(|ctx| ctx.resistance_to_transformation_bytes(x, tx, max_order)) -} - -#[cfg(test)] -mod tests { - use super::*; - - fn test_match_backend() -> RateBackend { - RateBackend::Match { - hash_bits: 12, - min_len: 2, - max_len: 16, - base_mix: 0.01, - confidence_scale: 1.0, - } - } - - fn test_ppmd_backend() -> RateBackend { - RateBackend::Ppmd { - order: 4, - memory_mb: 1, - } - } - - fn test_calibrated_backend() -> RateBackend { - RateBackend::Calibrated { - spec: Arc::new(CalibratedSpec { - base: test_match_backend(), - context: CalibrationContextKind::Text, - bins: 16, - learning_rate: 0.05, - bias_clip: 4.0, - }), - } - } - - fn test_mixture_backend() -> RateBackend { - RateBackend::Mixture { - spec: Arc::new(MixtureSpec::new( - MixtureKind::Bayes, - vec![ - MixtureExpertSpec { - name: Some("match".to_string()), - log_prior: 0.0, - max_order: -1, - backend: test_match_backend(), - }, - MixtureExpertSpec { - name: Some("ppmd".to_string()), - log_prior: 0.0, - max_order: -1, - backend: test_ppmd_backend(), - }, - ], - )), - } - } - - fn test_particle_backend() -> RateBackend { - RateBackend::Particle { - spec: Arc::new(ParticleSpec { - num_particles: 4, - num_cells: 4, - cell_dim: 8, - num_rules: 2, - selector_hidden: 16, - rule_hidden: 16, - context_window: 8, - unroll_steps: 1, - ..ParticleSpec::default() - }), - } - } - - fn continuation_prompt() -> &'static [u8] { - b"If a frog is green, dogs are red.\nIf a toad is green, cats are red.\nIf a dog is green, frogs are red.\nIf a cat is green, toads are red.\nIf a frog is red, dogs are green.\nIf a toad is red, cats are green.\nIf a dog is red, frogs are green.\nIf a cat is red, toads are \n" - } - - fn assert_deterministic_generate_for_backend( - backend: RateBackend, - max_order: i64, - bytes: usize, - label: &str, - ) { - let prompt = continuation_prompt(); - let a = generate_rate_backend_chain( - &[prompt], - bytes, - max_order, - &backend, - GenerationConfig::default(), - ); - let b = generate_rate_backend_chain( - &[prompt], - bytes, - max_order, - &backend, - GenerationConfig::default(), - ); - assert_eq!( - a, b, - "{label} generation should be deterministic for identical input" - ); - assert_eq!( - a.len(), - bytes, - "{label} generation should emit requested byte count" - ); - } - - fn assert_sampled_generate_for_backend( - backend: RateBackend, - max_order: i64, - bytes: usize, - label: &str, - ) { - let prompt = continuation_prompt(); - let config = GenerationConfig::sampled_frozen(42); - let a = generate_rate_backend_chain(&[prompt], bytes, max_order, &backend, config); - let b = generate_rate_backend_chain(&[prompt], bytes, max_order, &backend, config); - assert_eq!( - a, b, - "{label} sampled generation should be deterministic for a fixed seed" - ); - assert_eq!( - a.len(), - bytes, - "{label} sampled generation should emit requested byte count" - ); - } - - #[cfg(feature = "backend-zpaq")] - #[test] - fn ncd_basic_identity_nonnegative() { - let x = b"abcdabcdabcd"; - let d = ncd_bytes(x, x, "5", NcdVariant::Vitanyi); - assert!(d >= -1e-9); - } - - #[test] - fn shannon_identities_marginal_aligned() { - let x = b"abracadabra"; - let y = b"abracadabra"; - - let h = marginal_entropy_bytes(x); - let mi = mutual_information_bytes(x, y, 0); - let h_xy = joint_marginal_entropy_bytes(x, y); - let h_x_given_y = conditional_entropy_bytes(x, y, 0); - let ned = ned_bytes(x, y, 0); - let nte = nte_bytes(x, y, 0); - - assert!((h_xy - h).abs() < 1e-12); - assert!(h_x_given_y.abs() < 1e-12); - assert!((mi - h).abs() < 1e-12); - assert!(ned.abs() < 1e-12); - assert!(nte.abs() < 1e-12); - } - - #[test] - fn shannon_identities_rate_aligned_reasonable() { - let x = b"the quick brown fox jumps over the lazy dog"; - let y = b"the quick brown fox jumps over the lazy dog"; - let max_order = 8; - let prev = get_default_ctx(); - set_default_ctx(InfotheoryCtx::new( - RateBackend::RosaPlus, - CompressionBackend::default(), - )); - - let h_x = entropy_rate_bytes(x, max_order); - let h_xy = joint_entropy_rate_bytes(x, y, max_order); - let h_x_given_y = conditional_entropy_rate_bytes(x, y, max_order); - let mi = mutual_information_bytes(x, y, max_order); - let ned = ned_bytes(x, y, max_order); - - // Finite-sample estimators won't be exact; allow reasonable tolerance. - let tol = 0.2; - assert!((h_xy - h_x).abs() < tol); - assert!(h_x_given_y < tol); - assert!((mi - h_x).abs() < tol); - assert!(ned < tol); - set_default_ctx(prev); - } - - #[test] - fn resistance_identity_is_one() { - let x = b"some repeated repeated repeated text"; - let prev = get_default_ctx(); - set_default_ctx(InfotheoryCtx::new( - RateBackend::RosaPlus, - CompressionBackend::default(), - )); - let r0 = resistance_to_transformation_bytes(x, x, 0); - let r8 = resistance_to_transformation_bytes(x, x, 8); - assert!((r0 - 1.0).abs() < 1e-12); - assert!((r8 - 1.0).abs() < 1e-6); - set_default_ctx(prev); - } - - #[test] - fn marginal_metrics_empty_inputs_are_zero() { - let empty: &[u8] = &[]; - let x = b"abc"; - - assert_eq!(tvd_bytes(empty, x, 0), 0.0); - assert_eq!(tvd_bytes(x, empty, 0), 0.0); - assert_eq!(nhd_bytes(empty, x, 0), 0.0); - assert_eq!(nhd_bytes(x, empty, 0), 0.0); - assert_eq!(d_kl_bytes(empty, x), 0.0); - assert_eq!(d_kl_bytes(x, empty), 0.0); - assert_eq!(js_div_bytes(empty, x), 0.0); - assert_eq!(js_div_bytes(x, empty), 0.0); - } - - #[test] - fn marginal_cross_entropy_empty_test_is_zero() { - let empty: &[u8] = &[]; - let y = b"abc"; - let ctx = InfotheoryCtx::with_zpaq("5"); - assert_eq!(ctx.cross_entropy_bytes(empty, y, 0), 0.0); - } - - #[cfg(not(feature = "backend-zpaq"))] - #[test] - #[should_panic(expected = "CompressionBackend::Zpaq is unavailable")] - fn explicit_zpaq_backend_fails_loudly() { - let backend = CompressionBackend::Zpaq { - method: "5".to_string(), - }; - let _ = compress_size_backend(b"abc", &backend); - } - - #[cfg(not(feature = "backend-zpaq"))] - #[test] - fn default_compression_backend_falls_back_to_rate_coding() { - let backend = CompressionBackend::default(); - assert!(matches!( - &backend, - CompressionBackend::Rate { - coder: crate::coders::CoderType::AC, - framing: crate::compression::FramingMode::Raw, - .. - } - )); - assert!(compress_size_backend(b"abc", &backend) > 0); - } - - #[test] - fn backend_switching_test() { - let x = b"hello world context"; - - // Default is RosaPlus - let h_rosa = entropy_rate_bytes(x, 8); - - // Switch to CTW - set_default_ctx(InfotheoryCtx::new( - RateBackend::Ctw { depth: 16 }, - CompressionBackend::default(), - )); - - let h_ctw = entropy_rate_bytes(x, 8); - - // They should generally be different, but most importantly, CTW worked - assert!(h_ctw > 0.0); - - // Reset to default - set_default_ctx(InfotheoryCtx::default()); - let h_rosa_back = entropy_rate_bytes(x, 8); - assert!((h_rosa - h_rosa_back).abs() < 1e-12); - } - - #[test] - fn ctw_early_updates_work() { - // Test that CTW produces valid predictions from the very start, - // not just after `depth` symbols have been processed. - use crate::ctw::ContextTree; - - let mut tree = ContextTree::new(16); - - // Even the first prediction should be valid (not NaN, not 0) - let p0 = tree.predict(false); - let p1 = tree.predict(true); - - // Initial KT estimator gives 0.5 / 1 = 0.5 for each symbol - assert!((p0 - 0.5).abs() < 1e-10, "p0 should be ~0.5, got {}", p0); - assert!((p1 - 0.5).abs() < 1e-10, "p1 should be ~0.5, got {}", p1); - assert!((p0 + p1 - 1.0).abs() < 1e-10, "p0 + p1 should = 1.0"); - - // Update with a few symbols and verify log_prob becomes negative (valid) - for _ in 0..5 { - tree.update(true); - tree.update(false); - } - - let log_prob = tree.get_log_block_probability(); - assert!( - log_prob < 0.0, - "log_prob should be negative (< log 1), got {}", - log_prob - ); - assert!(log_prob.is_finite(), "log_prob should be finite"); - } - - #[test] - fn nte_can_exceed_one() { - // Test that NTE is properly clamped to [0, 2] instead of [0, 1] - // For independent sequences with similar entropy, NTE can approach 2.0 - // - // Note: For *marginal* NTE, due to how joint entropy works for aligned pairs, - // it's mathematically bounded differently. The fix for NTE clamping primarily - // affects *rate*-based NTE where VI can truly be 2*max(H). - // - // We test that the clamp upper bound is at least > 1.0 for cases where VI > max(H) - - // Use CTW backend for rate-based test - set_default_ctx(InfotheoryCtx::new( - RateBackend::Ctw { depth: 8 }, - CompressionBackend::default(), - )); - - // Generate two completely different patterns - should have high VI - let x: Vec = (0..200).map(|i| (i % 2) as u8).collect(); // 010101... - let y: Vec = (0..200).map(|i| ((i + 1) % 2) as u8).collect(); // 101010... - - let nte_rate = nte_rate_backend(&x, &y, -1, &RateBackend::Ctw { depth: 8 }); - - // With the fix, NTE should not be clamped to 1.0 - // It may or may not exceed 1.0 depending on the specifics, but it should be allowed to - assert!( - (0.0..=2.0 + 1e-9).contains(&nte_rate), - "NTE should be in [0, 2], got {}", - nte_rate - ); - - // Reset context - set_default_ctx(InfotheoryCtx::default()); - } - - #[test] - fn ctw_empty_data_returns_zero() { - // Verify empty data doesn't cause division-by-zero or NaN - set_default_ctx(InfotheoryCtx::new( - RateBackend::Ctw { depth: 16 }, - CompressionBackend::default(), - )); - - let empty: &[u8] = &[]; - let h = entropy_rate_bytes(empty, -1); - assert_eq!(h, 0.0, "empty data should return 0.0 entropy"); - - // Reset - set_default_ctx(InfotheoryCtx::default()); - } - - #[test] - fn joint_entropy_rate_aligns_inputs_and_handles_empty_cases() { - let cases = vec![ - ("ctw", RateBackend::Ctw { depth: 8 }), - ( - "fac-ctw", - RateBackend::FacCtw { - base_depth: 8, - num_percept_bits: 8, - encoding_bits: 8, - }, - ), - ("match", test_match_backend()), - ]; - - for (name, backend) in cases { - assert_eq!( - joint_entropy_rate_backend(b"", b"nonempty", -1, &backend), - 0.0, - "{name} should return 0.0 for empty aligned pairs" - ); - assert_eq!( - joint_entropy_rate_backend(b"nonempty", b"", -1, &backend), - 0.0, - "{name} should return 0.0 when alignment truncates to empty" - ); - - let aligned = joint_entropy_rate_backend(b"abcd", b"wxyz", -1, &backend); - let truncated = joint_entropy_rate_backend(b"abcdextra", b"wxyz", -1, &backend); - assert!( - (aligned - truncated).abs() < 1e-12, - "{name} should score only the aligned prefix: aligned={aligned} truncated={truncated}" - ); - } - } - - #[test] - fn biased_entropy_is_repeatable_across_backend_families() { - let data = b"ABABABAABBABABABAABB"; - let cases = vec![ - ("match", test_match_backend()), - ("ppmd", test_ppmd_backend()), - ("calibrated", test_calibrated_backend()), - ("ctw", RateBackend::Ctw { depth: 8 }), - ("mixture", test_mixture_backend()), - ("particle", test_particle_backend()), - ]; - - for (name, backend) in cases { - let h1 = biased_entropy_rate_backend(data, -1, &backend); - let h2 = biased_entropy_rate_backend(data, -1, &backend); - assert!(h1.is_finite(), "{name} biased entropy should be finite"); - assert!( - (h1 - h2).abs() < 1e-12, - "{name} biased entropy leaked mutable state across calls: h1={h1} h2={h2}" - ); - } - } - - #[test] - fn generate_bytes_chain_matches_flat_prompt() { - let prompt = continuation_prompt(); - let split_at = prompt.len() / 2; - let front = &prompt[..split_at]; - let back = &prompt[split_at..]; - let backend = RateBackend::Ctw { depth: 32 }; - let bytes = 8usize; - let max_order = -1; - - let flat = generate_rate_backend_chain( - &[prompt], - bytes, - max_order, - &backend, - GenerationConfig::default(), - ); - let chained = generate_rate_backend_chain( - &[front, back], - bytes, - max_order, - &backend, - GenerationConfig::default(), - ); - assert_eq!( - flat, chained, - "chain conditioning should match flat prompt conditioning" - ); - } - - #[test] - fn generate_bytes_api_is_deterministic_for_ctw_rosa_match_ppmd() { - assert_deterministic_generate_for_backend(RateBackend::Ctw { depth: 32 }, -1, 8, "ctw"); - assert_deterministic_generate_for_backend(RateBackend::RosaPlus, -1, 8, "rosaplus"); - assert_deterministic_generate_for_backend(test_match_backend(), -1, 8, "match"); - assert_deterministic_generate_for_backend(test_ppmd_backend(), -1, 8, "ppmd"); - } - - #[cfg(feature = "backend-rwkv")] - #[test] - fn generate_bytes_api_is_deterministic_for_rwkv_method() { - let backend = RateBackend::Rwkv7Method { - method: "cfg:hidden=64,layers=1,intermediate=64,decay_rank=8,a_rank=8,v_rank=8,g_rank=8,seed=31,train=none,lr=0.0,stride=1;policy:schedule=0..100:infer".to_string(), - }; - assert_deterministic_generate_for_backend(backend, -1, 8, "rwkv7"); - } - - #[test] - fn sampled_generation_is_deterministic_for_ctw_rosa_match_ppmd() { - assert_sampled_generate_for_backend(RateBackend::Ctw { depth: 32 }, -1, 8, "ctw"); - assert_sampled_generate_for_backend(RateBackend::RosaPlus, -1, 8, "rosaplus"); - assert_sampled_generate_for_backend(test_match_backend(), -1, 8, "match"); - assert_sampled_generate_for_backend(test_ppmd_backend(), -1, 8, "ppmd"); - } - - #[cfg(feature = "backend-rwkv")] - #[test] - fn sampled_generation_is_deterministic_for_rwkv_method() { - let backend = RateBackend::Rwkv7Method { - method: "cfg:hidden=64,layers=1,intermediate=64,decay_rank=8,a_rank=8,v_rank=8,g_rank=8,seed=31,train=none,lr=0.0,stride=1;policy:schedule=0..100:infer".to_string(), - }; - assert_sampled_generate_for_backend(backend, -1, 8, "rwkv7"); - } - - #[test] - fn rosaplus_sampled_generation_predicts_green_continuation() { - let out = generate_rate_backend_chain( - &[continuation_prompt()], - 8, - -1, - &RateBackend::RosaPlus, - GenerationConfig::sampled_frozen(42), - ); - assert_eq!(out, b" green.\n"); - } - - #[test] - fn rate_backend_session_matches_ctx_generation() { - let prompt = continuation_prompt(); - let backend = RateBackend::Ppmd { - order: 12, - memory_mb: 8, - }; - let mut session = - RateBackendSession::from_backend(backend.clone(), -1, Some((prompt.len() + 8) as u64)) - .expect("session init"); - session.observe(prompt); - let from_session = session.generate_bytes(8, GenerationConfig::sampled_frozen(42)); - session.finish().expect("session finish"); - - let ctx = InfotheoryCtx::new(backend, CompressionBackend::default()); - let from_ctx = - ctx.generate_bytes_with_config(prompt, 8, -1, GenerationConfig::sampled_frozen(42)); - assert_eq!(from_session, from_ctx); - } - - #[test] - fn biased_entropy_ctw_uses_frozen_plugin_scoring() { - let backend = RateBackend::Ctw { depth: 8 }; - let data = b"AAAAAAAA"; - let plugin = biased_entropy_rate_backend(data, -1, &backend); - let prequential = entropy_rate_backend(data, -1, &backend); - assert!( - plugin + 1e-9 < prequential, - "expected plugin scoring to beat prequential scoring: plugin={plugin} prequential={prequential}" - ); - } - - #[test] - fn rosa_plugin_entropy_matches_direct_model_api() { - let data = b"abracadabra"; - let backend = RateBackend::RosaPlus; - - let plugin = biased_entropy_rate_backend(data, 3, &backend); - - let mut direct = rosaplus::RosaPlus::new(3, false, 0, 42); - direct.train_example(data); - direct.build_lm(); - let expected = direct.cross_entropy(data); - - assert!( - (plugin - expected).abs() < 1e-12, - "rosa plugin entropy must match direct model API: plugin={plugin} expected={expected}" - ); - } - - #[test] - fn rosa_plugin_cross_entropy_matches_direct_model_api() { - let train = b"alakazam"; - let test = b"abracadabra"; - let backend = RateBackend::RosaPlus; - - let plugin = cross_entropy_rate_backend(test, train, 3, &backend); - - let mut direct = rosaplus::RosaPlus::new(3, false, 0, 42); - direct.train_example(train); - direct.build_lm(); - let expected = direct.cross_entropy(test); - - assert!( - (plugin - expected).abs() < 1e-12, - "rosa plugin cross entropy must match direct model API: plugin={plugin} expected={expected}" - ); - } - - #[test] - fn datagen_bernoulli_entropy_estimate() { - // Test that estimated entropy is close to theoretical for Bernoulli(0.5) - let p = 0.5; - let theoretical_h = crate::datagen::bernoulli_entropy(p); - assert!((theoretical_h - 1.0).abs() < 1e-10); - - // Generate data and check marginal entropy is close to theoretical - let data = crate::datagen::bernoulli(10000, p, 42); - let estimated_h = marginal_entropy_bytes(&data); - - // Should be close to 1.0 bit (since values are 0 or 1) - assert!( - (estimated_h - theoretical_h).abs() < 0.1, - "estimated H={} should be close to theoretical H={}", - estimated_h, - theoretical_h - ); - } - - #[cfg(feature = "backend-rwkv")] - #[test] - fn rwkv_method_entropy_is_stable_across_calls() { - let method = "cfg:hidden=64,layers=1,intermediate=64,decay_rank=8,a_rank=8,v_rank=8,g_rank=8,seed=21,train=sgd,lr=0.01,stride=1;policy:schedule=0..100:infer"; - let backend = RateBackend::Rwkv7Method { - method: method.to_string(), - }; - let data = b"rwkv method entropy stability regression sample"; - - let h1 = entropy_rate_backend(data, -1, &backend); - let h2 = entropy_rate_backend(data, -1, &backend); - assert!( - (h1 - h2).abs() < 1e-12, - "rwkv method entropy leaked mutable state across calls: h1={h1}, h2={h2}" - ); - } - - #[cfg(feature = "backend-rwkv")] - #[test] - fn rwkv_method_without_policy_is_accepted_by_public_api() { - let backend = RateBackend::Rwkv7Method { - method: "cfg:hidden=64,layers=1,intermediate=64".to_string(), - }; - let data = b"rwkv method without policy"; - let h1 = entropy_rate_backend(data, -1, &backend); - let h2 = biased_entropy_rate_backend(data, -1, &backend); - assert!(h1.is_finite()); - assert!(h2.is_finite()); - } - - #[cfg(feature = "backend-rwkv")] - #[test] - fn rwkv_infer_only_plugin_collapses_to_single_pass_entropy() { - let backend = RateBackend::Rwkv7Method { - method: "cfg:hidden=64,layers=1,intermediate=64,decay_rank=8,a_rank=8,v_rank=8,g_rank=8,seed=25,train=none,lr=0.0,stride=1;policy:schedule=0..100:infer".to_string(), - }; - let data = b"rwkv infer-only plugin equality sample"; - let h = entropy_rate_backend(data, -1, &backend); - let plugin = biased_entropy_rate_backend(data, -1, &backend); - assert!( - (h - plugin).abs() < 1e-12, - "infer-only rwkv plugin should equal single-pass entropy: h={h}, plugin={plugin}" - ); - } - - #[cfg(feature = "backend-rwkv")] - #[test] - fn rwkv_method_biased_entropy_is_stable_across_calls_with_training_policy() { - let backend = RateBackend::Rwkv7Method { - method: "cfg:hidden=64,layers=1,intermediate=64,decay_rank=8,a_rank=8,v_rank=8,g_rank=8,seed=23,train=sgd,lr=0.01,stride=1;policy:schedule=0..100:train(scope=head+bias,opt=sgd,lr=0.01,stride=1,bptt=1,clip=0,momentum=0.0)".to_string(), - }; - let data = b"rwkv plugin stability sample"; - let h1 = biased_entropy_rate_backend(data, -1, &backend); - let h2 = biased_entropy_rate_backend(data, -1, &backend); - assert!( - (h1 - h2).abs() < 1e-12, - "rwkv method biased entropy leaked mutable state across calls: h1={h1}, h2={h2}" - ); - } - - #[cfg(feature = "backend-rwkv")] - #[test] - fn rwkv_method_conditional_chain_is_stable_across_calls() { - let method = "cfg:hidden=64,layers=1,intermediate=64,decay_rank=8,a_rank=8,v_rank=8,g_rank=8,seed=22,train=sgd,lr=0.01,stride=1;policy:schedule=0..100:infer"; - let ctx = InfotheoryCtx::new( - RateBackend::Rwkv7Method { - method: method.to_string(), - }, - CompressionBackend::default(), - ); - - let prefix = b"universal prior slice"; - let data = b"query payload"; - let h1 = ctx.cross_entropy_conditional_chain(&[prefix.as_slice()], data); - let h2 = ctx.cross_entropy_conditional_chain(&[prefix.as_slice()], data); - assert!( - (h1 - h2).abs() < 1e-12, - "rwkv method conditional chain leaked mutable state across calls: h1={h1}, h2={h2}" - ); - } - - #[cfg(feature = "backend-mamba")] - #[test] - fn mamba_method_without_policy_is_accepted_by_public_api() { - let backend = RateBackend::MambaMethod { - method: "cfg:hidden=64,layers=1,intermediate=96".to_string(), - }; - let data = b"mamba method without policy"; - let h1 = entropy_rate_backend(data, -1, &backend); - let h2 = biased_entropy_rate_backend(data, -1, &backend); - assert!(h1.is_finite()); - assert!(h2.is_finite()); - } - - #[cfg(feature = "backend-mamba")] - #[test] - fn mamba_infer_only_plugin_collapses_to_single_pass_entropy() { - let backend = RateBackend::MambaMethod { - method: "cfg:hidden=64,layers=1,intermediate=96,state=16,conv=4,dt_rank=16,seed=26,train=none,lr=0.0,stride=1;policy:schedule=0..100:infer".to_string(), - }; - let data = b"mamba infer-only plugin equality sample"; - let h = entropy_rate_backend(data, -1, &backend); - let plugin = biased_entropy_rate_backend(data, -1, &backend); - assert!( - (h - plugin).abs() < 1e-12, - "infer-only mamba plugin should equal single-pass entropy: h={h}, plugin={plugin}" - ); - } - - #[cfg(feature = "backend-mamba")] - #[test] - fn mamba_method_biased_entropy_is_stable_across_calls_with_training_policy() { - let backend = RateBackend::MambaMethod { - method: "cfg:hidden=64,layers=1,intermediate=96,state=16,conv=4,dt_rank=16,seed=24,train=sgd,lr=0.01,stride=1;policy:schedule=0..100:train(scope=head+bias,opt=sgd,lr=0.01,stride=1,bptt=1,clip=0,momentum=0.0)".to_string(), - }; - let data = b"mamba plugin stability sample"; - let h1 = biased_entropy_rate_backend(data, -1, &backend); - let h2 = biased_entropy_rate_backend(data, -1, &backend); - assert!( - (h1 - h2).abs() < 1e-12, - "mamba method biased entropy leaked mutable state across calls: h1={h1}, h2={h2}" - ); - } - - #[test] - fn particle_entropy_rate_in_valid_range() { - let rb = test_particle_backend(); - let data = b"hello world particle backend test"; - let rate = entropy_rate_backend(data, -1, &rb); - assert!( - rate > 0.0 && rate < 8.0, - "particle entropy rate out of (0, 8) range: {rate}" - ); - } - - #[test] - fn particle_cross_entropy_stability() { - let rb = test_particle_backend(); - let train = b"ABCABC"; - let test = b"ABC"; - let h1 = cross_entropy_rate_backend(test, train, -1, &rb); - let h2 = cross_entropy_rate_backend(test, train, -1, &rb); - assert!( - (h1 - h2).abs() < 1e-12, - "particle cross entropy not deterministic: h1={h1}, h2={h2}" - ); - } - - #[test] - fn particle_empty_input() { - let rb = RateBackend::Particle { - spec: Arc::new(ParticleSpec::default()), - }; - let rate = entropy_rate_backend(b"", -1, &rb); - assert!( - rate == 0.0, - "particle entropy rate for empty input should be 0.0, got {rate}" - ); - } - - #[test] - fn particle_joint_entropy_rate() { - let rb = test_particle_backend(); - let x = b"AAAA"; - let y = b"BBBB"; - let joint = joint_entropy_rate_backend(x, y, -1, &rb); - assert!( - joint > 0.0 && joint < 16.0, - "particle joint entropy rate out of range: {joint}" - ); - } -} diff --git a/src/main.rs b/src/main.rs deleted file mode 100644 index ce969f8d..00000000 --- a/src/main.rs +++ /dev/null @@ -1,4133 +0,0 @@ -//! # InfoTheory CLI -//! -//! Command-line interface for the `infotheory` library. -//! Provides access to compression-based (NCD) and entropy-based (Shannon, ROSA, CTW) -//! estimators for files, as well as AIXI agents. -//! -//! ## Usage -//! -//! ### Single-file mode: -//! ```bash -//! infotheory [method/max_order] -//! ``` -//! -//! ### Search mode: -//! ```bash -//! infotheory search [options] -//! ``` -//! -//! ### AIXI Agent mode: -//! ```bash -//! infotheory aixi -//! ``` -//! -//! ### Batch JSON mode (for programmatic use): -//! ```bash -//! infotheory batch < input.json > output.json -//! echo '{"op":"metrics","text":"hello world"}' | infotheory batch -//! ``` -//! -//! See `print_usage` for details on supported primitives. - -use infotheory::aixi::agent::{Agent, AgentConfig}; -use infotheory::aixi::aiqi::{AiqiAgent, AiqiConfig}; -use infotheory::aixi::common::{ObservationKeyMode, RandomGenerator}; -use infotheory::aixi::environment::{ - BiasedRockPaperScissor, CoinFlip, CtwTest, Environment, ExtendedTiger, KuhnPoker, TicTacToe, -}; -#[cfg(feature = "vm")] -use infotheory::aixi::vm_nyx::{ - FuzzMutator as NyxFuzzMutator, NyxActionFilter, NyxActionSource, NyxActionSpec, NyxFuzzConfig, - NyxObservationPolicy, NyxObservationStreamMode, NyxProtocolConfig, NyxRewardPolicy, - NyxRewardShaping, NyxTraceConfig, NyxVmConfig, NyxVmEnvironment, - PayloadEncoding as NyxPayloadEncoding, -}; -use infotheory::sequitur::{CanonicalSymbol, SequiturModel}; -use infotheory::*; -#[cfg(feature = "vm")] -use nyx_lite::SharedMemoryPolicy; -use std::env; -use std::fs::File; -use std::io::{self, BufRead, BufWriter, IsTerminal, Read, Write}; -use std::path::Path; -use std::sync::Arc; -#[cfg(feature = "vm")] -use std::time::{Duration, Instant}; - -#[cfg(not(feature = "vm"))] -use std::time::Instant; - -use infotheory::search; - -struct AixiRunLogger { - bits01: Option>, - jsonl: Option>, - flush_every: usize, - step: usize, -} - -impl AixiRunLogger { - fn new(v: &serde_json::Value) -> anyhow::Result> { - let bits01_path = v["trace_bits01_path"].as_str(); - let jsonl_path = v["trace_jsonl_path"].as_str(); - if bits01_path.is_none() && jsonl_path.is_none() { - return Ok(None); - } - - let bits01 = if let Some(p) = bits01_path { - let f = File::create(p)?; - Some(BufWriter::new(f)) - } else { - None - }; - let jsonl = if let Some(p) = jsonl_path { - let f = File::create(p)?; - Some(BufWriter::new(f)) - } else { - None - }; - let flush_every = v["trace_flush_every"].as_u64().unwrap_or(1024) as usize; - - Ok(Some(Self { - bits01, - jsonl, - flush_every, - step: 0, - })) - } - - fn write_bits01(&mut self, bits: &[bool]) -> anyhow::Result<()> { - if let Some(w) = self.bits01.as_mut() { - for &b in bits { - w.write_all(&[if b { 1u8 } else { 0u8 }])?; - } - } - Ok(()) - } - - fn log_percept( - &mut self, - observations: &[u64], - reward: i64, - observation_bits: usize, - reward_bits: usize, - reward_offset: i64, - ) -> anyhow::Result<()> { - // Exact same bit encoding the agent uses internally. - let mut bits = Vec::new(); - for &obs in observations { - infotheory::aixi::common::encode(&mut bits, obs, observation_bits); - } - infotheory::aixi::common::encode_reward_offset( - &mut bits, - reward, - reward_bits, - reward_offset, - ); - - self.write_bits01(&bits)?; - - if let Some(w) = self.jsonl.as_mut() { - let rec = serde_json::json!({ - "t": self.step, - "kind": "percept", - "observations": observations, - "reward": reward, - }); - writeln!(w, "{rec}")?; - } - Ok(()) - } - - fn log_action(&mut self, action: u64, action_bits: usize) -> anyhow::Result<()> { - let mut bits = Vec::new(); - infotheory::aixi::common::encode(&mut bits, action, action_bits); - self.write_bits01(&bits)?; - - if let Some(w) = self.jsonl.as_mut() { - let rec = serde_json::json!({ - "t": self.step, - "kind": "action", - "action": action, - }); - writeln!(w, "{rec}")?; - } - Ok(()) - } - - fn next_step(&mut self) -> anyhow::Result<()> { - self.step = self.step.saturating_add(1); - if self.flush_every > 0 && self.step.is_multiple_of(self.flush_every) { - if let Some(w) = self.bits01.as_mut() { - w.flush()?; - } - if let Some(w) = self.jsonl.as_mut() { - w.flush()?; - } - } - Ok(()) - } -} - -#[cfg(feature = "backend-rwkv")] -fn rwkv7_model_path_from_env() -> String { - env::var("RWKV7_MODEL_PATH").unwrap_or_else(|_| { - eprintln!("Error: RWKV7_MODEL_PATH env var must be set when using rwkv7 backends"); - std::process::exit(1); - }) -} - -#[cfg(feature = "backend-mamba")] -fn mamba_model_path_from_env() -> String { - env::var("MAMBA_MODEL_PATH").unwrap_or_else(|_| { - eprintln!("Error: MAMBA_MODEL_PATH env var must be set when using mamba backends"); - std::process::exit(1); - }) -} - -fn parse_rate_backend(v: &str) -> Option<&'static str> { - match infotheory::backends::resolve_rate_backend_name(v) { - Some(infotheory::backends::BackendAvailability::Enabled(name)) => Some(name), - Some(infotheory::backends::BackendAvailability::Disabled { canonical, feature }) => { - eprintln!( - "Error: rate backend '{canonical}' requires infotheory built with feature '{feature}'" - ); - std::process::exit(1); - } - None => None, - } -} - -fn parse_compression_backend(v: &str) -> Option<&'static str> { - match infotheory::backends::resolve_compression_backend_name(v) { - Some(infotheory::backends::BackendAvailability::Enabled(name)) => Some(name), - Some(infotheory::backends::BackendAvailability::Disabled { canonical, feature }) => { - eprintln!( - "Error: compression backend '{canonical}' requires infotheory built with feature '{feature}'" - ); - std::process::exit(1); - } - None => None, - } -} - -fn load_mixture_spec(path: &str) -> anyhow::Result { - load_mixture_spec_with_depth(path, MAX_MIXTURE_NESTING) -} - -fn load_mixture_spec_with_depth(path: &str, depth: usize) -> anyhow::Result { - let raw = std::fs::read(path)?; - let value: serde_json::Value = match serde_json::from_slice(&raw) { - Ok(v) => v, - Err(_) => { - #[cfg(feature = "backend-zpaq")] - { - let decompressed = zpaq_rs::decompress_to_vec(&raw)?; - serde_json::from_slice(&decompressed)? - } - #[cfg(not(feature = "backend-zpaq"))] - { - return Err(anyhow::anyhow!( - "Failed to parse mixture JSON, and zpaq support is disabled at compile time" - )); - } - } - }; - let base_dir = Path::new(path).parent().unwrap_or_else(|| Path::new(".")); - parse_mixture_spec_value(&value, base_dir, depth) -} - -fn load_particle_spec(path: &str) -> anyhow::Result { - let raw = std::fs::read(path)?; - let value: serde_json::Value = serde_json::from_slice(&raw)?; - let spec = parse_particle_spec_value(&value)?; - spec.validate() - .map_err(|e| anyhow::anyhow!("invalid particle spec: {e}"))?; - Ok(spec) -} - -fn load_calibrated_spec(path: &str) -> anyhow::Result { - let raw = std::fs::read(path)?; - let value: serde_json::Value = serde_json::from_slice(&raw)?; - let base_dir = Path::new(path).parent().unwrap_or_else(|| Path::new(".")); - parse_calibrated_spec_value(&value, base_dir, 4) -} - -fn load_expert_spec(path: &str) -> anyhow::Result { - let raw = std::fs::read(path)?; - let value: serde_json::Value = serde_json::from_slice(&raw)?; - let base_dir = Path::new(path).parent().unwrap_or_else(|| Path::new(".")); - parse_mixture_expert_value(&value, base_dir, MAX_MIXTURE_NESTING) -} - -fn parse_particle_spec_value(v: &serde_json::Value) -> anyhow::Result { - if v.get("experts").is_some() { - return Err(anyhow::anyhow!( - "looks like a mixture spec (found 'experts'); --rate-backend particle expects a ParticleSpec JSON" - )); - } - if let Some(kind) = v.get("kind").and_then(|k| k.as_str()) { - let k = kind.to_ascii_lowercase(); - if matches!( - k.as_str(), - "bayes" - | "fading" - | "fading-bayes" - | "switch" - | "switching" - | "mdl" - | "neural" - | "mixture" - ) { - return Err(anyhow::anyhow!( - "looks like a mixture spec (kind='{kind}'); --rate-backend particle expects a ParticleSpec JSON" - )); - } - } - let d = ParticleSpec::default(); - Ok(ParticleSpec { - num_particles: v["num_particles"] - .as_u64() - .unwrap_or(d.num_particles as u64) as usize, - context_window: v["context_window"] - .as_u64() - .unwrap_or(d.context_window as u64) as usize, - unroll_steps: v["unroll_steps"].as_u64().unwrap_or(d.unroll_steps as u64) as usize, - num_cells: v["num_cells"].as_u64().unwrap_or(d.num_cells as u64) as usize, - cell_dim: v["cell_dim"].as_u64().unwrap_or(d.cell_dim as u64) as usize, - num_rules: v["num_rules"].as_u64().unwrap_or(d.num_rules as u64) as usize, - selector_hidden: v["selector_hidden"] - .as_u64() - .unwrap_or(d.selector_hidden as u64) as usize, - rule_hidden: v["rule_hidden"].as_u64().unwrap_or(d.rule_hidden as u64) as usize, - noise_dim: v["noise_dim"].as_u64().unwrap_or(d.noise_dim as u64) as usize, - deterministic: v["deterministic"].as_bool().unwrap_or(d.deterministic), - enable_noise: v["enable_noise"].as_bool().unwrap_or(d.enable_noise), - noise_scale: v["noise_scale"].as_f64().unwrap_or(d.noise_scale), - noise_anneal_steps: v["noise_anneal_steps"] - .as_u64() - .unwrap_or(d.noise_anneal_steps as u64) as usize, - learning_rate_readout: v["learning_rate_readout"] - .as_f64() - .unwrap_or(d.learning_rate_readout), - learning_rate_selector: v["learning_rate_selector"] - .as_f64() - .unwrap_or(d.learning_rate_selector), - learning_rate_rule: v["learning_rate_rule"] - .as_f64() - .unwrap_or(d.learning_rate_rule), - bptt_depth: v["bptt_depth"].as_u64().unwrap_or(d.bptt_depth as u64) as usize, - optimizer_momentum: v["optimizer_momentum"] - .as_f64() - .unwrap_or(d.optimizer_momentum), - grad_clip: v["grad_clip"].as_f64().unwrap_or(d.grad_clip), - state_clip: v["state_clip"].as_f64().unwrap_or(d.state_clip), - forget_lambda: v["forget_lambda"].as_f64().unwrap_or(d.forget_lambda), - resample_threshold: v["resample_threshold"] - .as_f64() - .unwrap_or(d.resample_threshold), - mutate_fraction: v["mutate_fraction"].as_f64().unwrap_or(d.mutate_fraction), - mutate_scale: v["mutate_scale"].as_f64().unwrap_or(d.mutate_scale), - mutate_model_params: v["mutate_model_params"] - .as_bool() - .unwrap_or(d.mutate_model_params), - diagnostics_interval: v["diagnostics_interval"] - .as_u64() - .unwrap_or(d.diagnostics_interval as u64) as usize, - min_prob: v["min_prob"].as_f64().unwrap_or(d.min_prob), - seed: v["seed"].as_u64().unwrap_or(d.seed), - }) -} - -fn parse_calibration_context_kind(value: Option<&str>) -> anyhow::Result { - match value.unwrap_or("text").trim().to_ascii_lowercase().as_str() { - "global" => Ok(CalibrationContextKind::Global), - "byteclass" | "byte-class" | "byte_class" => Ok(CalibrationContextKind::ByteClass), - "text" => Ok(CalibrationContextKind::Text), - "repeat" => Ok(CalibrationContextKind::Repeat), - "textrepeat" | "text-repeat" | "text_repeat" => Ok(CalibrationContextKind::TextRepeat), - other => Err(anyhow::anyhow!("unknown calibration context '{other}'")), - } -} - -fn parse_calibrated_spec_value( - v: &serde_json::Value, - base_dir: &Path, - depth: usize, -) -> anyhow::Result { - if depth == 0 { - return Err(anyhow::anyhow!("calibrated spec nesting too deep")); - } - let base_backend = if let Some(base_v) = v.get("base") { - parse_mixture_expert_value(base_v, base_dir, depth - 1)?.backend - } else if let Some(path) = v["base_path"].as_str().or_else(|| v["path"].as_str()) { - let full = base_dir.join(path); - let raw = std::fs::read(&full)?; - let value: serde_json::Value = serde_json::from_slice(&raw)?; - parse_mixture_expert_value(&value, full.parent().unwrap_or(base_dir), depth - 1)?.backend - } else { - return Err(anyhow::anyhow!( - "calibrated expert requires 'base' or 'base_path'" - )); - }; - Ok(CalibratedSpec { - base: base_backend, - context: parse_calibration_context_kind(v["context"].as_str())?, - bins: v["bins"].as_u64().unwrap_or(33) as usize, - learning_rate: v["learning_rate"].as_f64().unwrap_or(0.02), - bias_clip: v["bias_clip"].as_f64().unwrap_or(4.0), - }) -} - -fn parse_mixture_kind(kind: &str) -> anyhow::Result { - infotheory::parse_mixture_kind_name(kind).map_err(anyhow::Error::msg) -} - -fn parse_mixture_schedule(schedule: &str) -> anyhow::Result { - infotheory::parse_mixture_schedule_name(schedule).map_err(anyhow::Error::msg) -} - -fn parse_mixture_spec_value( - v: &serde_json::Value, - base_dir: &Path, - depth: usize, -) -> anyhow::Result { - if depth == 0 { - return Err(anyhow::anyhow!("mixture spec nesting too deep")); - } - let kind_str = v["kind"] - .as_str() - .or_else(|| v["mixture_kind"].as_str()) - .or_else(|| v["mix_kind"].as_str()) - .unwrap_or("bayes"); - let kind = parse_mixture_kind(kind_str)?; - let schedule = v["schedule"] - .as_str() - .or_else(|| v["schedule_mode"].as_str()) - .or_else(|| v["mixture_schedule"].as_str()) - .map(parse_mixture_schedule) - .transpose()? - .unwrap_or(MixtureScheduleMode::Default); - let alpha = v["alpha"].as_f64().unwrap_or(0.01); - let decay = v["decay"].as_f64(); - let experts_v = v["experts"] - .as_array() - .ok_or_else(|| anyhow::anyhow!("mixture spec missing 'experts' array"))?; - if experts_v.is_empty() { - return Err(anyhow::anyhow!( - "mixture spec must include at least one expert" - )); - } - let mut experts = Vec::with_capacity(experts_v.len()); - for e in experts_v { - experts.push(parse_mixture_expert_value(e, base_dir, depth - 1)?); - } - let mut spec = MixtureSpec::new(kind, experts) - .with_schedule(schedule) - .with_alpha(alpha); - if let Some(decay) = decay { - spec = spec.with_decay(decay); - } - spec.validate().map_err(anyhow::Error::msg)?; - Ok(spec) -} - -fn parse_mixture_expert_value( - v: &serde_json::Value, - base_dir: &Path, - depth: usize, -) -> anyhow::Result { - if depth == 0 { - return Err(anyhow::anyhow!("mixture spec nesting too deep")); - } - let raw_kind = v["kind"] - .as_str() - .or_else(|| v["type"].as_str()) - .or_else(|| v["backend"].as_str()) - .ok_or_else(|| anyhow::anyhow!("expert missing 'kind'"))?; - let kind = match infotheory::backends::resolve_rate_backend_name(raw_kind) { - Some(infotheory::backends::BackendAvailability::Enabled(name)) => name, - Some(infotheory::backends::BackendAvailability::Disabled { canonical, feature }) => { - return Err(anyhow::anyhow!( - "expert backend '{canonical}' requires infotheory feature '{feature}'" - )); - } - None => return Err(anyhow::anyhow!("unknown expert kind '{raw_kind}'")), - }; - let name = v["name"].as_str().map(|s| s.to_string()); - let log_prior = v["log_prior"] - .as_f64() - .or_else(|| v["prior"].as_f64()) - .unwrap_or(0.0); - - match kind { - "rosaplus" => { - let max_order = v["max_order"] - .as_i64() - .or_else(|| v["order"].as_i64()) - .unwrap_or(8); - Ok(MixtureExpertSpec { - name, - log_prior, - max_order, - backend: RateBackend::RosaPlus, - }) - } - "match" => Ok(MixtureExpertSpec { - name, - log_prior, - max_order: -1, - backend: RateBackend::Match { - hash_bits: v["hash_bits"].as_u64().unwrap_or(20) as usize, - min_len: v["min_len"].as_u64().unwrap_or(4) as usize, - max_len: v["max_len"].as_u64().unwrap_or(255) as usize, - base_mix: v["base_mix"].as_f64().unwrap_or(0.02), - confidence_scale: v["confidence_scale"].as_f64().unwrap_or(1.0), - }, - }), - "sparse-match" => Ok(MixtureExpertSpec { - name, - log_prior, - max_order: -1, - backend: RateBackend::SparseMatch { - hash_bits: v["hash_bits"].as_u64().unwrap_or(19) as usize, - min_len: v["min_len"].as_u64().unwrap_or(3) as usize, - max_len: v["max_len"].as_u64().unwrap_or(64) as usize, - gap_min: v["gap_min"].as_u64().unwrap_or(1) as usize, - gap_max: v["gap_max"].as_u64().unwrap_or(2) as usize, - base_mix: v["base_mix"].as_f64().unwrap_or(0.05), - confidence_scale: v["confidence_scale"].as_f64().unwrap_or(1.0), - }, - }), - "ppmd" => Ok(MixtureExpertSpec { - name, - log_prior, - max_order: -1, - backend: RateBackend::Ppmd { - order: v["order"].as_u64().unwrap_or(10) as usize, - memory_mb: v["memory_mb"].as_u64().unwrap_or(64) as usize, - }, - }), - "sequitur" => Ok(MixtureExpertSpec { - name, - log_prior, - max_order: -1, - backend: RateBackend::Sequitur { - context_bytes: v["context_bytes"].as_u64().unwrap_or(64) as usize, - }, - }), - "calibrated" => { - let spec = parse_calibrated_spec_value(v, base_dir, depth - 1)?; - Ok(MixtureExpertSpec { - name, - log_prior, - max_order: -1, - backend: RateBackend::Calibrated { - spec: Arc::new(spec), - }, - }) - } - "ctw" => { - let depth = v["depth"] - .as_u64() - .or_else(|| v["ct_depth"].as_u64()) - .unwrap_or(16) as usize; - Ok(MixtureExpertSpec { - name, - log_prior, - max_order: -1, - backend: RateBackend::Ctw { depth }, - }) - } - "fac-ctw" => { - let base_depth = v["base_depth"] - .as_u64() - .or_else(|| v["ct_depth"].as_u64()) - .unwrap_or(16) as usize; - let encoding_bits = v["encoding_bits"].as_u64().unwrap_or(8) as usize; - let num_percept_bits = v["num_percept_bits"] - .as_u64() - .unwrap_or(encoding_bits as u64) as usize; - Ok(MixtureExpertSpec { - name, - log_prior, - max_order: -1, - backend: RateBackend::FacCtw { - base_depth, - num_percept_bits, - encoding_bits, - }, - }) - } - "zpaq" => { - let method = v["method"].as_str().unwrap_or("2").to_string(); - if let Err(err) = validate_zpaq_rate_method(&method) { - return Err(anyhow::anyhow!( - "unsupported ZPAQ rate method '{method}': {err}" - )); - } - Ok(MixtureExpertSpec { - name, - log_prior, - max_order: -1, - backend: RateBackend::Zpaq { method }, - }) - } - "mamba" => { - #[cfg(feature = "backend-mamba")] - { - if let Some(method) = v["method"].as_str().or_else(|| v["mamba_method"].as_str()) { - Ok(MixtureExpertSpec { - name, - log_prior, - max_order: -1, - backend: RateBackend::MambaMethod { - method: method.to_string(), - }, - }) - } else { - let model_path = v["model_path"] - .as_str() - .or_else(|| v["mamba_model_path"].as_str()) - .ok_or_else(|| { - anyhow::anyhow!("mamba expert missing model_path or method") - })?; - let model_path = base_dir.join(model_path); - let model = load_mamba_model_from_path(model_path.to_string_lossy().as_ref()); - Ok(MixtureExpertSpec { - name, - log_prior, - max_order: -1, - backend: RateBackend::Mamba { model }, - }) - } - } - #[cfg(not(feature = "backend-mamba"))] - { - let _ = (name, log_prior); - Err(anyhow::anyhow!( - "mamba expert requires 'backend-mamba' feature in infotheory" - )) - } - } - "rwkv7" => { - #[cfg(feature = "backend-rwkv")] - { - if let Some(method) = v["method"].as_str().or_else(|| v["rwkv_method"].as_str()) { - Ok(MixtureExpertSpec { - name, - log_prior, - max_order: -1, - backend: RateBackend::Rwkv7Method { - method: method.to_string(), - }, - }) - } else { - let model_path = v["model_path"] - .as_str() - .or_else(|| v["rwkv_model_path"].as_str()) - .ok_or_else(|| { - anyhow::anyhow!("rwkv expert missing model_path or method") - })?; - let model_path = base_dir.join(model_path); - let model = load_rwkv7_model_from_path(model_path.to_string_lossy().as_ref()); - Ok(MixtureExpertSpec { - name, - log_prior, - max_order: -1, - backend: RateBackend::Rwkv7 { model }, - }) - } - } - #[cfg(not(feature = "backend-rwkv"))] - { - let _ = (name, log_prior); - Err(anyhow::anyhow!( - "rwkv expert requires 'backend-rwkv' feature in infotheory" - )) - } - } - "mixture" => { - let spec = if let Some(spec_v) = v.get("spec") { - parse_mixture_spec_value(spec_v, base_dir, depth - 1)? - } else if let Some(path) = v["spec_path"].as_str().or_else(|| v["path"].as_str()) { - let full = base_dir.join(path); - load_mixture_spec_with_depth(full.to_str().unwrap_or(path), depth - 1)? - } else { - return Err(anyhow::anyhow!( - "mixture expert requires 'spec' (inline) or 'spec_path'" - )); - }; - Ok(MixtureExpertSpec { - name, - log_prior, - max_order: -1, - backend: RateBackend::Mixture { - spec: Arc::new(spec), - }, - }) - } - "particle" => { - let spec = if let Some(spec_v) = v.get("spec") { - parse_particle_spec_value(spec_v)? - } else if let Some(path) = v["spec_path"].as_str().or_else(|| v["path"].as_str()) { - let full = base_dir.join(path); - load_particle_spec(full.to_str().unwrap_or(path))? - } else { - ParticleSpec::default() - }; - spec.validate() - .map_err(|e| anyhow::anyhow!("invalid particle spec: {e}"))?; - Ok(MixtureExpertSpec { - name, - log_prior, - max_order: -1, - backend: RateBackend::Particle { - spec: Arc::new(spec), - }, - }) - } - other => Err(anyhow::anyhow!("unsupported expert kind '{other}'")), - } -} - -#[cfg(feature = "vm")] -fn parse_shared_memory_policy(v: Option<&str>) -> SharedMemoryPolicy { - match v.unwrap_or("snapshot") { - "preserve" | "keep" => SharedMemoryPolicy::Preserve, - _ => SharedMemoryPolicy::Snapshot, - } -} - -#[cfg(feature = "vm")] -fn parse_nyx_environment_config( - v: &serde_json::Value, - observation_bits: usize, - reward_bits: usize, - agent_horizon: usize, - base_dir: &Path, -) -> anyhow::Result { - let vm = &v["vm_config"]; - if vm.is_null() { - return Err(anyhow::anyhow!("vm_config is required for environment=vm")); - } - - let firecracker_config = vm["firecracker_config"] - .as_str() - .or_else(|| vm["config"].as_str()) - .or_else(|| v["firecracker_config"].as_str()) - .ok_or_else(|| anyhow::anyhow!("vm_config.firecracker_config is required"))? - .to_string(); - - let instance_id = vm["instance_id"].as_str().unwrap_or("aixi-nyx").to_string(); - let shared_region_name = vm["shared_region_name"] - .as_str() - .unwrap_or("shared") - .to_string(); - let shared_region_size = vm["shared_region_size"].as_u64().unwrap_or(4096) as usize; - let shared_memory_policy = parse_shared_memory_policy( - vm["shared_memory_policy"] - .as_str() - .or_else(|| v["shared_memory_policy"].as_str()), - ); - - let step_timeout_ms = vm["step_timeout_ms"].as_u64().unwrap_or(100); - let boot_timeout_ms = vm["boot_timeout_ms"].as_u64().unwrap_or(30_000); - let episode_steps = vm["episode_steps"].as_u64().unwrap_or(agent_horizon as u64) as usize; - let step_cost = vm["step_cost"].as_i64().unwrap_or(1); - let debug_mode = vm["verbose"] - .as_bool() - .or_else(|| vm["debug"].as_bool()) - .unwrap_or(false); - - let protocol = parse_nyx_protocol_config(if !vm["protocol"].is_null() { - &vm["protocol"] - } else { - &v["vm_protocol"] - }); - let stats_backend = parse_vm_stats_backend( - if !vm["stats_backend"].is_null() { - &vm["stats_backend"] - } else { - &v["vm_stats_backend"] - }, - v, - base_dir, - )?; - let trace = parse_nyx_trace_config(if !vm["trace"].is_null() { - &vm["trace"] - } else { - &v["vm_trace"] - })?; - let action_source = parse_nyx_actions(if !vm["actions"].is_null() { - &vm["actions"] - } else { - &v["vm_actions"] - })?; - let observation_policy = parse_nyx_observation_policy(if !vm["observation"].is_null() { - &vm["observation"] - } else { - &v["vm_observation"] - }); - let observation_stream_len = - parse_observation_stream_len_for_vm(if !vm["observation"].is_null() { - &vm["observation"] - } else { - &v["vm_observation"] - }); - let observation_stream_mode = - parse_nyx_observation_stream_mode(if !vm["observation"].is_null() { - &vm["observation"] - } else { - &v["vm_observation"] - }); - let observation_stream_pad_byte = - parse_nyx_observation_pad_byte(if !vm["observation"].is_null() { - &vm["observation"] - } else { - &v["vm_observation"] - }); - let reward_policy = parse_nyx_reward_policy(if !vm["reward"].is_null() { - &vm["reward"] - } else { - &v["vm_reward"] - })?; - let reward_shaping = if !vm["reward_shaping"].is_null() { - parse_nyx_reward_shaping(&vm["reward_shaping"])? - } else if !v["vm_reward_shaping"].is_null() { - parse_nyx_reward_shaping(&v["vm_reward_shaping"])? - } else if !vm["reward"].is_null() && !vm["reward"]["shaping"].is_null() { - parse_nyx_reward_shaping(&vm["reward"]["shaping"])? - } else { - None - }; - let action_filter = parse_nyx_filter( - if !vm["filter"].is_null() { - &vm["filter"] - } else { - &v["vm_filter"] - }, - step_cost, - )?; - - Ok(NyxVmConfig { - firecracker_config, - instance_id, - shared_region_name, - shared_region_size, - shared_memory_policy, - step_timeout: Duration::from_millis(step_timeout_ms), - boot_timeout: Duration::from_millis(boot_timeout_ms), - episode_steps, - step_cost, - observation_policy, - observation_bits, - observation_stream_len, - observation_stream_mode, - observation_pad_byte: observation_stream_pad_byte, - reward_bits, - reward_policy, - reward_shaping, - action_source, - action_filter, - protocol, - stats_backend, - trace, - debug_mode, - crash_log: vm["crash_log"].as_str().map(|s| s.to_string()), - }) -} - -fn parse_vm_stats_backend( - cfg: &serde_json::Value, - root: &serde_json::Value, - base_dir: &Path, -) -> anyhow::Result { - let fallback = default_vm_stats_backend(root)?; - if cfg.is_null() { - return Ok(fallback); - } - - let name = cfg - .get("name") - .and_then(|v| v.as_str()) - .or_else(|| cfg.get("rate_backend").and_then(|v| v.as_str())) - .or_else(|| cfg.as_str()) - .unwrap_or("rosaplus"); - - let resolved = match infotheory::backends::resolve_rate_backend_name(name) { - Some(infotheory::backends::BackendAvailability::Enabled(name)) => name, - Some(infotheory::backends::BackendAvailability::Disabled { canonical, feature }) => { - return Err(anyhow::anyhow!( - "rate backend '{canonical}' requires infotheory feature '{feature}'" - )); - } - None => return Err(anyhow::anyhow!("unknown vm stats backend '{name}'")), - }; - - match resolved { - "rosaplus" => Ok(RateBackend::RosaPlus), - "ctw" => { - let depth = cfg["ct_depth"] - .as_u64() - .or_else(|| cfg["depth"].as_u64()) - .unwrap_or(32) as usize; - Ok(RateBackend::Ctw { depth }) - } - "fac-ctw" => { - let base_depth = cfg["base_depth"] - .as_u64() - .or_else(|| cfg["ct_depth"].as_u64()) - .unwrap_or(32) as usize; - let encoding_bits = cfg["encoding_bits"].as_u64().unwrap_or(8) as usize; - - // Fix: Default num_percept_bits to observation_bits + reward_bits if available, - // fallback to encoding_bits if not. - let obs_bits = root["observation_bits"].as_u64().unwrap_or(16); - let rew_bits = root["reward_bits"].as_u64().unwrap_or(8); - let default_percept_bits = obs_bits + rew_bits; - - let num_percept_bits = cfg["num_percept_bits"] - .as_u64() - .unwrap_or(default_percept_bits) as usize; - - Ok(RateBackend::FacCtw { - base_depth, - num_percept_bits, - encoding_bits, - }) - } - "mamba" => { - #[cfg(feature = "backend-mamba")] - { - if let Some(method) = cfg["method"] - .as_str() - .or_else(|| cfg["mamba_method"].as_str()) - { - Ok(RateBackend::MambaMethod { - method: method.to_string(), - }) - } else { - let path = cfg["mamba_model_path"] - .as_str() - .or_else(|| cfg["model_path"].as_str()) - .or_else(|| root["mamba_model_path"].as_str()) - .map(|s| s.to_string()) - .unwrap_or_else(mamba_model_path_from_env); - let model = load_mamba_model_from_path(&path); - Ok(RateBackend::Mamba { model }) - } - } - #[cfg(not(feature = "backend-mamba"))] - { - Err(anyhow::anyhow!( - "mamba stats backend requires 'backend-mamba' feature in infotheory" - )) - } - } - "rwkv7" => { - #[cfg(feature = "backend-rwkv")] - { - if let Some(method) = cfg["method"] - .as_str() - .or_else(|| cfg["rwkv_method"].as_str()) - { - Ok(RateBackend::Rwkv7Method { - method: method.to_string(), - }) - } else { - let path = cfg["rwkv_model_path"] - .as_str() - .or_else(|| cfg["model_path"].as_str()) - .or_else(|| root["rwkv_model_path"].as_str()) - .map(|s| s.to_string()) - .unwrap_or_else(rwkv7_model_path_from_env); - let model = load_rwkv7_model_from_path(&path); - Ok(RateBackend::Rwkv7 { model }) - } - } - #[cfg(not(feature = "backend-rwkv"))] - { - Err(anyhow::anyhow!( - "rwkv7 stats backend requires 'backend-rwkv' feature in infotheory" - )) - } - } - "zpaq" => { - let method = cfg["method"] - .as_str() - .or_else(|| cfg["zpaq_method"].as_str()) - .or_else(|| root["method"].as_str()) - .unwrap_or("2") - .to_string(); - if let Err(err) = validate_zpaq_rate_method(&method) { - return Err(anyhow::anyhow!( - "unsupported ZPAQ rate method '{method}': {err}" - )); - } - Ok(RateBackend::Zpaq { method }) - } - "match" => Ok(RateBackend::Match { - hash_bits: cfg["hash_bits"].as_u64().unwrap_or(20) as usize, - min_len: cfg["min_len"].as_u64().unwrap_or(4) as usize, - max_len: cfg["max_len"].as_u64().unwrap_or(255) as usize, - base_mix: cfg["base_mix"].as_f64().unwrap_or(0.02), - confidence_scale: cfg["confidence_scale"].as_f64().unwrap_or(1.0), - }), - "sparse-match" => Ok(RateBackend::SparseMatch { - hash_bits: cfg["hash_bits"].as_u64().unwrap_or(19) as usize, - min_len: cfg["min_len"].as_u64().unwrap_or(3) as usize, - max_len: cfg["max_len"].as_u64().unwrap_or(64) as usize, - gap_min: cfg["gap_min"].as_u64().unwrap_or(1) as usize, - gap_max: cfg["gap_max"].as_u64().unwrap_or(2) as usize, - base_mix: cfg["base_mix"].as_f64().unwrap_or(0.05), - confidence_scale: cfg["confidence_scale"].as_f64().unwrap_or(1.0), - }), - "ppmd" => Ok(RateBackend::Ppmd { - order: cfg["order"].as_u64().unwrap_or(10) as usize, - memory_mb: cfg["memory_mb"].as_u64().unwrap_or(64) as usize, - }), - "sequitur" => Ok(RateBackend::Sequitur { - context_bytes: cfg["context_bytes"].as_u64().unwrap_or(64) as usize, - }), - "mixture" => { - let spec = if let Some(spec_v) = cfg.get("spec").filter(|value| value.is_object()) { - parse_mixture_spec_value(spec_v, base_dir, MAX_MIXTURE_NESTING)? - } else if let Some(path) = cfg["mixture_spec"] - .as_str() - .or_else(|| cfg["spec_path"].as_str()) - .or_else(|| cfg["spec"].as_str()) - .or_else(|| root["mixture_spec"].as_str()) - { - let full = base_dir.join(path); - load_mixture_spec(full.to_str().unwrap_or(path))? - } else { - return Err(anyhow::anyhow!( - "mixture stats backend requires inline 'spec' or 'mixture_spec' path" - )); - }; - Ok(RateBackend::Mixture { - spec: Arc::new(spec), - }) - } - "particle" => { - let spec = if let Some(spec_v) = cfg.get("spec").filter(|value| value.is_object()) { - parse_particle_spec_value(spec_v)? - } else if let Some(path) = cfg["particle_spec"] - .as_str() - .or_else(|| cfg["spec_path"].as_str()) - .or_else(|| cfg["spec"].as_str()) - .or_else(|| root["particle_spec"].as_str()) - { - let full = base_dir.join(path); - load_particle_spec(full.to_str().unwrap_or(path))? - } else { - parse_particle_spec_value(cfg)? - }; - spec.validate() - .map_err(|e| anyhow::anyhow!("invalid particle spec: {e}"))?; - Ok(RateBackend::Particle { - spec: Arc::new(spec), - }) - } - "calibrated" => { - let spec = if let Some(spec_v) = cfg.get("spec").filter(|value| value.is_object()) { - parse_calibrated_spec_value(spec_v, base_dir, 4)? - } else if let Some(path) = cfg["calibrated_spec"] - .as_str() - .or_else(|| cfg["spec_path"].as_str()) - .or_else(|| cfg["spec"].as_str()) - .or_else(|| root["calibrated_spec"].as_str()) - { - let full = base_dir.join(path); - load_calibrated_spec(full.to_str().unwrap_or(path))? - } else { - parse_calibrated_spec_value(cfg, base_dir, 4)? - }; - Ok(RateBackend::Calibrated { - spec: Arc::new(spec), - }) - } - other => Err(anyhow::anyhow!("unsupported vm stats backend '{other}'")), - } -} - -fn default_vm_stats_backend(root: &serde_json::Value) -> anyhow::Result { - let algo = root["algorithm"].as_str().unwrap_or("ctw"); - let ct_depth = root["ct_depth"].as_u64().unwrap_or(20) as usize; - match algo { - "ctw" | "ac-ctw" | "ctw-context-tree" => Ok(RateBackend::Ctw { depth: ct_depth }), - "fac-ctw" => Ok(RateBackend::FacCtw { - base_depth: ct_depth, - num_percept_bits: 8, - encoding_bits: 8, - }), - "sequitur" => Ok(RateBackend::Sequitur { - context_bytes: root["context_bytes"].as_u64().unwrap_or(64) as usize, - }), - "mamba" | "mamba1" => { - #[cfg(feature = "backend-mamba")] - { - let path = root["mamba_model_path"] - .as_str() - .map(|s| s.to_string()) - .unwrap_or_else(mamba_model_path_from_env); - let model = load_mamba_model_from_path(&path); - Ok(RateBackend::Mamba { model }) - } - #[cfg(not(feature = "backend-mamba"))] - { - Err(anyhow::anyhow!( - "mamba default stats backend requires 'backend-mamba' feature in infotheory" - )) - } - } - "rosa" | "rosaplus" => Ok(RateBackend::RosaPlus), - "rwkv" | "rwkv7" => { - #[cfg(feature = "backend-rwkv")] - { - let path = root["rwkv_model_path"] - .as_str() - .map(|s| s.to_string()) - .unwrap_or_else(rwkv7_model_path_from_env); - let model = load_rwkv7_model_from_path(&path); - Ok(RateBackend::Rwkv7 { model }) - } - #[cfg(not(feature = "backend-rwkv"))] - { - Err(anyhow::anyhow!( - "rwkv7 default stats backend requires 'backend-rwkv' feature in infotheory" - )) - } - } - "zpaq" => Ok(RateBackend::Zpaq { - method: { - let method = root["method"].as_str().unwrap_or("2").to_string(); - if let Err(err) = validate_zpaq_rate_method(&method) { - return Err(anyhow::anyhow!( - "unsupported ZPAQ rate method '{method}': {err}" - )); - } - method - }, - }), - "mixture" | "mix" => { - let spec_path = root["mixture_spec"] - .as_str() - .ok_or_else(|| anyhow::anyhow!("mixture stats backend requires mixture_spec"))?; - let spec = load_mixture_spec(spec_path)?; - Ok(RateBackend::Mixture { - spec: Arc::new(spec), - }) - } - _ => Ok(RateBackend::RosaPlus), - } -} - -#[cfg(feature = "vm")] -fn parse_nyx_trace_config(v: &serde_json::Value) -> anyhow::Result> { - if v.is_null() { - return Ok(None); - } - let max_bytes = v["max_bytes"].as_u64().unwrap_or(1_000_000) as usize; - let reset_on_episode = v["reset_on_episode"].as_bool().unwrap_or(false); - let shared_region_name = v["shared_region_name"] - .as_str() - .or_else(|| v["shared_region"].as_str()) - .or_else(|| v["name"].as_str()) - .or_else(|| { - if v["mode"].as_str() == Some("shared-memory") { - Some("trace") - } else { - None - } - }) - .map(|s| s.to_string()) - .or(Some("trace".to_string())); - - Ok(Some(NyxTraceConfig { - shared_region_name, - max_bytes, - reset_on_episode, - })) -} - -#[cfg(feature = "vm")] -fn parse_nyx_protocol_config(v: &serde_json::Value) -> NyxProtocolConfig { - let mut cfg = NyxProtocolConfig::default(); - if let Some(s) = v["action_prefix"].as_str() { - cfg.action_prefix = s.to_string(); - } - if let Some(s) = v["action_suffix"].as_str() { - cfg.action_suffix = s.to_string(); - } - if let Some(s) = v["obs_prefix"].as_str() { - cfg.obs_prefix = s.to_string(); - } - if let Some(s) = v["rew_prefix"].as_str() { - cfg.rew_prefix = s.to_string(); - } - if let Some(s) = v["done_prefix"].as_str() { - cfg.done_prefix = s.to_string(); - } - if let Some(s) = v["data_prefix"].as_str() { - cfg.data_prefix = s.to_string(); - } - if let Some(s) = v["wire_encoding"].as_str() { - if let Some(enc) = NyxPayloadEncoding::parse(s) { - cfg.wire_encoding = enc; - } - } - cfg -} - -#[cfg(feature = "vm")] -fn parse_nyx_actions(v: &serde_json::Value) -> anyhow::Result { - let mode = v["mode"].as_str().unwrap_or("literal"); - match mode { - "fuzz" => { - let fuzz = if v["fuzz"].is_null() { v } else { &v["fuzz"] }; - let seed_encoding = - NyxPayloadEncoding::parse(fuzz["seed_encoding"].as_str().unwrap_or("utf8")) - .unwrap_or(NyxPayloadEncoding::Utf8); - let mut seeds = Vec::new(); - if let Some(arr) = fuzz["seed_paths"].as_array() { - for item in arr { - if let Some(path) = item.as_str() { - let data = std::fs::read(path)?; - seeds.push(data); - } - } - } - if let Some(arr) = fuzz["seed_inputs"].as_array() { - for item in arr { - if let Some(text) = item.as_str() { - seeds.push(seed_encoding.decode(text)?); - } - } - } - - let mut mutators = Vec::new(); - if let Some(arr) = fuzz["mutators"].as_array() { - for item in arr { - if let Some(name) = item.as_str() { - if let Some(m) = parse_nyx_fuzz_mutator(name) { - mutators.push(m); - } - } - } - } - let min_len = fuzz["min_len"].as_u64().unwrap_or(1) as usize; - let max_len = fuzz["max_len"].as_u64().unwrap_or(4096) as usize; - let dict_encoding = - NyxPayloadEncoding::parse(fuzz["dict_encoding"].as_str().unwrap_or("utf8")) - .unwrap_or(NyxPayloadEncoding::Utf8); - let mut dictionary = Vec::new(); - if let Some(arr) = fuzz["dictionary"].as_array() { - for item in arr { - if let Some(text) = item.as_str() { - dictionary.push(dict_encoding.decode(text)?); - } - } - } - let rng_seed = fuzz["rng_seed"].as_u64().unwrap_or(0); - Ok(NyxActionSource::Fuzz(NyxFuzzConfig { - seeds, - mutators, - min_len, - max_len, - dictionary, - rng_seed, - })) - } - _ => { - let mut actions = Vec::new(); - if let Some(arr) = v["actions"].as_array() { - for item in arr { - if let Some(text) = item.as_str() { - let payload = NyxPayloadEncoding::Utf8.decode(text)?; - actions.push(NyxActionSpec { - name: None, - payload, - }); - continue; - } - let payload = item["payload"].as_str().unwrap_or_default(); - let encoding = - NyxPayloadEncoding::parse(item["encoding"].as_str().unwrap_or("utf8")) - .unwrap_or(NyxPayloadEncoding::Utf8); - let payload = encoding.decode(payload)?; - let name = item["name"].as_str().map(|s| s.to_string()); - actions.push(NyxActionSpec { name, payload }); - } - } - Ok(NyxActionSource::Literal(actions)) - } - } -} - -#[cfg(feature = "vm")] -#[cfg(feature = "vm")] -fn parse_nyx_fuzz_mutator(name: &str) -> Option { - match name { - "flip_bit" | "flipbit" => Some(NyxFuzzMutator::FlipBit), - "flip_byte" | "flipbyte" => Some(NyxFuzzMutator::FlipByte), - "insert" | "insert_byte" => Some(NyxFuzzMutator::InsertByte), - "delete" | "delete_byte" => Some(NyxFuzzMutator::DeleteByte), - "splice" | "splice_seed" => Some(NyxFuzzMutator::SpliceSeed), - "reset" | "reset_seed" => Some(NyxFuzzMutator::ResetSeed), - "havoc" => Some(NyxFuzzMutator::Havoc), - _ => None, - } -} - -#[cfg(feature = "vm")] -fn parse_nyx_observation_policy(v: &serde_json::Value) -> NyxObservationPolicy { - match v["mode"].as_str().unwrap_or("guest") { - "raw" | "raw-bytes" | "bytes" | "stream" => NyxObservationPolicy::RawOutput, - "hash" | "output-hash" => NyxObservationPolicy::OutputHash, - "shared-memory" | "shared_mem" | "shared" => NyxObservationPolicy::SharedMemory, - _ => NyxObservationPolicy::FromGuest, - } -} - -fn parse_observation_stream_len(v: &serde_json::Value) -> usize { - v["observation_stream_len"].as_u64().unwrap_or(1) as usize -} - -fn parse_observation_key_mode(v: &serde_json::Value) -> ObservationKeyMode { - parse_observation_key_mode_str(v["observation_key_mode"].as_str().unwrap_or("full")) -} - -fn parse_observation_key_mode_str(s: &str) -> ObservationKeyMode { - match s { - "full" | "full-stream" | "stream" => ObservationKeyMode::FullStream, - "last" => ObservationKeyMode::Last, - "hash" | "stream-hash" => ObservationKeyMode::StreamHash, - _ => ObservationKeyMode::First, - } -} - -fn parse_observation_stream_len_for_env(v: &serde_json::Value, env_name: &str) -> usize { - if env_name == "vm" || env_name == "nyx" || env_name == "nyx-vm" { - if v["vm_observation"].is_null() { - parse_observation_stream_len(v) - } else { - parse_observation_stream_len_for_vm(&v["vm_observation"]) - } - } else { - parse_observation_stream_len(v) - } -} - -fn parse_observation_key_mode_for_env(v: &serde_json::Value, env_name: &str) -> ObservationKeyMode { - if env_name == "vm" || env_name == "nyx" || env_name == "nyx-vm" { - if v["vm_observation"].is_null() { - parse_observation_key_mode(v) - } else { - parse_observation_key_mode_for_vm(&v["vm_observation"]) - } - } else { - parse_observation_key_mode(v) - } -} - -fn parse_observation_key_mode_for_vm(v: &serde_json::Value) -> ObservationKeyMode { - if v.is_null() { - return ObservationKeyMode::FullStream; - } - parse_observation_key_mode_str( - v["key_mode"] - .as_str() - .unwrap_or_else(|| v["observation_key_mode"].as_str().unwrap_or("full")), - ) -} - -fn parse_observation_stream_len_for_vm(v: &serde_json::Value) -> usize { - if v.is_null() { - return 1; - } - v["stream_len"] - .as_u64() - .or_else(|| v["observation_stream_len"].as_u64()) - .unwrap_or(1) as usize -} - -#[cfg(feature = "vm")] -fn parse_nyx_observation_stream_mode(v: &serde_json::Value) -> NyxObservationStreamMode { - match v["stream_mode"].as_str().unwrap_or("pad-truncate") { - "pad" => NyxObservationStreamMode::Pad, - "truncate" => NyxObservationStreamMode::Truncate, - _ => NyxObservationStreamMode::PadTruncate, - } -} - -#[cfg(feature = "vm")] -fn parse_nyx_observation_pad_byte(v: &serde_json::Value) -> u8 { - v["pad_byte"].as_u64().unwrap_or(0) as u8 -} - -fn extract_observation_stream_len_raw(v: &serde_json::Value) -> Option { - v["observation_stream_len"].as_u64().map(|n| n as usize) -} - -fn extract_vm_observation_stream_len_raw(v: &serde_json::Value) -> Option { - if v.is_null() { - return None; - } - v["stream_len"] - .as_u64() - .or_else(|| v["observation_stream_len"].as_u64()) - .map(|n| n as usize) -} - -fn extract_observation_key_mode_raw(v: &serde_json::Value) -> Option { - v["observation_key_mode"] - .as_str() - .map(parse_observation_key_mode_str) -} - -fn extract_vm_observation_key_mode_raw(v: &serde_json::Value) -> Option { - if v.is_null() { - return None; - } - v["key_mode"] - .as_str() - .or_else(|| v["observation_key_mode"].as_str()) - .map(parse_observation_key_mode_str) -} - -fn validate_observation_config( - env_name: &str, - v: &serde_json::Value, - observation_stream_len: usize, - observation_key_mode: ObservationKeyMode, -) -> anyhow::Result<()> { - if observation_stream_len == 0 { - return Err(anyhow::anyhow!("observation_stream_len must be > 0")); - } - if env_name == "vm" || env_name == "nyx" || env_name == "nyx-vm" { - if let (Some(top_len), Some(vm_len)) = ( - extract_observation_stream_len_raw(v), - extract_vm_observation_stream_len_raw(&v["vm_observation"]), - ) && top_len != vm_len - { - return Err(anyhow::anyhow!( - "observation_stream_len ({}) conflicts with vm_observation.stream_len ({})", - top_len, - vm_len - )); - } - if let (Some(top_mode), Some(vm_mode)) = ( - extract_observation_key_mode_raw(v), - extract_vm_observation_key_mode_raw(&v["vm_observation"]), - ) && top_mode != vm_mode - { - return Err(anyhow::anyhow!( - "observation_key_mode ({:?}) conflicts with vm_observation.key_mode ({:?})", - top_mode, - vm_mode - )); - } - } - if observation_stream_len > 1 && matches!(observation_key_mode, ObservationKeyMode::First) { - eprintln!( - "Warning: observation_key_mode=first collapses multi-symbol observation streams; prefer \"full\" for paper-accurate expectimax." - ); - } - if observation_stream_len > 1 && !matches!(observation_key_mode, ObservationKeyMode::FullStream) - { - eprintln!( - "Warning: observation_key_mode {:?} reduces multi-symbol observation streams and deviates from paper-accurate expectimax.", - observation_key_mode - ); - } - Ok(()) -} - -/// Validates that the actual observation stream length matches the configured value. -/// -/// This is a hard error to prevent FAC-CTW bit cycling desynchronization. -fn validate_obs_stream_len(expected: usize, actual: usize) -> anyhow::Result<()> { - if actual != expected { - return Err(anyhow::anyhow!( - "Observation stream length mismatch: config expects {} symbols, but environment returned {}. \ - This causes FAC-CTW bit cycling desynchronization. \ - Fix your `observation_stream_len` config or environment implementation.", - expected, - actual - )); - } - Ok(()) -} - -fn aiqi_backend_label(config: &AiqiConfig) -> String { - if let Some(rate_backend) = &config.rate_backend { - let name = infotheory::mixture::RateBackendPredictor::default_name( - rate_backend, - config.rate_backend_max_order, - ); - format!("rate_backend={name}") - } else { - format!("algorithm={}", config.algorithm) - } -} - -#[cfg(feature = "vm")] -fn parse_nyx_reward_policy(v: &serde_json::Value) -> anyhow::Result { - match v["mode"].as_str().unwrap_or("guest") { - "pattern" => { - let pattern = v["pattern"] - .as_str() - .ok_or_else(|| anyhow::anyhow!("vm_reward.pattern is required"))? - .to_string(); - let base_reward = v["base_reward"].as_i64().unwrap_or(0); - let bonus_reward = v["bonus_reward"].as_i64().unwrap_or(10); - Ok(NyxRewardPolicy::Pattern { - pattern, - base_reward, - bonus_reward, - }) - } - _ => Ok(NyxRewardPolicy::FromGuest), - } -} - -#[cfg(feature = "vm")] -fn parse_nyx_reward_shaping(v: &serde_json::Value) -> anyhow::Result> { - if v.is_null() { - return Ok(None); - } - match v["mode"].as_str().unwrap_or("none") { - "entropy-reduction" | "entropy_reduction" => { - let baseline_path = v["baseline_path"] - .as_str() - .ok_or_else(|| anyhow::anyhow!("vm_reward_shaping.baseline_path is required"))?; - let baseline_bytes = std::fs::read(baseline_path)?; - let max_order = v["max_order"].as_i64().unwrap_or(8); - let scale = v["scale"].as_f64().unwrap_or(10.0); - let crash_bonus = v["crash_bonus"].as_i64(); - let timeout_bonus = v["timeout_bonus"].as_i64(); - Ok(Some(NyxRewardShaping::EntropyReduction { - baseline_bytes, - max_order, - scale, - crash_bonus, - timeout_bonus, - })) - } - "trace-entropy" | "trace_entropy" => { - let max_order = v["max_order"].as_i64().unwrap_or(8); - let scale = v["scale"].as_f64().unwrap_or(1.0); - let normalize = v["normalize"].as_bool().unwrap_or(false); - Ok(Some(NyxRewardShaping::TraceEntropy { - max_order, - scale, - normalize, - })) - } - "none" | "off" => Ok(None), - _ => Ok(None), - } -} - -#[cfg(feature = "vm")] -fn parse_nyx_filter( - v: &serde_json::Value, - step_cost: i64, -) -> anyhow::Result> { - if v.is_null() { - return Ok(None); - } - let novelty_prior = if let Some(path) = v["novelty_prior_path"].as_str() { - Some(std::fs::read(path)?) - } else { - None - }; - let reject_reward = v["reject_reward"].as_i64().or_else(|| Some(-step_cost)); - Ok(Some(NyxActionFilter { - min_entropy: v["min_entropy"].as_f64(), - max_entropy: v["max_entropy"].as_f64(), - min_intrinsic_dependence: v["min_intrinsic_dependence"].as_f64(), - min_novelty: v["min_novelty"].as_f64(), - novelty_prior, - max_order: v["max_order"].as_i64().unwrap_or(8), - reject_reward, - })) -} - -struct BuiltCtx { - ctx: InfotheoryCtx, - expert_spec_max_order: Option, -} - -fn build_ctx( - rate_backend: &str, - compression_backend: &str, - method: Option<&str>, - expert_spec_path: Option<&str>, -) -> BuiltCtx { - let (rate_backend, expert_spec_max_order) = if let Some(path) = expert_spec_path { - let spec = load_expert_spec(path).unwrap_or_else(|e| { - eprintln!("Error: failed to load expert spec '{path}': {e}"); - std::process::exit(1); - }); - (spec.backend, Some(spec.max_order)) - } else { - ( - match rate_backend { - "mamba" => { - #[cfg(feature = "backend-mamba")] - { - if let Some(m) = method { - RateBackend::MambaMethod { - method: m.to_string(), - } - } else { - let p = mamba_model_path_from_env(); - let model = load_mamba_model_from_path(&p); - RateBackend::Mamba { model } - } - } - #[cfg(not(feature = "backend-mamba"))] - { - eprintln!( - "Error: rate backend 'mamba' requires infotheory built with feature 'backend-mamba'" - ); - std::process::exit(1); - } - } - "rwkv7" => { - #[cfg(feature = "backend-rwkv")] - { - if let Some(m) = method { - RateBackend::Rwkv7Method { - method: m.to_string(), - } - } else { - let p = rwkv7_model_path_from_env(); - let model = load_rwkv7_model_from_path(&p); - RateBackend::Rwkv7 { model } - } - } - #[cfg(not(feature = "backend-rwkv"))] - { - eprintln!( - "Error: rate backend 'rwkv7' requires infotheory built with feature 'backend-rwkv'" - ); - std::process::exit(1); - } - } - "match" => RateBackend::Match { - hash_bits: 20, - min_len: 4, - max_len: 255, - base_mix: 0.02, - confidence_scale: 1.0, - }, - "sparse-match" => RateBackend::SparseMatch { - hash_bits: 19, - min_len: 3, - max_len: 64, - gap_min: 1, - gap_max: 2, - base_mix: 0.05, - confidence_scale: 1.0, - }, - "ppmd" => RateBackend::Ppmd { - order: method.and_then(|m| m.parse::().ok()).unwrap_or(10), - memory_mb: 64, - }, - "sequitur" => RateBackend::Sequitur { - context_bytes: method.and_then(|m| m.parse::().ok()).unwrap_or(64), - }, - "ctw" => { - let depth = if let Some(m) = method { - m.parse::().unwrap_or(20) - } else { - 20 - }; - RateBackend::Ctw { depth } - } - "fac-ctw" => { - let depth = if let Some(m) = method { - m.parse::().unwrap_or(20) - } else { - 20 - }; - RateBackend::FacCtw { - base_depth: depth, - num_percept_bits: 8, // Default for byte-oriented CLI - encoding_bits: 8, // Default for byte-oriented CLI - } - } - "zpaq" => { - let m = method.unwrap_or("2").to_string(); - if let Err(err) = validate_zpaq_rate_method(&m) { - eprintln!("Error: unsupported ZPAQ rate method '{m}': {err}"); - std::process::exit(1); - } - RateBackend::Zpaq { method: m } - } - "mixture" => { - let path = method.unwrap_or_else(|| { - eprintln!("Error: --rate-backend mixture requires --method "); - std::process::exit(1); - }); - let spec = load_mixture_spec(path).unwrap_or_else(|e| { - eprintln!("Error: failed to load mixture spec '{path}': {e}"); - std::process::exit(1); - }); - RateBackend::Mixture { - spec: Arc::new(spec), - } - } - "particle" => { - let path = method.unwrap_or_else(|| { - eprintln!("Error: --rate-backend particle requires --method "); - std::process::exit(1); - }); - let spec = load_particle_spec(path).unwrap_or_else(|e| { - eprintln!("Error: failed to load particle spec '{path}': {e}"); - std::process::exit(1); - }); - RateBackend::Particle { - spec: Arc::new(spec), - } - } - "calibrated" => { - let path = method.unwrap_or_else(|| { - eprintln!("Error: --rate-backend calibrated requires --method "); - std::process::exit(1); - }); - let spec = load_calibrated_spec(path).unwrap_or_else(|e| { - eprintln!("Error: failed to load calibrated spec '{path}': {e}"); - std::process::exit(1); - }); - RateBackend::Calibrated { - spec: Arc::new(spec), - } - } - _ => RateBackend::RosaPlus, - }, - None, - ) - }; - - let compression_backend = match compression_backend { - "rwkv7" => { - #[cfg(feature = "backend-rwkv")] - { - match method { - // Legacy: allow passing only coder (ac/rans), model path from env. - Some(m) if infotheory::backends::parse_rwkv7_coder(m).is_some() => { - let model_path = rwkv7_model_path_from_env(); - let model = load_rwkv7_model_from_path(&model_path); - let coder = infotheory::backends::parse_rwkv7_coder(m) - .unwrap_or(rwkvzip::CoderType::AC); - CompressionBackend::Rwkv7 { model, coder } - } - // Method-based RWKV config: file:/... or cfg:... - Some(m) => match rwkvzip::parse_method_spec(m) { - Ok(rwkvzip::MethodSpec::File { path, policy: None }) => { - let model = load_rwkv7_model_from_path(path.to_string_lossy().as_ref()); - CompressionBackend::Rwkv7 { - model, - coder: rwkvzip::CoderType::AC, - } - } - Ok(rwkvzip::MethodSpec::File { - policy: Some(_), .. - }) - | Ok(rwkvzip::MethodSpec::Online { .. }) => CompressionBackend::Rate { - rate_backend: RateBackend::Rwkv7Method { - method: m.to_string(), - }, - coder: rwkvzip::CoderType::AC, - framing: infotheory::compression::FramingMode::Raw, - }, - Err(err) => { - eprintln!( - "Error: invalid RWKV method for --compression-backend rwkv7: {err}" - ); - std::process::exit(1); - } - }, - // No method: model path from env. - None => { - let model_path = rwkv7_model_path_from_env(); - let model = load_rwkv7_model_from_path(&model_path); - CompressionBackend::Rwkv7 { - model, - coder: rwkvzip::CoderType::AC, - } - } - } - } - #[cfg(not(feature = "backend-rwkv"))] - { - eprintln!( - "Error: compression backend 'rwkv7' requires infotheory built with feature 'backend-rwkv'" - ); - std::process::exit(1); - } - } - "rate-ac" => CompressionBackend::Rate { - rate_backend: rate_backend.clone(), - coder: infotheory::coders::CoderType::AC, - framing: infotheory::compression::FramingMode::Raw, - }, - "rate-rans" => CompressionBackend::Rate { - rate_backend: rate_backend.clone(), - coder: infotheory::coders::CoderType::RANS, - framing: infotheory::compression::FramingMode::Raw, - }, - _ => { - let m = method.unwrap_or("5").to_string(); - CompressionBackend::Zpaq { method: m } - } - }; - - BuiltCtx { - ctx: InfotheoryCtx::new(rate_backend, compression_backend), - expert_spec_max_order, - } -} - -fn read_file(path: &str) -> Vec { - match std::fs::read(path) { - Ok(data) => data, - Err(e) => { - eprintln!("Error reading file '{}': {}", path, e); - std::process::exit(1); - } - } -} - -fn parse_hex_bytes(raw: &str) -> anyhow::Result> { - fn nibble(byte: u8) -> anyhow::Result { - match byte { - b'0'..=b'9' => Ok(byte - b'0'), - b'a'..=b'f' => Ok(byte - b'a' + 10), - b'A'..=b'F' => Ok(byte - b'A' + 10), - _ => Err(anyhow::anyhow!("invalid hex digit '{}'", byte as char)), - } - } - - let cleaned: Vec = raw - .bytes() - .filter(|b| !matches!(b, b' ' | b'\n' | b'\r' | b'\t' | b'_')) - .collect(); - if cleaned.len() % 2 != 0 { - return Err(anyhow::anyhow!( - "hex input must have an even number of digits" - )); - } - let mut out = Vec::with_capacity(cleaned.len() / 2); - let mut i = 0usize; - while i < cleaned.len() { - let hi = nibble(cleaned[i])?; - let lo = nibble(cleaned[i + 1])?; - out.push((hi << 4) | lo); - i += 2; - } - Ok(out) -} - -fn bytes_to_hex(bytes: &[u8]) -> String { - const HEX: &[u8; 16] = b"0123456789abcdef"; - let mut out = String::with_capacity(bytes.len() * 2); - for &byte in bytes { - out.push(HEX[(byte >> 4) as usize] as char); - out.push(HEX[(byte & 0x0F) as usize] as char); - } - out -} - -fn read_stdin_all_for_generate() -> Vec { - let stdin = io::stdin(); - if stdin.is_terminal() { - eprintln!("Error: 'generate' requires or piped stdin"); - std::process::exit(1); - } - let mut data = Vec::new(); - if let Err(e) = stdin.lock().read_to_end(&mut data) { - eprintln!("Error reading stdin: {e}"); - std::process::exit(1); - } - data -} - -fn file_roundtrip_backend(backend: &CompressionBackend) -> CompressionBackend { - match backend { - CompressionBackend::Rate { - rate_backend, - coder, - .. - } => CompressionBackend::Rate { - rate_backend: rate_backend.clone(), - coder: *coder, - framing: infotheory::compression::FramingMode::Framed, - }, - _ => backend.clone(), - } -} - -fn maybe_export_online_model( - export_path: Option<&str>, - _ctx: &InfotheoryCtx, - _parts: &[&[u8]], -) -> anyhow::Result<()> { - let Some(_path) = export_path else { - return Ok(()); - }; - - #[cfg(feature = "backend-rwkv")] - { - let rwkv_method = match &_ctx.rate_backend { - RateBackend::Rwkv7Method { method } => Some(method.as_str()), - _ => match &_ctx.compression_backend { - CompressionBackend::Rate { - rate_backend: RateBackend::Rwkv7Method { method }, - .. - } => Some(method.as_str()), - _ => None, - }, - }; - if let Some(method) = rwkv_method { - let mut compressor = rwkvzip::Compressor::new_from_method(method)?; - let _ = compressor.compress_size_chain(_parts, infotheory::coders::CoderType::AC)?; - compressor.export_online(_path)?; - return Ok(()); - } - // Pre-loaded model Arc variants do not carry a method string and cannot - // replay training for export. Warn the user explicitly. - let has_rwkv_model = matches!(&_ctx.rate_backend, RateBackend::Rwkv7 { .. }) - || matches!( - &_ctx.compression_backend, - CompressionBackend::Rate { - rate_backend: RateBackend::Rwkv7 { .. }, - .. - } | CompressionBackend::Rwkv7 { .. } - ); - if has_rwkv_model { - eprintln!( - "Warning: --model-export is not supported for pre-loaded RWKV7 model backends. \ - Use --method with a cfg:/file: spec to enable online model export." - ); - return Ok(()); - } - } - - #[cfg(feature = "backend-mamba")] - { - let mamba_method = match &_ctx.rate_backend { - RateBackend::MambaMethod { method } => Some(method.as_str()), - _ => match &_ctx.compression_backend { - CompressionBackend::Rate { - rate_backend: RateBackend::MambaMethod { method }, - .. - } => Some(method.as_str()), - _ => None, - }, - }; - if let Some(method) = mamba_method { - let mut compressor = mambazip::Compressor::new_from_method(method)?; - let _ = compressor.compress_size_chain(_parts, infotheory::coders::CoderType::AC)?; - compressor.export_online(_path)?; - return Ok(()); - } - let has_mamba_model = matches!(&_ctx.rate_backend, RateBackend::Mamba { .. }) - || matches!( - &_ctx.compression_backend, - CompressionBackend::Rate { - rate_backend: RateBackend::Mamba { .. }, - .. - } - ); - if has_mamba_model { - eprintln!( - "Warning: --model-export is not supported for pre-loaded Mamba model backends. \ - Use --method with a cfg: spec to enable online model export." - ); - return Ok(()); - } - } - - eprintln!( - "Warning: --model-export was requested but the current backend does not support \ - online model export. Only RWKV7 and Mamba method-based backends support export." - ); - Ok(()) -} - -// ============================================================ -// Batch JSON Mode - For programmatic use from Python -// ============================================================ - -/// ROSA-based symmetric codelength distance (NCD-like but faster) -/// d_ROSA(x,y) = 0.5 * (H_y(x)/H_x(x) + H_x(y)/H_y(y)) - 1 -/// Clamped to [0, 1] -fn rosa_distance(x: &[u8], y: &[u8], max_order: i64) -> f64 { - if x.is_empty() || y.is_empty() { - return 1.0; - } - - // Self-entropy rates (biased/plugin estimator for consistency) - let h_x_x = biased_entropy_rate_bytes(x, max_order); - let h_y_y = biased_entropy_rate_bytes(y, max_order); - - // Cross-entropy rates - let h_y_x = cross_entropy_rate_bytes(x, y, max_order); // score x under model trained on y - let h_x_y = cross_entropy_rate_bytes(y, x, max_order); // score y under model trained on x - - // Avoid division by zero - if h_x_x < 1e-9 || h_y_y < 1e-9 { - return 1.0; - } - - let d = 0.5 * (h_y_x / h_x_x + h_x_y / h_y_y) - 1.0; - d.clamp(0.0, 1.0) -} - -/// Process a single JSON line and return result -fn process_json_line(line: &str) -> String { - let line = line.trim(); - if line.is_empty() { - return r#"{"error":"empty input"}"#.to_string(); - } - - let v: serde_json::Value = match serde_json::from_str(line) { - Ok(v) => v, - Err(e) => { - return serde_json::json!({ - "error": format!("invalid json: {e}") - }) - .to_string(); - } - }; - let op = v.get("op").and_then(|x| x.as_str()).unwrap_or(""); - - match op { - "metrics" => { - // Single text metrics: H0, H_rate, ID - let text = v - .get("text") - .and_then(|x| x.as_str()) - .unwrap_or_default() - .to_string(); - let max_order = v.get("max_order").and_then(|x| x.as_i64()).unwrap_or(-1); - let data = text.as_bytes(); - - if data.is_empty() { - return r#"{"error":"empty text"}"#.to_string(); - } - - let h0 = marginal_entropy_bytes(data); - let h_rate = entropy_rate_bytes(data, max_order); - let id = if h0 < 1e-9 { 0.0 } else { ((h0 - h_rate) / h0).clamp(0.0, 1.0) }; - - format!( - r#"{{"h0":{:.6},"h_rate":{:.6},"id":{:.6},"len":{}}}"#, - h0, h_rate, id, data.len() - ) - } - - "metrics_file" => { - // File-based metrics - let path = v - .get("path") - .and_then(|x| x.as_str()) - .unwrap_or_default() - .to_string(); - let max_order = v.get("max_order").and_then(|x| x.as_i64()).unwrap_or(-1); - - match std::fs::read(&path) { - Ok(data) => { - let h0 = marginal_entropy_bytes(&data); - let h_rate = entropy_rate_bytes(&data, max_order); - let id = if h0 < 1e-9 { 0.0 } else { ((h0 - h_rate) / h0).clamp(0.0, 1.0) }; - - format!( - r#"{{"h0":{:.6},"h_rate":{:.6},"id":{:.6},"len":{}}}"#, - h0, h_rate, id, data.len() - ) - } - Err(e) => format!(r#"{{"error":"failed to read file: {}"}}"#, e), - } - } - - "ncd" => { - // NCD between two texts - let text1 = v - .get("text1") - .and_then(|x| x.as_str()) - .unwrap_or_default() - .to_string(); - let text2 = v - .get("text2") - .and_then(|x| x.as_str()) - .unwrap_or_default() - .to_string(); - let method = v - .get("method") - .and_then(|x| x.as_str()) - .unwrap_or("5") - .to_string(); - let variant = v - .get("variant") - .and_then(|x| x.as_str()) - .unwrap_or("vitanyi") - .to_string(); - - let x = text1.as_bytes(); - let y = text2.as_bytes(); - - if x.is_empty() || y.is_empty() { - return r#"{"error":"empty text(s)"}"#.to_string(); - } - - let ncd_variant = match variant.as_str() { - "sym" | "sym_vitanyi" => NcdVariant::SymVitanyi, - "cons" => NcdVariant::Cons, - "sym_cons" => NcdVariant::SymCons, - _ => NcdVariant::Vitanyi, - }; - - let ncd = ncd_bytes(x, y, &method, ncd_variant); - format!(r#"{{"ncd":{:.6}}}"#, ncd) - } - "ncd_files" => { - // NCD between two files - let path1 = v - .get("path1") - .and_then(|x| x.as_str()) - .unwrap_or_default() - .to_string(); - let path2 = v - .get("path2") - .and_then(|x| x.as_str()) - .unwrap_or_default() - .to_string(); - let method = v - .get("method") - .and_then(|x| x.as_str()) - .unwrap_or("5") - .to_string(); - let variant = v - .get("variant") - .and_then(|x| x.as_str()) - .unwrap_or("vitanyi") - .to_string(); - - let ncd_variant = match variant.as_str() { - "sym" | "sym_vitanyi" => NcdVariant::SymVitanyi, - "cons" => NcdVariant::Cons, - "sym_cons" => NcdVariant::SymCons, - _ => NcdVariant::Vitanyi, - }; - - let ncd = ncd_paths(&path1, &path2, &method, ncd_variant); - format!(r#"{{"ncd":{:.6}}}"#, ncd) - } - - "rosa_dist" => { - // ROSA-based distance (faster than NCD) - let text1 = v - .get("text1") - .and_then(|x| x.as_str()) - .unwrap_or_default() - .to_string(); - let text2 = v - .get("text2") - .and_then(|x| x.as_str()) - .unwrap_or_default() - .to_string(); - let max_order = v.get("max_order").and_then(|x| x.as_i64()).unwrap_or(-1); - - let x = text1.as_bytes(); - let y = text2.as_bytes(); - - if x.is_empty() || y.is_empty() { - return r#"{"error":"empty text(s)"}"#.to_string(); - } - - let dist = rosa_distance(x, y, max_order); - format!(r#"{{"rosa_dist":{:.6}}}"#, dist) - } - - "cross_entropy" => { - // Cross-entropy H_y(x) - score x under model trained on y - let text_x = v - .get("text_x") - .and_then(|x| x.as_str()) - .unwrap_or_default() - .to_string(); - let text_y = v - .get("text_y") - .and_then(|x| x.as_str()) - .unwrap_or_default() - .to_string(); - let max_order = v.get("max_order").and_then(|x| x.as_i64()).unwrap_or(-1); - - let x = text_x.as_bytes(); - let y = text_y.as_bytes(); - - if x.is_empty() || y.is_empty() { - return r#"{"error":"empty text(s)"}"#.to_string(); - } - - let xe = cross_entropy_rate_bytes(x, y, max_order); - format!(r#"{{"cross_entropy":{:.6}}}"#, xe) - } - "batch_metrics" => { - // Batch metrics for multiple texts - let texts: Vec = v - .get("texts") - .and_then(|x| x.as_array()) - .map(|arr| { - arr.iter() - .filter_map(|item| item.as_str().map(ToString::to_string)) - .collect() - }) - .unwrap_or_default(); - let max_order = v.get("max_order").and_then(|x| x.as_i64()).unwrap_or(-1); - - let results: Vec = texts.iter().map(|text| { - let data = text.as_bytes(); - if data.is_empty() { - r#"{"h0":0,"h_rate":0,"id":0,"len":0}"#.to_string() - } else { - let h0 = marginal_entropy_bytes(data); - let h_rate = entropy_rate_bytes(data, max_order); - let id = if h0 < 1e-9 { 0.0 } else { ((h0 - h_rate) / h0).clamp(0.0, 1.0) }; - format!( - r#"{{"h0":{:.6},"h_rate":{:.6},"id":{:.6},"len":{}}}"#, - h0, h_rate, id, data.len() - ) - } - }).collect(); - - format!(r#"{{"results":[{}]}}"#, results.join(",")) - } - - "ncd_matrix" => { - // NCD matrix for multiple texts (for diversity/clustering) - let texts: Vec = v - .get("texts") - .and_then(|x| x.as_array()) - .map(|arr| { - arr.iter() - .filter_map(|item| item.as_str().map(ToString::to_string)) - .collect() - }) - .unwrap_or_default(); - let method = v - .get("method") - .and_then(|x| x.as_str()) - .unwrap_or("5") - .to_string(); - let variant = v - .get("variant") - .and_then(|x| x.as_str()) - .unwrap_or("vitanyi") - .to_string(); - - let ncd_variant = match variant.as_str() { - "sym" | "sym_vitanyi" => NcdVariant::SymVitanyi, - "cons" => NcdVariant::Cons, - "sym_cons" => NcdVariant::SymCons, - _ => NcdVariant::Vitanyi, - }; - - let datas: Vec> = texts.iter().map(|t| t.as_bytes().to_vec()).collect(); - let matrix = ncd_matrix_bytes(&datas, &method, ncd_variant); - let n = datas.len(); - - // Format as row-major array of arrays - let rows: Vec = (0..n).map(|i| { - let row: Vec = (0..n).map(|j| format!("{:.6}", matrix[i * n + j])).collect(); - format!("[{}]", row.join(",")) - }).collect(); - - format!(r#"{{"matrix":[{}],"n":{}}}"#, rows.join(","), n) - } - "rosa_matrix" => { - // ROSA distance matrix (faster than NCD matrix) - let texts: Vec = v - .get("texts") - .and_then(|x| x.as_array()) - .map(|arr| { - arr.iter() - .filter_map(|item| item.as_str().map(ToString::to_string)) - .collect() - }) - .unwrap_or_default(); - let max_order = v.get("max_order").and_then(|x| x.as_i64()).unwrap_or(-1); - - let n = texts.len(); - let datas: Vec<&[u8]> = texts.iter().map(|t| t.as_bytes()).collect(); - - // Compute matrix (symmetric) - let mut matrix = vec![0.0f64; n * n]; - for i in 0..n { - for j in i..n { - let d = if i == j { - 0.0 - } else { - rosa_distance(datas[i], datas[j], max_order) - }; - matrix[i * n + j] = d; - matrix[j * n + i] = d; - } - } - - // Format as row-major array of arrays - let rows: Vec = (0..n).map(|i| { - let row: Vec = (0..n).map(|j| format!("{:.6}", matrix[i * n + j])).collect(); - format!("[{}]", row.join(",")) - }).collect(); - - format!(r#"{{"matrix":[{}],"n":{}}}"#, rows.join(","), n) - } - "spam_check" => { - // Quick spam/quality check for a single text - let text = v - .get("text") - .and_then(|x| x.as_str()) - .unwrap_or_default() - .to_string(); - let h0_threshold = v.get("h0_min").and_then(|x| x.as_f64()).unwrap_or(1.0); - let h_rate_threshold = v.get("h_rate_min").and_then(|x| x.as_f64()).unwrap_or(0.5); - let id_threshold = v.get("id_max").and_then(|x| x.as_f64()).unwrap_or(0.95); - let min_len = v.get("min_len").and_then(|x| x.as_i64()).unwrap_or(10) as usize; - - let data = text.as_bytes(); - let len = data.len(); - - if len < min_len { - return format!(r#"{{"pass":false,"reason":"too_short","len":{}}}"#, len); - } - - let h0 = marginal_entropy_bytes(data); - if h0 < h0_threshold { - return format!(r#"{{"pass":false,"reason":"low_entropy","h0":{:.4}}}"#, h0); - } - - let h_rate = entropy_rate_bytes(data, -1); - if h_rate < h_rate_threshold { - return format!(r#"{{"pass":false,"reason":"low_entropy_rate","h_rate":{:.4}}}"#, h_rate); - } - - let id = if h0 < 1e-9 { 0.0 } else { ((h0 - h_rate) / h0).clamp(0.0, 1.0) }; - if id > id_threshold { - return format!(r#"{{"pass":false,"reason":"high_redundancy","id":{:.4}}}"#, id); - } - - format!(r#"{{"pass":true,"h0":{:.4},"h_rate":{:.4},"id":{:.4},"len":{}}}"#, h0, h_rate, id, len) - } - "help" => { - r#"{"ops":["metrics","metrics_file","ncd","ncd_files","rosa_dist","cross_entropy","batch_metrics","ncd_matrix","rosa_matrix","spam_check"]}"#.to_string() - } - _ => { - format!(r#"{{"error":"unknown op: {}"}}"#, op) - } - } -} - -fn run_batch_mode() { - let stdin = io::stdin(); - for line in stdin.lock().lines() { - match line { - Ok(l) => println!("{}", process_json_line(&l)), - Err(_) => continue, - } - } -} - -fn run_aixi_mode(config_path: &str) -> anyhow::Result<()> { - let mut file = File::open(config_path)?; - let mut content = String::new(); - file.read_to_string(&mut content)?; - let v: serde_json::Value = serde_json::from_str(&content)?; - let config_dir = Path::new(config_path).parent().unwrap_or(Path::new(".")); - - let env_name = v["environment"].as_str().unwrap_or("coin-flip"); - let mut env: Box = match env_name { - "coin-flip" => Box::new(CoinFlip::new(0.9)), - "ctw-test" | "ctwtest" => Box::new(CtwTest::new()), - "extended-tiger" => Box::new(ExtendedTiger::new()), - "tictactoe" => Box::new(TicTacToe::new()), - "biased-rock-paper-scissor" => Box::new(BiasedRockPaperScissor::new()), - "kuhn-poker" => Box::new(KuhnPoker::new()), - "external" => { - return Err(anyhow::anyhow!( - "environment=external (ProcessEnvironment) has been removed for security reasons. \ - Use NyxVmEnvironment with --features vm for secure sandboxed environments." - )); - } - "vm" | "nyx" | "nyx-vm" => { - #[cfg(not(feature = "vm"))] - { - return Err(anyhow::anyhow!( - "VM environments require the `vm` feature (enable with --features vm)" - )); - } - #[cfg(feature = "vm")] - { - let observation_bits = v["observation_bits"].as_u64().unwrap_or(16) as usize; - let reward_bits = v["reward_bits"].as_u64().unwrap_or(8) as usize; - let agent_horizon = v["agent_horizon"].as_u64().unwrap_or(3) as usize; - let vm_cfg = parse_nyx_environment_config( - &v, - observation_bits, - reward_bits, - agent_horizon, - config_dir, - )?; - Box::new(NyxVmEnvironment::new(vm_cfg)?) - } - } - _ => return Err(anyhow::anyhow!("Unknown environment: {}", env_name)), - }; - - // Use the run seed for environment stochasticity as well, so environment - // trajectories are reproducible across repeated runs. - let run_random_seed = v["random_seed"].as_u64().or_else(|| v["rng_seed"].as_u64()); - if let Some(seed) = run_random_seed { - env.set_random_seed(seed); - } - - let log_every = v["log_every"].as_u64().unwrap_or(1) as usize; - let perf = v["perf"].as_bool().unwrap_or(false); - let vm_perf_only = v["vm_perf_only"].as_bool().unwrap_or(false); - - if vm_perf_only { - let cycles = v["perf_cycles"] - .as_u64() - .or_else(|| v["terminate-lifetime"].as_u64()) - .unwrap_or(1000) as usize; - let observation_stream_len = parse_observation_stream_len_for_env(&v, env_name); - let mut obs_stream = env.drain_observations(); - validate_obs_stream_len(observation_stream_len, obs_stream.len())?; - let mut obs = obs_stream.first().copied().unwrap_or(0); - let mut rew = env.get_reward(); - let start = Instant::now(); - for t in 0..cycles { - if log_every > 0 && t % log_every == 0 { - println!("Cycle {}: Obs={}, Rew={}", t, obs, rew); - } - env.perform_action(0); - obs_stream = env.drain_observations(); - validate_obs_stream_len(observation_stream_len, obs_stream.len())?; - obs = obs_stream.first().copied().unwrap_or(0); - rew = env.get_reward(); - } - if perf { - let elapsed = start.elapsed().as_secs_f64().max(1e-9); - let cps = cycles as f64 / elapsed; - println!("Perf cycles/s: {:.2}", cps); - } - return Ok(()); - } - - let observation_bits = v["observation_bits"] - .as_u64() - .map(|n| n as usize) - .unwrap_or_else(|| env.get_observation_bits()); - let observation_stream_len = parse_observation_stream_len_for_env(&v, env_name); - let observation_key_mode = parse_observation_key_mode_for_env(&v, env_name); - validate_observation_config(env_name, &v, observation_stream_len, observation_key_mode)?; - let reward_bits = v["reward_bits"] - .as_u64() - .map(|n| n as usize) - .unwrap_or_else(|| env.get_reward_bits()); - let agent_actions = v["agent_actions"] - .as_u64() - .map(|n| n as usize) - .unwrap_or_else(|| env.get_num_actions()); - let min_reward = env.min_reward(); - let max_reward = env.max_reward(); - let reward_offset = v["reward_offset"] - .as_i64() - .unwrap_or_else(|| (-min_reward).max(0)); - let discount_gamma = v["discount_gamma"].as_f64().unwrap_or(1.0); - if !(0.0..=1.0).contains(&discount_gamma) { - return Err(anyhow::anyhow!( - "discount_gamma must be in [0, 1] (got {})", - discount_gamma - )); - } - - let planner = v["planner"] - .as_str() - .or_else(|| v["solver"].as_str()) - .unwrap_or("mc-aixi"); - let planner_norm = planner.to_ascii_lowercase(); - if !matches!(planner_norm.as_str(), "mc-aixi" | "aiqi") { - return Err(anyhow::anyhow!( - "Unknown planner/solver '{}'. Supported values: mc-aixi, aiqi", - planner - )); - } - if planner_norm.as_str() == "aiqi" { - let aiqi_random_seed = v["aiqi_random_seed"].as_u64().or(run_random_seed); - let aiqi_rate_backend = if !v["aiqi_rate_backend"].is_null() { - Some(parse_vm_stats_backend( - &v["aiqi_rate_backend"], - &v, - config_dir, - )?) - } else if !v["rate_backend"].is_null() { - Some(parse_vm_stats_backend(&v["rate_backend"], &v, config_dir)?) - } else { - None - }; - - let aiqi_discount_gamma = if v["discount_gamma"].is_null() { - 0.99 - } else { - discount_gamma - }; - - let aiqi_config = AiqiConfig { - algorithm: v["algorithm"].as_str().unwrap_or("ac-ctw").to_string(), - ct_depth: v["ct_depth"].as_u64().unwrap_or(20) as usize, - observation_bits, - observation_stream_len, - reward_bits, - agent_actions, - min_reward, - max_reward, - reward_offset, - discount_gamma: aiqi_discount_gamma, - return_horizon: v["return_horizon"] - .as_u64() - .or_else(|| v["agent_horizon"].as_u64()) - .unwrap_or(3) as usize, - return_bins: v["return_bins"] - .as_u64() - .or_else(|| v["aiqi_bins"].as_u64()) - .unwrap_or(16) as usize, - augmentation_period: v["augmentation_period"] - .as_u64() - .or_else(|| v["aiqi_period"].as_u64()) - .or_else(|| v["return_horizon"].as_u64()) - .or_else(|| v["agent_horizon"].as_u64()) - .unwrap_or(3) as usize, - history_prune_keep_steps: v["history_prune_keep_steps"] - .as_u64() - .or_else(|| v["aiqi_history_prune_keep_steps"].as_u64()) - .map(|n| n as usize), - baseline_exploration: v["baseline_exploration"] - .as_f64() - .or_else(|| v["tau"].as_f64()) - .unwrap_or(0.01), - random_seed: aiqi_random_seed, - rate_backend: aiqi_rate_backend, - rate_backend_max_order: v["rate_backend_max_order"] - .as_i64() - .or_else(|| v["max_order"].as_i64()) - .or_else(|| v["rosa_max_order"].as_i64()) - .unwrap_or(20), - rwkv_model_path: v["rwkv_model_path"].as_str().map(|s| s.to_string()), - rosa_max_order: v["rosa_max_order"].as_u64().map(|n| n as i64), - zpaq_method: v["zpaq_method"].as_str().map(|s| s.to_string()), - }; - let aiqi_backend_desc = aiqi_backend_label(&aiqi_config); - let mut aiqi = AiqiAgent::new(aiqi_config).map_err(|e| anyhow::anyhow!(e))?; - - println!( - "AIQI initialized ({}) for {} environment.", - aiqi_backend_desc, env_name - ); - - let learn_cycles = v["learn_cycles"].as_u64().map(|n| n as usize); - let eval_cycles = v["eval_cycles"].as_u64().map(|n| n as usize); - let cycles = v["terminate-lifetime"].as_u64().unwrap_or(20) as usize; - - let (learn_cycles, eval_cycles) = match (learn_cycles, eval_cycles) { - (Some(l), Some(e)) => (l, e), - (Some(l), None) => (l, 0usize), - (None, Some(e)) => (cycles, e), - (None, None) => (cycles, 0usize), - }; - - let mut obs_stream = env.drain_observations(); - validate_obs_stream_len(observation_stream_len, obs_stream.len())?; - let mut rew = env.get_reward(); - let mut learn_total_reward: i64 = 0; - let mut eval_total_reward: i64 = 0; - - let explore_epsilon = v["explore_epsilon"].as_f64().unwrap_or(0.0); - let explore_gamma = v["explore_gamma"].as_f64().unwrap_or(1.0); - - let mut trace_logger = AixiRunLogger::new(&v)?; - - let learn_start = Instant::now(); - for t in 0..learn_cycles { - let extra_explore_p = if explore_epsilon > 0.0 { - (explore_epsilon * explore_gamma.powi(t as i32)).min(1.0) - } else { - 0.0 - }; - - let action = aiqi.get_planned_action_with_extra_exploration(extra_explore_p); - if log_every > 0 && t % log_every == 0 { - println!( - "Cycle {}: Action={} Obs={:?} Rew={}", - t, action, obs_stream, rew - ); - } - - if let Some(l) = trace_logger.as_mut() { - let action_bits = env.get_action_bits(); - l.log_action(action, action_bits)?; - } - - env.perform_action(action); - obs_stream = env.drain_observations(); - validate_obs_stream_len(observation_stream_len, obs_stream.len())?; - rew = env.get_reward(); - - if let Some(l) = trace_logger.as_mut() { - l.log_percept( - &obs_stream, - rew, - observation_bits, - reward_bits, - reward_offset, - )?; - l.next_step()?; - } - - aiqi.observe_transition(action, &obs_stream, rew) - .map_err(|e| anyhow::anyhow!(e))?; - learn_total_reward += rew; - } - - if perf && learn_cycles > 0 { - let elapsed = learn_start.elapsed().as_secs_f64().max(1e-9); - let cps = learn_cycles as f64 / elapsed; - println!("Learn cycles/s: {:.2}", cps); - } - - if eval_cycles > 0 { - let eval_start = Instant::now(); - for t in 0..eval_cycles { - let step = learn_cycles + t; - let action = aiqi.get_planned_action(); - if log_every > 0 && step % log_every == 0 { - println!( - "Cycle {}: Action={} Obs={:?} Rew={}", - step, action, obs_stream, rew - ); - } - - if let Some(l) = trace_logger.as_mut() { - let action_bits = env.get_action_bits(); - l.log_action(action, action_bits)?; - } - - env.perform_action(action); - obs_stream = env.drain_observations(); - validate_obs_stream_len(observation_stream_len, obs_stream.len())?; - rew = env.get_reward(); - - if let Some(l) = trace_logger.as_mut() { - l.log_percept( - &obs_stream, - rew, - observation_bits, - reward_bits, - reward_offset, - )?; - l.next_step()?; - } - - aiqi.observe_transition(action, &obs_stream, rew) - .map_err(|e| anyhow::anyhow!(e))?; - eval_total_reward += rew; - } - - if perf && eval_cycles > 0 { - let elapsed = eval_start.elapsed().as_secs_f64().max(1e-9); - let cps = eval_cycles as f64 / elapsed; - println!("Eval cycles/s: {:.2}", cps); - } - - let avg = (eval_total_reward as f64) / (eval_cycles as f64); - println!("Eval Total Reward: {}", eval_total_reward); - println!("Eval Average Reward per Cycle: {:.6}", avg); - } - - println!("Total Reward: {}", learn_total_reward); - return Ok(()); - } - - let mcaixi_random_seed = v["mcaixi_random_seed"].as_u64().or(run_random_seed); - let config = AgentConfig { - algorithm: v["algorithm"].as_str().unwrap_or("ctw").to_string(), - ct_depth: v["ct_depth"].as_u64().unwrap_or(20) as usize, - agent_horizon: v["agent_horizon"].as_u64().unwrap_or(3) as usize, - observation_bits, - observation_stream_len, - observation_key_mode, - reward_bits, - agent_actions, - num_simulations: v["num_simulations"].as_u64().unwrap_or(50) as usize, - exploration_exploitation_ratio: v["exploration_exploitation_ratio"].as_f64().unwrap_or(1.4), - discount_gamma, - min_reward, - max_reward, - reward_offset, - random_seed: mcaixi_random_seed, - rate_backend: if !v["rate_backend"].is_null() { - Some(parse_vm_stats_backend(&v["rate_backend"], &v, config_dir)?) - } else { - None - }, - rate_backend_max_order: v["rate_backend_max_order"] - .as_i64() - .or_else(|| v["max_order"].as_i64()) - .or_else(|| v["rosa_max_order"].as_i64()) - .unwrap_or(20), - rwkv_model_path: v["rwkv_model_path"].as_str().map(|s| s.to_string()), - rwkv_method: v["rwkv_method"].as_str().map(|s| s.to_string()), - mamba_model_path: v["mamba_model_path"].as_str().map(|s| s.to_string()), - mamba_method: v["mamba_method"].as_str().map(|s| s.to_string()), - rosa_max_order: v["rosa_max_order"].as_u64().map(|n| n as i64), - zpaq_method: v["zpaq_method"].as_str().map(|s| s.to_string()), - }; - - let mut agent = Agent::try_new(config).map_err(|err| anyhow::anyhow!(err))?; - println!( - "Agent initialized with {} algorithm for {} environment.", - v["algorithm"].as_str().unwrap_or("ctw"), - env_name - ); - - let learn_cycles = v["learn_cycles"].as_u64().map(|n| n as usize); - let eval_cycles = v["eval_cycles"].as_u64().map(|n| n as usize); - let cycles = v["terminate-lifetime"].as_u64().unwrap_or(20) as usize; - - let (learn_cycles, eval_cycles) = match (learn_cycles, eval_cycles) { - (Some(l), Some(e)) => (l, e), - (Some(l), None) => (l, 0usize), - (None, Some(e)) => (cycles, e), - (None, None) => (cycles, 0usize), - }; - - let mut total_reward = 0; - let mut prev_action = 0; - let mut obs_stream = env.drain_observations(); - validate_obs_stream_len(observation_stream_len, obs_stream.len())?; - let mut obs_repr = agent.observation_repr_from_stream(&obs_stream); - let mut rew = env.get_reward(); - - let explore_epsilon = v["explore_epsilon"].as_f64().unwrap_or(0.0); - let explore_gamma = v["explore_gamma"].as_f64().unwrap_or(1.0); - let mut explore_rng = if let Some(seed) = mcaixi_random_seed { - RandomGenerator::from_seed(seed).fork_with(0x4558504c4f52455f) - } else { - RandomGenerator::new() - }; - - // Optional trace logger: can emit a RWKV-friendly stream (0/1 bytes) and/or JSONL metadata. - // NOTE: The bits are emitted in the same order the agent consumes them: - // percept bits (obs stream + reward) first, then action bits. - let mut trace_logger = AixiRunLogger::new(&v)?; - - let learn_start = Instant::now(); - for t in 0..learn_cycles { - if log_every > 0 && t % log_every == 0 { - println!("Cycle {}: Obs={:?}, Rew={}", t, obs_repr, rew); - } - if let Some(l) = trace_logger.as_mut() { - l.log_percept( - &obs_stream, - rew, - observation_bits, - reward_bits, - reward_offset, - )?; - } - agent.model_update_percept_stream(&obs_stream, rew); - - let explore_p = if explore_epsilon > 0.0 { - explore_epsilon * explore_gamma.powi(t as i32) - } else { - 0.0 - }; - let action = if explore_p > 0.0 && explore_rng.gen_bool(explore_p.min(1.0)) { - explore_rng.gen_range(agent_actions) as u64 - } else { - agent.get_planned_action(&obs_stream, rew, prev_action) - }; - if log_every > 0 && t % log_every == 0 { - println!("Cycle {}: Planned Action={}", t, action); - } - - if let Some(l) = trace_logger.as_mut() { - // Mirror the agent's action encoding (same number of bits). - let action_bits = env.get_action_bits(); - l.log_action(action, action_bits)?; - } - agent.model_update_action_external(action); - env.perform_action(action); - obs_stream = env.drain_observations(); - validate_obs_stream_len(observation_stream_len, obs_stream.len())?; - obs_repr = agent.observation_repr_from_stream(&obs_stream); - rew = env.get_reward(); - prev_action = action; - total_reward += rew; - - if let Some(l) = trace_logger.as_mut() { - l.next_step()?; - } - } - - if perf && learn_cycles > 0 { - let elapsed = learn_start.elapsed().as_secs_f64().max(1e-9); - let cps = learn_cycles as f64 / elapsed; - println!("Learn cycles/s: {:.2}", cps); - } - - if eval_cycles > 0 { - let mut eval_total_reward: i64 = 0; - let eval_start = Instant::now(); - for t in 0..eval_cycles { - let step = learn_cycles + t; - if log_every > 0 && step % log_every == 0 { - println!("Cycle {}: Obs={:?}, Rew={}", step, obs_repr, rew); - } - if let Some(l) = trace_logger.as_mut() { - l.log_percept( - &obs_stream, - rew, - observation_bits, - reward_bits, - reward_offset, - )?; - } - agent.model_update_percept_stream(&obs_stream, rew); - - let action = agent.get_planned_action(&obs_stream, rew, prev_action); - if log_every > 0 && step % log_every == 0 { - println!("Cycle {}: Planned Action={}", step, action); - } - - if let Some(l) = trace_logger.as_mut() { - let action_bits = env.get_action_bits(); - l.log_action(action, action_bits)?; - } - agent.model_update_action_external(action); - env.perform_action(action); - obs_stream = env.drain_observations(); - validate_obs_stream_len(observation_stream_len, obs_stream.len())?; - obs_repr = agent.observation_repr_from_stream(&obs_stream); - rew = env.get_reward(); - prev_action = action; - eval_total_reward += rew; - - if let Some(l) = trace_logger.as_mut() { - l.next_step()?; - } - } - - if perf && eval_cycles > 0 { - let elapsed = eval_start.elapsed().as_secs_f64().max(1e-9); - let cps = eval_cycles as f64 / elapsed; - println!("Eval cycles/s: {:.2}", cps); - } - - let avg = (eval_total_reward as f64) / (eval_cycles as f64); - println!("Eval Total Reward: {}", eval_total_reward); - println!("Eval Average Reward per Cycle: {:.6}", avg); - } - - println!("Total Reward: {}", total_reward); - Ok(()) -} - -fn search_command(args: &[String]) { - if args.len() < 4 { - eprintln!("Error: 'search' requires query and target path."); - std::process::exit(1); - } - let query = &args[2]; - let target = &args[3]; - - // Preserve the legacy behavior (and avoid extra parsing work) when no flags are given. - if args.len() == 4 { - search::run_search(query, target); - return; - } - - let mut opts = search::SearchOptions::default(); - let mut rate_backend = "rosaplus".to_string(); - let compression_backend = "zpaq".to_string(); - let mut method: Option = None; - let mut expert_spec_path: Option = None; - let mut stage2_prior_mode: Option = None; - - let mut i = 4usize; - while i < args.len() { - match args[i].as_str() { - "--level" => { - i += 1; - let v = args - .get(i) - .unwrap_or_exit("Error: --level requires snippet|file"); - opts.granularity = if v == "snippet" { - search::SearchGranularity::Snippet - } else { - search::SearchGranularity::File - }; - } - "--prior" => { - i += 1; - opts.universal_prior = args.get(i).cloned(); - } - "--max-order" => { - i += 1; - opts.max_order = args.get(i).and_then(|s| s.parse().ok()).unwrap_or(-1); - } - "--top-k" => { - i += 1; - opts.top_k = args.get(i).and_then(|s| s.parse().ok()).unwrap_or(10); - } - "--rate-backend" => { - i += 1; - let v = args - .get(i) - .unwrap_or_exit("Error: --rate-backend requires a value"); - rate_backend = parse_rate_backend(v).unwrap_or("rosaplus").to_string(); - } - "--method" => { - i += 1; - method = args.get(i).cloned(); - } - "--expert-spec" => { - i += 1; - expert_spec_path = args.get(i).cloned(); - } - "--stage2-prior-mode" => { - i += 1; - if let Some(v) = args.get(i) { - stage2_prior_mode = match v.as_str() { - "none" | "no-prior" => Some(search::Stage2PriorMode::Disable), - "summarize" | "summarize-prior" => Some(search::Stage2PriorMode::Summarize), - "use" | "use-prior" => Some(search::Stage2PriorMode::Use), - _ => Some(search::Stage2PriorMode::Use), - }; - } - } - _ => { - i += 1; - } - } - i += 1; - } - if let Some(mode) = stage2_prior_mode { - opts.stage2_prior_mode = mode; - } - opts.ctx = build_ctx( - &rate_backend, - &compression_backend, - method.as_deref(), - expert_spec_path.as_deref(), - ) - .ctx; - search::run_search_with_options(query, target, &opts); -} - -trait OptionExt { - fn unwrap_or_exit(self, msg: &str) -> T; -} -impl OptionExt for Option { - fn unwrap_or_exit(self, msg: &str) -> T { - self.unwrap_or_else(|| { - eprintln!("{}", msg); - std::process::exit(1); - }) - } -} - -fn main() { - let args: Vec = env::args().collect(); - - // Check for help flag early - if args.len() > 1 && (args[1] == "--help" || args[1] == "-h") { - print_usage(); - return; - } - - if args.len() < 2 { - print_usage(); - return; - } - - let primitive = &args[1]; - if primitive == "batch" { - run_batch_mode(); - return; - } - - // Common positional and flag parsing. - // Collect positionals only up to the first flag token, then parse flags separately. - let mut file1: Option = None; - let mut file2: Option = None; - let mut pos_arg3: Option = None; - let mut flags_start = 2usize; - - if primitive != "search" && primitive != "aixi" { - let mut positionals: Vec = Vec::new(); - let mut i = 2usize; - while i < args.len() { - let tok = &args[i]; - if tok.starts_with('-') { - break; - } - positionals.push(tok.clone()); - i += 1; - } - flags_start = i; - file1 = positionals.first().cloned(); - file2 = positionals.get(1).cloned(); - pos_arg3 = positionals.get(2).cloned(); - } - - let mut rate_backend_str = "rosaplus".to_string(); - let mut compression_backend_str = "zpaq".to_string(); - let mut method_str: Option = None; - let mut expert_spec_path: Option = None; - let mut model_export_path: Option = None; - let mut diagnostic_mixture_path: Option = None; - let mut diagnostic_out_prefix: Option = None; - let mut sequitur_debug_hexes: Vec = Vec::new(); - let mut sequitur_context_bytes: usize = 64; - let mut sequitur_alphabet_prefix: usize = 4; - let mut generate_len_bytes: usize = 8; - let mut generate_config = GenerationConfig::default(); - let mut rate_backend_specified = false; - - let mut i = flags_start; - while i < args.len() { - match args[i].as_str() { - "--rate-backend" => { - i += 1; - let v = args - .get(i) - .unwrap_or_exit("Error: --rate-backend requires a value"); - rate_backend_str = parse_rate_backend(v).unwrap_or("rosaplus").to_string(); - rate_backend_specified = true; - } - "--ncd-backend" => { - i += 1; - let v = args - .get(i) - .unwrap_or_exit("Error: --compression-backend requires a value"); - compression_backend_str = - parse_compression_backend(v).unwrap_or("zpaq").to_string(); - } - "--compression-backend" => { - i += 1; - let v = args - .get(i) - .unwrap_or_exit("Error: --compression-backend requires a value"); - compression_backend_str = - parse_compression_backend(v).unwrap_or("zpaq").to_string(); - } - "--method" => { - i += 1; - method_str = args.get(i).cloned(); - } - "--expert-spec" => { - i += 1; - expert_spec_path = args.get(i).cloned(); - rate_backend_specified = true; - } - "--model-export" | "--rwkv-export" => { - i += 1; - model_export_path = args.get(i).cloned(); - } - "--mixture" => { - i += 1; - diagnostic_mixture_path = args.get(i).cloned(); - } - "--out-prefix" => { - i += 1; - diagnostic_out_prefix = args.get(i).cloned(); - } - "--hex" => { - i += 1; - if let Some(value) = args.get(i) { - sequitur_debug_hexes.push(value.clone()); - } - } - "--context-bytes" => { - i += 1; - let raw = args - .get(i) - .unwrap_or_exit("Error: --context-bytes requires a positive integer"); - sequitur_context_bytes = raw.parse::().unwrap_or_else(|_| { - eprintln!("Error: --context-bytes must be a positive integer, got '{raw}'"); - std::process::exit(1); - }); - } - "--alphabet-prefix" => { - i += 1; - let raw = args - .get(i) - .unwrap_or_exit("Error: --alphabet-prefix requires a positive integer"); - sequitur_alphabet_prefix = raw.parse::().unwrap_or_else(|_| { - eprintln!("Error: --alphabet-prefix must be a positive integer, got '{raw}'"); - std::process::exit(1); - }); - } - "--bytes" => { - i += 1; - let raw = args - .get(i) - .unwrap_or_exit("Error: --bytes requires a non-negative integer"); - generate_len_bytes = raw.parse::().unwrap_or_else(|_| { - eprintln!("Error: --bytes must be a non-negative integer, got '{raw}'"); - std::process::exit(1); - }); - } - "--sample" => { - generate_config.strategy = GenerationStrategy::Sample; - } - "--greedy" => { - generate_config.strategy = GenerationStrategy::Greedy; - } - "--adaptive" => { - generate_config.update_mode = GenerationUpdateMode::Adaptive; - } - "--seed" => { - i += 1; - let raw = args - .get(i) - .unwrap_or_exit("Error: --seed requires an unsigned integer"); - generate_config.seed = raw.parse::().unwrap_or_else(|_| { - eprintln!("Error: --seed must be an unsigned integer, got '{raw}'"); - std::process::exit(1); - }); - generate_config.strategy = GenerationStrategy::Sample; - } - "--temperature" => { - i += 1; - let raw = args - .get(i) - .unwrap_or_exit("Error: --temperature requires a finite number"); - generate_config.temperature = raw.parse::().unwrap_or_else(|_| { - eprintln!("Error: --temperature must be a finite number, got '{raw}'"); - std::process::exit(1); - }); - if !generate_config.temperature.is_finite() || generate_config.temperature < 0.0 { - eprintln!( - "Error: --temperature must be finite and non-negative, got '{}'", - generate_config.temperature - ); - std::process::exit(1); - } - generate_config.strategy = GenerationStrategy::Sample; - } - "--top-k" => { - i += 1; - let raw = args - .get(i) - .unwrap_or_exit("Error: --top-k requires a non-negative integer"); - generate_config.top_k = raw.parse::().unwrap_or_else(|_| { - eprintln!("Error: --top-k must be a non-negative integer, got '{raw}'"); - std::process::exit(1); - }); - generate_config.strategy = GenerationStrategy::Sample; - } - "--top-p" => { - i += 1; - let raw = args - .get(i) - .unwrap_or_exit("Error: --top-p requires a number in (0, 1]"); - generate_config.top_p = raw.parse::().unwrap_or_else(|_| { - eprintln!("Error: --top-p must be a number in (0, 1], got '{raw}'"); - std::process::exit(1); - }); - if !generate_config.top_p.is_finite() - || generate_config.top_p <= 0.0 - || generate_config.top_p > 1.0 - { - eprintln!( - "Error: --top-p must be in (0, 1], got '{}'", - generate_config.top_p - ); - std::process::exit(1); - } - generate_config.strategy = GenerationStrategy::Sample; - } - _ => {} - } - i += 1; - } - - if primitive == "ac-log-loss" || primitive == "ac_log_loss" { - let input_path = file1.unwrap_or_exit( - "Error: 'ac-log-loss' requires --mixture --out-prefix ", - ); - let mixture_path = diagnostic_mixture_path - .unwrap_or_exit("Error: 'ac-log-loss' requires --mixture "); - let out_prefix = diagnostic_out_prefix - .unwrap_or_exit("Error: 'ac-log-loss' requires --out-prefix "); - let spec = load_mixture_spec(&mixture_path).unwrap_or_else(|e| { - eprintln!( - "Error: failed to load mixture spec '{}': {}", - mixture_path, e - ); - std::process::exit(1); - }); - let data = read_file(&input_path); - match infotheory::diagnostics::run_ac_log_loss_mixture_bytes(&data, &spec, &out_prefix) { - Ok(summary) => { - println!( - "wrote {} rows to {}, nodes to {}, summary to {}", - summary.positions, - summary.trace_path.display(), - summary.nodes_path.display(), - summary.summary_path.display() - ); - } - Err(err) => { - eprintln!("Error: AC log-loss diagnostic failed: {err:#}"); - std::process::exit(1); - } - } - return; - } - - if primitive == "sequitur-debug" || primitive == "sequitur_debug" { - let inputs = if !sequitur_debug_hexes.is_empty() { - sequitur_debug_hexes - .iter() - .map(|raw_hex| { - parse_hex_bytes(raw_hex).unwrap_or_else(|e| { - eprintln!("Error: invalid --hex input for 'sequitur-debug': {e}"); - std::process::exit(1); - }) - }) - .collect::>() - } else { - let input_path = - file1.unwrap_or_exit("Error: 'sequitur-debug' requires or --hex "); - vec![read_file(&input_path)] - }; - let alphabet_prefix = sequitur_alphabet_prefix.clamp(1, 256); - let cases = inputs - .iter() - .map(|data| { - let mut model = SequiturModel::new(sequitur_context_bytes); - let trace = model.predictive_trace(data, alphabet_prefix); - let rules = model - .canonical_grammar() - .rules - .iter() - .map(|rule| { - let rhs = rule - .rhs - .iter() - .map(|sym| match sym { - CanonicalSymbol::Terminal(byte) => serde_json::json!(*byte as i64), - CanonicalSymbol::NonTerminal(rule_id) => { - serde_json::json!(-((*rule_id as i64) + 1)) - } - }) - .collect::>(); - serde_json::json!({ - "id": rule.id, - "rhs": rhs, - }) - }) - .collect::>(); - serde_json::json!({ - "input_hex": bytes_to_hex(data), - "decoded_hex": bytes_to_hex(&model.decode()), - "rules": rules, - "trace": trace, - }) - }) - .collect::>(); - let output = serde_json::json!({ - "context_bytes": sequitur_context_bytes, - "alphabet_prefix": alphabet_prefix, - "cases": cases, - }); - println!( - "{}", - serde_json::to_string(&output).expect("sequitur debug json serialization") - ); - return; - } - - let built_ctx = build_ctx( - &rate_backend_str, - &compression_backend_str, - method_str.as_deref(), - expert_spec_path.as_deref(), - ); - let ctx = built_ctx.ctx; - let expert_spec_max_order = built_ctx.expert_spec_max_order; - set_default_ctx(ctx.clone()); - - match primitive.as_str() { - "aixi" => { - if let Some(p) = args.get(2) { - if let Err(e) = run_aixi_mode(p) { - eprintln!("Error: {}", e); - std::process::exit(1); - } - } else { - eprintln!("Error: 'aixi' requires config.json"); - std::process::exit(1); - } - } - "search" => search_command(&args), - "compress" => { - let in_path = file1.unwrap_or_exit("Error: 'compress' requires "); - let out_path = file2.unwrap_or_exit("Error: 'compress' requires "); - let data = read_file(&in_path); - let backend = file_roundtrip_backend(&ctx.compression_backend); - let compressed = match compress_bytes_backend(&data, &backend) { - Ok(v) => v, - Err(e) => { - eprintln!("Error: compression failed: {e}"); - std::process::exit(1); - } - }; - if let Err(e) = std::fs::write(&out_path, &compressed) { - eprintln!("Error: failed to write output '{}': {}", out_path, e); - std::process::exit(1); - } - println!( - "compressed {} bytes -> {} bytes", - data.len(), - compressed.len() - ); - if let Err(e) = maybe_export_online_model(model_export_path.as_deref(), &ctx, &[&data]) - { - eprintln!("Error exporting online model: {e}"); - std::process::exit(1); - } - } - "decompress" => { - let in_path = file1.unwrap_or_exit("Error: 'decompress' requires "); - let out_path = file2.unwrap_or_exit("Error: 'decompress' requires "); - let input = read_file(&in_path); - let backend = file_roundtrip_backend(&ctx.compression_backend); - let decoded = match decompress_bytes_backend(&input, &backend) { - Ok(v) => v, - Err(e) => { - eprintln!("Error: decompression failed: {e}"); - std::process::exit(1); - } - }; - if let Err(e) = std::fs::write(&out_path, &decoded) { - eprintln!("Error: failed to write output '{}': {}", out_path, e); - std::process::exit(1); - } - println!( - "decompressed {} bytes -> {} bytes", - input.len(), - decoded.len() - ); - if let Err(e) = - maybe_export_online_model(model_export_path.as_deref(), &ctx, &[&decoded]) - { - eprintln!("Error exporting online model: {e}"); - std::process::exit(1); - } - } - "generate" => { - // Disambiguate positional args for `generate [file] [max_order]`. - // When stdin is piped and the first positional looks like an integer, - // treat it as max_order (not a file path). - let stdin_is_piped = !io::stdin().is_terminal(); - let (file_path, explicit_max_order) = match (file1.as_deref(), file2.as_deref()) { - // `generate ` — both present - (Some(f), Some(mo)) => (Some(f), mo.parse::().ok()), - // `generate ` — single positional: - // if stdin is piped and it parses as an integer, it's max_order - // otherwise it's a file path - (Some(arg), None) if stdin_is_piped && arg.parse::().is_ok() => { - (None, arg.parse::().ok()) - } - (Some(f), None) => (Some(f), None), - // No positionals at all - (None, _) => (None, None), - }; - let max_order = explicit_max_order - .or(pos_arg3.as_deref().and_then(|s| s.parse().ok())) - .or(expert_spec_max_order) - .unwrap_or(-1); - let input = if let Some(path) = file_path { - read_file(path) - } else { - read_stdin_all_for_generate() - }; - let generated = ctx.generate_bytes_with_config( - &input, - generate_len_bytes, - max_order, - generate_config, - ); - if let Err(e) = io::stdout().write_all(&generated) { - eprintln!("Error writing generated output: {e}"); - std::process::exit(1); - } - if let Err(e) = io::stdout().flush() { - eprintln!("Error flushing generated output: {e}"); - std::process::exit(1); - } - if let Err(e) = maybe_export_online_model(model_export_path.as_deref(), &ctx, &[&input]) - { - eprintln!("Error exporting online model: {e}"); - std::process::exit(1); - } - } - "ncd" | "ncd_vitanyi" | "ncd_sym" | "ncd_sym_vitanyi" | "ncd_cons" | "ncd_sym_cons" => { - let f1 = file1.unwrap_or_exit("Error: NCD requires two files"); - let f2 = file2.unwrap_or_exit("Error: NCD requires two files"); - let _method = pos_arg3.or(method_str).unwrap_or_else(|| "5".to_string()); - let variant = match primitive.as_str() { - "ncd_sym" | "ncd_sym_vitanyi" => NcdVariant::SymVitanyi, - "ncd_cons" => NcdVariant::Cons, - "ncd_sym_cons" => NcdVariant::SymCons, - _ => NcdVariant::Vitanyi, - }; - let b1 = read_file(&f1); - let b2 = read_file(&f2); - println!( - "{}", - ncd_bytes_backend(&b1, &b2, &ctx.compression_backend, variant) - ); - if let Err(e) = - maybe_export_online_model(model_export_path.as_deref(), &ctx, &[&b1, &b2]) - { - eprintln!("Error exporting online model: {e}"); - std::process::exit(1); - } - } - "entropy" | "h" | "entropy_rate" | "h_rate" => { - let f1 = file1.unwrap_or_exit("Error: 'h' requires a file"); - let default_order = if primitive.contains("rate") || rate_backend_specified { - expert_spec_max_order.unwrap_or(-1) - } else { - 0 - }; - let max_order = pos_arg3 - .and_then(|s| s.parse().ok()) - .unwrap_or(default_order); - let data = read_file(&f1); - if max_order == 0 && !primitive.contains("rate") && !rate_backend_specified { - println!("{}", marginal_entropy_bytes(&data)); - } else { - println!("{}", ctx.entropy_rate_bytes(&data, max_order)); - } - if let Err(e) = maybe_export_online_model(model_export_path.as_deref(), &ctx, &[&data]) - { - eprintln!("Error exporting online model: {e}"); - std::process::exit(1); - } - } - "id" | "intrinsic_dep" => { - let f1 = file1.unwrap_or_exit("Error: 'id' requires a file"); - let max_order = pos_arg3 - .and_then(|s| s.parse().ok()) - .unwrap_or(expert_spec_max_order.unwrap_or(-1)); - let data = read_file(&f1); - println!("{:.6}", intrinsic_dependence_bytes(&data, max_order)); - if let Err(e) = maybe_export_online_model(model_export_path.as_deref(), &ctx, &[&data]) - { - eprintln!("Error exporting online model: {e}"); - std::process::exit(1); - } - } - other => { - let f1 = file1.unwrap_or_exit("Error: requires two files"); - let f2 = file2.unwrap_or_exit("Error: requires two files"); - let default_order = if rate_backend_specified { - expert_spec_max_order.unwrap_or(-1) - } else { - 0 - }; - let max_order = pos_arg3 - .and_then(|s| s.parse().ok()) - .unwrap_or(default_order); - let b1 = read_file(&f1); - let b2 = read_file(&f2); - let res = match other { - "ned" => ned_bytes(&b1, &b2, max_order), - "ned_cons" => ned_cons_bytes(&b1, &b2, max_order), - "nte" => nte_bytes(&b1, &b2, max_order), - "mi" | "mutual_info" => mutual_information_bytes(&b1, &b2, max_order), - "ce" | "conditional_entropy" => conditional_entropy_bytes(&b1, &b2, max_order), - "xe" | "cross_entropy" => cross_entropy_bytes(&b1, &b2, max_order), - "joint_entropy" | "h_xy" => { - if max_order == 0 { - joint_marginal_entropy_bytes(&b1, &b2) - } else { - joint_entropy_rate_bytes(&b1, &b2, max_order) - } - } - "rt" | "resistance" => resistance_to_transformation_bytes(&b1, &b2, max_order), - "tvd" => tvd_paths(&f1, &f2, max_order), - "nhd" => nhd_paths(&f1, &f2, max_order), - "kl" | "kl_divergence" => kl_divergence_paths(&f1, &f2), - "js" | "js_divergence" => js_divergence_paths(&f1, &f2), - _ => { - eprintln!("Unknown primitive: {}", other); - print_usage(); - return; - } - }; - println!("{}", res); - if let Err(e) = - maybe_export_online_model(model_export_path.as_deref(), &ctx, &[&b1, &b2]) - { - eprintln!("Error exporting online model: {e}"); - std::process::exit(1); - } - } - } -} - -fn print_usage() { - let rate_backends = infotheory::backends::AVAILABLE_RATE_BACKENDS - .iter() - .enumerate() - .map(|(idx, name)| { - if idx == 0 { - format!("'{name}' (default)") - } else { - format!("'{name}'") - } - }) - .collect::>() - .join(", "); - let compression_backends = infotheory::backends::AVAILABLE_COMPRESSION_BACKENDS - .iter() - .enumerate() - .map(|(idx, name)| { - if idx == 0 { - format!("'{name}' (default)") - } else { - format!("'{name}'") - } - }) - .collect::>() - .join(", "); - - eprintln!( - r#"InfoTheory CLI -Usage: infotheory [args...] [options] - -Primitives: - Entropy & Information: - h, entropy [max_order] Entropy (marginal if order=0, rate if >0) - h_rate, entropy_rate [max_order] Force entropy rate estimation - mi, mutual_info [max_order] Mutual Information I(X;Y) - xe, cross_entropy [max_order] Cross Entropy H(X,Y) - H(Y)? (Check def) - ce, conditional_entropy Conditional Entropy H(X|Y) - joint_entropy, h_xy Joint Entropy H(X,Y) - id, intrinsic_dep [max_order] Intrinsic Dependence - - Distance & Divergence: - ncd [method] Normalized Compression Distance (Vitanyi) - ncd_sym, ncd_cons, ncd_sym_cons NCD variants (Symmetric, Conservative, etc.) - ned [max_order] Normalized Entropy Distance - nte [max_order] Normalized Transform Effort - kl, kl_divergence Kullback-Leibler Divergence - js, js_divergence Jensen-Shannon Divergence - tvd Total Variation Distance - nhd Normalized Hellinger Distance - rt, resistance Resistance to Transformation - - Tools: - search [options] Search target using info-theoretic ranking - aixi Run AIXI agent - batch Run in JSON-L batch mode - generate [file] [max_order] Generate continuation from file or piped stdin - compress Compress file using selected compression backend - decompress Decompress file using selected compression backend - ac-log-loss --mixture --out-prefix - Emit exact AC/log-loss TSV diagnostics for a mixture - sequitur-debug |--hex [--hex ...] - Emit canonical Sequitur grammar and bounded predictive traces - -Options: - --rate-backend Backend for rate estimation: {rate_backends} - --compression-backend - Backend for NCD/compression: {compression_backends} - --ncd-backend Deprecated alias for --compression-backend - --method Method/config (e.g. '5' for zpaq, '16' for ctw, mixture spec path, - model method: file:/path/model.safetensors[;policy:...] or cfg:key=value,...[;policy:...]) - --expert-spec Load one exact standalone expert JSON (same schema as a mixture 'experts' entry) - --model-export Optional online model export path (.safetensors + .json sidecar) - --rwkv-export Backward-compatible alias for --model-export - --mixture Mixture spec for 'ac-log-loss' - --out-prefix Output prefix for 'ac-log-loss' TSVs - --hex Hex-encoded byte string for 'sequitur-debug' (repeatable) - --context-bytes Sequitur context width (default: 64) - --alphabet-prefix Prefix of predictive PDF to emit for 'sequitur-debug' - --bytes Bytes to generate for 'generate' (default: 8) - --sample Use seeded sampling for generation - --greedy Force deterministic greedy generation - --adaptive Keep fitting on generated bytes instead of frozen continuation - --seed RNG seed for sampled generation - --temperature Sampling temperature (default: 1.0) - --top-k Sample only from the top-k bytes (0 disables) - --top-p

Nucleus sampling threshold in (0, 1] - -Examples: - infotheory ncd file1.txt file2.txt --compression-backend zpaq --method 5 - infotheory ncd file1.txt file2.txt --compression-backend rate-ac --rate-backend ctw - infotheory h file.txt --expert-spec ./expert.json - infotheory h file.txt --rate-backend mamba --method "cfg:hidden=128,layers=2,intermediate=256,state=16,conv=4,train=adam,lr=0.001;policy:schedule=0..100:train(scope=head+bias,opt=adam,lr=0.001,stride=1,bptt=1,clip=0,momentum=0.9)" --model-export ./mamba_online.safetensors - infotheory h file.txt --rate-backend ctw --method 32 - infotheory h file.txt --rate-backend mixture --method mixture.json - infotheory sequitur-debug --hex 616263616263 --alphabet-prefix 8 - infotheory search "encryption" ./src --prior "codebase context" - cat prompt.txt | infotheory generate --rate-backend ctw --method 32 --bytes 8 - infotheory generate prompt.txt --rate-backend match --bytes 16 --sample --seed 7 - infotheory compress in.bin out.itc --compression-backend rate-ac --rate-backend mixture --method mixture.json - infotheory decompress out.itc restored.bin --compression-backend rate-ac --rate-backend mixture --method mixture.json - RAYON_NUM_THREADS=4 infotheory ac-log-loss corpus.bin --mixture examples/mixture_spec.json --out-prefix /tmp/mixture-diagnostic -"# - ); -} - -#[cfg(test)] -mod tests { - use super::*; - use serde_json::json; - #[cfg(any(feature = "backend-mamba", feature = "backend-rwkv"))] - use std::any::Any; - use std::panic; - use std::path::{Path, PathBuf}; - use std::sync::atomic::{AtomicU64, Ordering}; - use std::time::{SystemTime, UNIX_EPOCH}; - - static TEMP_TEST_PATH_COUNTER: AtomicU64 = AtomicU64::new(0); - - fn unique_temp_path(prefix: &str, suffix: &str) -> PathBuf { - let counter = TEMP_TEST_PATH_COUNTER.fetch_add(1, Ordering::Relaxed); - let nanos = SystemTime::now() - .duration_since(UNIX_EPOCH) - .map(|d| d.as_nanos()) - .unwrap_or(0); - std::env::temp_dir().join(format!( - "{prefix}-{}-{nanos}-{counter}{suffix}", - std::process::id() - )) - } - - #[test] - fn file_roundtrip_backend_keeps_zpaq_unchanged() { - let b = CompressionBackend::Zpaq { - method: "5".to_string(), - }; - let out = file_roundtrip_backend(&b); - assert!(matches!(out, CompressionBackend::Zpaq { method } if method == "5")); - } - - #[test] - fn file_roundtrip_backend_forces_rate_framed() { - let b = CompressionBackend::Rate { - rate_backend: RateBackend::Ctw { depth: 8 }, - coder: infotheory::coders::CoderType::AC, - framing: infotheory::compression::FramingMode::Raw, - }; - let out = file_roundtrip_backend(&b); - match out { - CompressionBackend::Rate { framing, .. } => { - assert_eq!(framing, infotheory::compression::FramingMode::Framed) - } - _ => panic!("expected rate backend"), - } - } - - #[test] - fn process_json_line_rejects_invalid_json() { - let out = process_json_line(r#"{"op":"metrics","text":"abc""#); - let parsed: serde_json::Value = serde_json::from_str(&out).expect("output should be json"); - assert!( - parsed - .get("error") - .and_then(|v| v.as_str()) - .unwrap_or("") - .contains("invalid json") - ); - } - - #[test] - fn process_json_line_parses_escaped_and_nested_json_correctly() { - let line = r#"{ - "op":"metrics", - "text":"hello\n\"json\"", - "meta":{"op":"ncd"}, - "max_order":-1 - }"#; - let out = process_json_line(line); - let parsed: serde_json::Value = serde_json::from_str(&out).expect("output should be json"); - assert!(parsed.get("h0").and_then(|v| v.as_f64()).unwrap_or(-1.0) >= 0.0); - assert_eq!(parsed.get("len").and_then(|v| v.as_u64()), Some(12)); - } - - #[cfg(feature = "backend-rwkv")] - #[test] - fn build_ctx_rwkv7_compression_accepts_cfg_method() { - let ctx = build_ctx( - "rosaplus", - "rwkv7", - Some( - "cfg:hidden=64,intermediate=64,layers=1,train=sgd,lr=0.01;policy:schedule=0..100:infer", - ), - None, - ) - .ctx; - - match ctx.compression_backend { - CompressionBackend::Rate { - rate_backend, - coder, - framing, - } => { - assert!(matches!(rate_backend, RateBackend::Rwkv7Method { .. })); - assert_eq!(coder, rwkvzip::CoderType::AC); - assert_eq!(framing, infotheory::compression::FramingMode::Raw); - } - _ => panic!("expected rate-coded RWKV backend for cfg: method"), - } - } - - #[test] - fn parse_backend_aliases_and_unknowns() { - assert_eq!(parse_rate_backend("rosa"), Some("rosaplus")); - assert_eq!(parse_rate_backend("facctw"), Some("fac-ctw")); - assert_eq!(parse_rate_backend("sparsematch"), Some("sparse-match")); - assert_eq!(parse_rate_backend("ppm"), Some("ppmd")); - assert_eq!(parse_rate_backend("cal"), Some("calibrated")); - assert_eq!(parse_rate_backend("unknown"), None); - - assert_eq!(parse_compression_backend("unknown"), None); - #[cfg(feature = "backend-zpaq")] - assert_eq!(parse_compression_backend("zpaq"), Some("zpaq")); - assert_eq!(parse_compression_backend("rate_ac"), Some("rate-ac")); - assert_eq!(parse_compression_backend("raterans"), Some("rate-rans")); - #[cfg(feature = "backend-rwkv")] - { - assert_eq!(parse_compression_backend("rwkv"), Some("rwkv7")); - } - #[cfg(feature = "backend-mamba")] - { - assert_eq!(parse_rate_backend("mamba1"), Some("mamba")); - } - } - - #[test] - fn parse_mixture_expert_supports_calibrated_and_match_backends() { - let base_dir = Path::new("."); - let expert = json!({ - "name": "cal-ctw", - "kind": "calibrated", - "context": "text", - "bins": 33, - "learning_rate": 0.02, - "bias_clip": 4.0, - "base": { - "kind": "match" - } - }); - let parsed = parse_mixture_expert_value(&expert, base_dir, 4).expect("expert should parse"); - match parsed.backend { - RateBackend::Calibrated { spec } => match spec.base { - RateBackend::Match { .. } => {} - _ => panic!("unexpected calibrated base"), - }, - _ => panic!("expected calibrated backend"), - } - } - - #[test] - fn parse_mixture_expert_supports_sequitur_backend() { - let base_dir = Path::new("."); - let expert = json!({ - "name": "sequitur", - "kind": "sequitur", - "context_bytes": 96 - }); - let parsed = parse_mixture_expert_value(&expert, base_dir, 4).expect("expert should parse"); - match parsed.backend { - RateBackend::Sequitur { context_bytes } => assert_eq!(context_bytes, 96), - _ => panic!("expected sequitur backend"), - } - } - - #[cfg(any(feature = "backend-mamba", feature = "backend-rwkv"))] - fn panic_message(payload: Box) -> String { - if let Some(s) = payload.downcast_ref::() { - return s.clone(); - } - if let Some(s) = payload.downcast_ref::<&str>() { - return (*s).to_string(); - } - "non-string panic payload".to_string() - } - - #[cfg(feature = "backend-mamba")] - #[test] - fn parse_mixture_expert_resolves_mamba_model_path_relative_to_base_dir() { - let base_dir = unique_temp_path("infotheory-mamba-relpath", ""); - std::fs::create_dir_all(base_dir.join("weights")).expect("create temp dir"); - let rel_path = "weights/model.safetensors"; - let expected = base_dir.join(rel_path).to_string_lossy().to_string(); - let expert = json!({ - "name": "mamba-relative", - "kind": "mamba", - "model_path": rel_path - }); - let panic = panic::catch_unwind(|| { - let _ = parse_mixture_expert_value(&expert, &base_dir, 4); - }) - .expect_err("missing model should panic during load"); - let msg = panic_message(panic); - assert!( - msg.contains(&expected), - "panic should mention resolved absolute model path. expected substring: {expected}, got: {msg}" - ); - let _ = std::fs::remove_dir_all(&base_dir); - } - - #[cfg(feature = "backend-rwkv")] - #[test] - fn parse_mixture_expert_resolves_rwkv_model_path_relative_to_base_dir() { - let base_dir = unique_temp_path("infotheory-rwkv-relpath", ""); - std::fs::create_dir_all(base_dir.join("weights")).expect("create temp dir"); - let rel_path = "weights/model.safetensors"; - let expected = base_dir.join(rel_path).to_string_lossy().to_string(); - let expert = json!({ - "name": "rwkv-relative", - "kind": "rwkv7", - "model_path": rel_path - }); - let panic = panic::catch_unwind(|| { - let _ = parse_mixture_expert_value(&expert, &base_dir, 4); - }) - .expect_err("missing model should panic during load"); - let msg = panic_message(panic); - assert!( - msg.contains(&expected), - "panic should mention resolved absolute model path. expected substring: {expected}, got: {msg}" - ); - let _ = std::fs::remove_dir_all(&base_dir); - } - - #[test] - fn load_expert_spec_preserves_exact_ppmd_settings() { - let expert_path = unique_temp_path("infotheory-expert-spec", ".json"); - std::fs::write( - &expert_path, - serde_json::to_vec(&json!({ - "name": "ppmd", - "kind": "ppmd", - "order": 12, - "memory_mb": 256 - })) - .expect("expert json"), - ) - .expect("write temp expert spec"); - - let parsed = load_expert_spec(expert_path.to_str().expect("utf8 path")) - .expect("ppmd expert should load"); - match parsed.backend { - RateBackend::Ppmd { order, memory_mb } => { - assert_eq!(order, 12); - assert_eq!(memory_mb, 256); - } - _ => panic!("expected ppmd backend"), - } - - let _ = std::fs::remove_file(&expert_path); - } - - #[test] - fn build_ctx_propagates_expert_spec_max_order_default() { - let expert_path = unique_temp_path("infotheory-expert-spec-rosa", ".json"); - std::fs::write( - &expert_path, - serde_json::to_vec(&json!({ - "name": "rosa", - "kind": "rosaplus", - "max_order": 32 - })) - .expect("expert json"), - ) - .expect("write temp expert spec"); - - let built = build_ctx( - "rosaplus", - "zpaq", - None, - Some(expert_path.to_str().expect("utf8 path")), - ); - assert_eq!(built.expert_spec_max_order, Some(32)); - assert!(matches!(built.ctx.rate_backend, RateBackend::RosaPlus)); - - let _ = std::fs::remove_file(&expert_path); - } - - #[test] - fn parse_observation_helpers_cover_vm_and_non_vm_cases() { - let base = json!({ - "observation_stream_len": 3, - "observation_key_mode": "stream-hash" - }); - assert_eq!(parse_observation_stream_len(&base), 3); - assert_eq!( - parse_observation_key_mode(&base), - ObservationKeyMode::StreamHash - ); - assert_eq!( - parse_observation_key_mode_str("full"), - ObservationKeyMode::FullStream - ); - assert_eq!( - parse_observation_key_mode_str("last"), - ObservationKeyMode::Last - ); - assert_eq!( - parse_observation_key_mode_str("unknown"), - ObservationKeyMode::First - ); - - let vm = json!({ - "observation_stream_len": 2, - "observation_key_mode": "full", - "vm_observation": { - "stream_len": 2, - "key_mode": "last" - } - }); - assert_eq!( - parse_observation_stream_len_for_vm(&vm["vm_observation"]), - 2 - ); - assert_eq!( - parse_observation_key_mode_for_vm(&vm["vm_observation"]), - ObservationKeyMode::Last - ); - assert_eq!(parse_observation_stream_len_for_env(&vm, "vm"), 2); - assert_eq!( - parse_observation_key_mode_for_env(&vm, "vm"), - ObservationKeyMode::Last - ); - assert_eq!( - parse_observation_key_mode_for_env(&vm, "coin"), - ObservationKeyMode::FullStream - ); - - let mismatch = json!({ - "observation_stream_len": 2, - "vm_observation": { - "stream_len": 3 - } - }); - let err = validate_observation_config("vm", &mismatch, 2, ObservationKeyMode::FullStream) - .expect_err("mismatched vm stream_len should fail"); - assert!(err.to_string().contains("conflicts")); - - let mismatch_mode = json!({ - "observation_key_mode": "full", - "vm_observation": { - "key_mode": "last" - } - }); - let err = validate_observation_config("nyx", &mismatch_mode, 1, ObservationKeyMode::Last) - .expect_err("mismatched vm key mode should fail"); - assert!(err.to_string().contains("conflicts")); - } - - #[test] - fn parse_mixture_kind_and_spec_validation() { - assert_eq!( - parse_mixture_kind("bayes-mix").expect("bayes alias"), - MixtureKind::Bayes - ); - assert_eq!( - parse_mixture_kind("switch").expect("switch alias"), - MixtureKind::Switching - ); - assert_eq!( - parse_mixture_kind("convex").expect("convex kind"), - MixtureKind::Convex - ); - assert_eq!( - parse_mixture_kind("neural").expect("neural kind"), - MixtureKind::Neural - ); - assert!(parse_mixture_kind("nonsense").is_err()); - assert_eq!( - parse_mixture_schedule("theorem").expect("theorem schedule"), - MixtureScheduleMode::Theorem - ); - assert!(parse_mixture_schedule("nonsense").is_err()); - - let base_dir = Path::new("."); - let missing_experts = json!({ - "kind": "bayes", - "experts": [] - }); - assert!(parse_mixture_spec_value(&missing_experts, base_dir, 8).is_err()); - - let fading_without_decay = json!({ - "kind": "fading", - "experts": [ - {"name": "ctw-e", "kind": "ctw", "depth": 4} - ] - }); - assert!(parse_mixture_spec_value(&fading_without_decay, base_dir, 8).is_err()); - - let valid = json!({ - "kind": "convex", - "schedule": "theorem", - "experts": [ - {"name": "ctw-e", "kind": "ctw", "depth": 8}, - {"name": "fac-e", "kind": "fac-ctw", "base_depth": 8, "encoding_bits": 8} - ] - }); - let spec = parse_mixture_spec_value(&valid, base_dir, 8).expect("valid mixture"); - assert_eq!(spec.schedule, MixtureScheduleMode::Theorem); - assert_eq!(spec.experts.len(), 2); - assert!(matches!(spec.kind, MixtureKind::Convex)); - - let nested = json!({ - "kind": "convex", - "alpha": 1.25, - "experts": [ - { - "name": "nested", - "kind": "mixture", - "spec": { - "kind": "bayes", - "experts": [ - {"name": "ctw-e", "kind": "ctw", "depth": 4} - ] - } - }, - {"name": "match-e", "kind": "match", "hash_bits": 18} - ] - }); - let nested_spec = parse_mixture_spec_value(&nested, base_dir, 8).expect("nested mixture"); - assert!(matches!(nested_spec.kind, MixtureKind::Convex)); - assert_eq!(nested_spec.experts.len(), 2); - match &nested_spec.experts[0].backend { - RateBackend::Mixture { spec } => { - assert!(matches!(spec.kind, MixtureKind::Bayes)); - assert_eq!(spec.experts.len(), 1); - } - _ => panic!("expected nested mixture backend"), - } - } - - #[cfg(feature = "vm")] - #[test] - fn parse_vm_stats_backend_supports_new_backends_and_rejects_unknowns() { - let root = json!({ - "algorithm": "ctw", - "ct_depth": 8, - "observation_bits": 8, - "reward_bits": 8 - }); - let base_dir = Path::new("."); - - let matched = - parse_vm_stats_backend(&json!({"name":"match","hash_bits":18}), &root, base_dir) - .expect("match backend should parse"); - assert!(matches!(matched, RateBackend::Match { hash_bits: 18, .. })); - - let sparse = parse_vm_stats_backend( - &json!({"name":"sparse-match","gap_min":2,"gap_max":4}), - &root, - base_dir, - ) - .expect("sparse-match backend should parse"); - assert!(matches!( - sparse, - RateBackend::SparseMatch { - gap_min: 2, - gap_max: 4, - .. - } - )); - - let ppmd = parse_vm_stats_backend(&json!({"name":"ppmd","order":12}), &root, base_dir) - .expect("ppmd backend should parse"); - assert!(matches!(ppmd, RateBackend::Ppmd { order: 12, .. })); - - let sequitur = parse_vm_stats_backend( - &json!({"name":"sequitur","context_bytes":72}), - &root, - base_dir, - ) - .expect("sequitur backend should parse"); - assert!(matches!( - sequitur, - RateBackend::Sequitur { context_bytes: 72 } - )); - - let particle = parse_vm_stats_backend( - &json!({ - "name":"particle", - "spec":{"num_particles":4,"num_cells":4,"cell_dim":8} - }), - &root, - base_dir, - ) - .expect("particle backend should parse"); - assert!(matches!(particle, RateBackend::Particle { .. })); - - let mixture = parse_vm_stats_backend( - &json!({ - "name":"mixture", - "spec":{"kind":"bayes","experts":[{"kind":"match"}]} - }), - &root, - base_dir, - ) - .expect("mixture backend should parse"); - assert!(matches!(mixture, RateBackend::Mixture { .. })); - - let calibrated = parse_vm_stats_backend( - &json!({ - "name":"calibrated", - "base":{"kind":"ctw","depth":8}, - "context":"text", - "bins":17, - "learning_rate":0.05, - "bias_clip":3.0 - }), - &root, - base_dir, - ) - .expect("calibrated backend should parse"); - assert!(matches!(calibrated, RateBackend::Calibrated { .. })); - - let err = match parse_vm_stats_backend(&json!("unknown-backend"), &root, base_dir) { - Ok(_) => panic!("unknown backend should not silently fall back"), - Err(err) => err, - }; - assert!( - err.to_string().contains("unknown vm stats backend"), - "unexpected error: {err}" - ); - } -} diff --git a/src/mixture.rs b/src/mixture.rs deleted file mode 100644 index 4f1f2f4e..00000000 --- a/src/mixture.rs +++ /dev/null @@ -1,3588 +0,0 @@ -//! Online mixtures of probabilistic predictors (log-loss Hedge / Bayes, switching, MDL). -//! -//! This module provides a small, rigorously correct toolkit for sequential model mixing. -//! Predictors expose per-symbol log-probabilities, which allows principled Bayesian -//! mixture updates and clean information-theoretic accounting. -//! -//! ## Rate-Backend Mixtures -//! -//! The mixture primitives here power `RateBackend::Mixture`, enabling Bayes, fading Bayes, -//! switching, and MDL-style selectors to be used anywhere a rate backend is accepted. - -use crate::backends::calibration::CalibratorCore; -use crate::backends::match_model::MatchModel; -use crate::backends::ppmd::PpmdModel; -use crate::backends::sequitur::{SequiturCheckpoint, SequiturModel}; -use crate::backends::sparse_match::SparseMatchModel; -use crate::backends::text_context::TextContextAnalyzer; -use crate::ctw::FacContextTree; -#[cfg(feature = "backend-mamba")] -use crate::mambazip; -use crate::neural_mix::{NeuralHistoryState, NeuralMixCore}; -use crate::rosaplus::RosaPlus; -#[cfg(feature = "backend-rwkv")] -use crate::rwkvzip; -use crate::zpaq_rate::ZpaqRateModel; -use crate::{CalibratedSpec, MixtureKind, MixtureScheduleMode, MixtureSpec, RateBackend}; -use std::sync::Arc; - -/// Default minimum probability floor to avoid log(0). -pub const DEFAULT_MIN_PROB: f64 = 5.960_464_477_539_063e-8; - -#[inline] -fn clamp_prob(p: f64, min_prob: f64) -> f64 { - if p.is_finite() { - p.max(min_prob) - } else { - min_prob - } -} - -#[inline] -fn clamp_unit_prob(p: f64, min_prob: f64) -> f64 { - clamp_prob(p, min_prob).min(1.0 - min_prob) -} - -#[inline] -fn build_calibrator(spec: &CalibratedSpec) -> CalibratorCore { - CalibratorCore::new(spec.context, spec.bins, spec.learning_rate, spec.bias_clip) -} - -#[inline] -fn logsumexp(xs: &[f64]) -> f64 { - let mut max_v = f64::NEG_INFINITY; - for &v in xs { - if v > max_v { - max_v = v; - } - } - if !max_v.is_finite() { - return max_v; - } - let mut sum = 0.0; - for &v in xs { - sum += (v - max_v).exp(); - } - max_v + sum.ln() -} - -#[inline] -fn logsumexp2(a: f64, b: f64) -> f64 { - let m = if a > b { a } else { b }; - if !m.is_finite() { - return m; - } - m + ((a - m).exp() + (b - m).exp()).ln() -} - -#[inline] -fn logsumexp_weights(experts: &[ExpertState]) -> f64 { - let mut max_v = f64::NEG_INFINITY; - for e in experts { - if e.log_weight > max_v { - max_v = e.log_weight; - } - } - if !max_v.is_finite() { - return max_v; - } - let mut sum = 0.0; - for e in experts { - sum += (e.log_weight - max_v).exp(); - } - max_v + sum.ln() -} - -fn normalize_simplex_weights(weights: &mut [f64]) { - if weights.is_empty() { - return; - } - let mut sum = 0.0; - for weight in weights.iter_mut() { - if !weight.is_finite() || *weight < 0.0 { - *weight = 0.0; - } - sum += *weight; - } - if !sum.is_finite() || sum <= 0.0 { - let uniform = 1.0 / (weights.len() as f64); - weights.fill(uniform); - return; - } - for weight in weights.iter_mut() { - *weight /= sum; - } -} - -pub(crate) fn project_simplex_with_scratch(weights: &mut [f64], scratch: &mut Vec) { - if weights.is_empty() { - return; - } - - scratch.clear(); - scratch.extend( - weights - .iter() - .map(|&weight| if weight.is_finite() { weight } else { 0.0 }), - ); - let sorted = scratch.as_mut_slice(); - sorted.sort_by(|a, b| b.total_cmp(a)); - - let mut cumulative = 0.0; - let mut rho = None; - for (index, value) in sorted.iter().enumerate() { - cumulative += *value; - let theta = (cumulative - 1.0) / ((index + 1) as f64); - if *value > theta { - rho = Some(index); - } - } - - let Some(rho_index) = rho else { - let uniform = 1.0 / (weights.len() as f64); - weights.fill(uniform); - return; - }; - - let theta = (sorted.iter().take(rho_index + 1).sum::() - 1.0) / ((rho_index + 1) as f64); - for weight in weights.iter_mut() { - *weight = (*weight - theta).max(0.0); - } - normalize_simplex_weights(weights); -} - -#[inline] -pub(crate) fn switching_alpha_for_update( - schedule: MixtureScheduleMode, - alpha: f64, - processed_symbols: u64, -) -> f64 { - match schedule { - MixtureScheduleMode::Default => alpha.clamp(0.0, 1.0), - MixtureScheduleMode::Theorem => 1.0 / ((processed_symbols + 2) as f64), - } -} - -#[inline] -pub(crate) fn convex_step_size_for_update( - schedule: MixtureScheduleMode, - alpha: f64, - update_index: u64, -) -> f64 { - let t = update_index.max(1) as f64; - match schedule { - MixtureScheduleMode::Default => alpha.max(1e-12) / t.sqrt(), - MixtureScheduleMode::Theorem => DEFAULT_MIN_PROB / t.sqrt(), - } -} - -fn normalized_prior_weights(configs: &[ExpertConfig]) -> Vec { - if configs.is_empty() { - return Vec::new(); - } - let max_log = configs - .iter() - .map(|cfg| cfg.log_prior) - .fold(f64::NEG_INFINITY, f64::max); - let mut weights = configs - .iter() - .map(|cfg| { - if max_log.is_finite() { - (cfg.log_prior - max_log).exp() - } else { - 0.0 - } - }) - .collect::>(); - normalize_simplex_weights(&mut weights); - weights -} - -fn set_log_weights_from_linear(experts: &mut [ExpertState], weights: &[f64]) { - for (expert, &weight) in experts.iter_mut().zip(weights.iter()) { - expert.log_weight = if weight > 0.0 { - weight.ln() - } else { - f64::NEG_INFINITY - }; - } -} - -/// Trait for online byte-level predictors that expose per-symbol log-probabilities. -pub trait OnlineBytePredictorClone { - /// Clone this predictor as a trait object. - /// - /// This supports `Clone` for `Box` via type erasure, - /// so mixture experts can be duplicated without knowing their concrete type. - fn clone_box(&self) -> Box; -} - -impl OnlineBytePredictorClone for T -where - T: 'static + OnlineBytePredictor + Clone, -{ - fn clone_box(&self) -> Box { - Box::new(self.clone()) - } -} - -impl Clone for Box { - fn clone(&self) -> Self { - self.clone_box() - } -} - -/// Trait for online byte-level predictors that expose per-symbol log-probabilities. -pub trait OnlineBytePredictor: Send + OnlineBytePredictorClone { - /// Optional stream-start hook. - /// - /// Predictors that require total symbol count (for example percent-based - /// policy schedules) can initialize runtime state here. - fn begin_stream(&mut self, _total_symbols: Option) -> Result<(), String> { - Ok(()) - } - - /// Optional stream-finalization hook. - fn finish_stream(&mut self) -> Result<(), String> { - Ok(()) - } - - /// Log-probability (natural log) of `symbol` given the current history. - fn log_prob(&mut self, symbol: u8) -> f64; - - /// Bulk 256-way log-probabilities for the next byte. - fn fill_log_probs(&mut self, out: &mut [f64; 256]) { - for (sym, slot) in out.iter_mut().enumerate() { - *slot = self.log_prob(sym as u8); - } - } - - /// Log-probability (natural log) of `symbol`, then update the predictor. - fn log_prob_update(&mut self, symbol: u8) -> f64 { - let logp = self.log_prob(symbol); - self.update(symbol); - logp - } - - /// Update the predictor with the observed `symbol`. - fn update(&mut self, symbol: u8); - - /// Reset only dynamic conditioning state while preserving fitted parameters/statistics. - /// - /// Predictors with latent/posterior state may also preserve their learned - /// parameter posterior here; "frozen" means no new parameter fitting during - /// the score pass, not necessarily a static hidden-state belief. - fn reset_frozen(&mut self, total_symbols: Option) -> Result<(), String> { - self.finish_stream()?; - self.begin_stream(total_symbols) - } - - /// Advance conditioning state without fitting or adapting parameters. - /// - /// For state-space or latent-variable models this may still update internal - /// filtering/posterior state needed for correct sequential predictions. - fn update_frozen(&mut self, symbol: u8) { - self.update(symbol); - } -} - -#[cfg(feature = "backend-rwkv")] -#[inline] -fn ensure_rwkv_primed(compressor: &mut rwkvzip::Compressor, primed: &mut bool) { - if !*primed { - compressor.reset_and_prime(); - *primed = true; - } -} - -#[inline] -fn ctw_log_prob_update_msb(tree: &mut FacContextTree, symbol: u8, min_prob: f64) -> f64 { - let mut logp = 0.0; - for bit_idx in 0..8 { - let bit = ((symbol >> (7 - bit_idx)) & 1) == 1; - let p = tree.predict(bit, bit_idx); - if p.is_finite() && p > 0.0 { - logp += p.ln(); - } else { - logp = f64::NEG_INFINITY; - } - tree.update_predicted(bit, bit_idx); - } - if logp.is_finite() { - logp.max(min_prob.ln()) - } else { - min_prob.ln() - } -} - -#[inline] -fn ctw_log_prob_update_lsb( - tree: &mut FacContextTree, - symbol: u8, - bits_per_symbol: usize, - min_prob: f64, -) -> f64 { - let mut logp = 0.0; - for bit_idx in 0..bits_per_symbol { - let bit = ((symbol >> bit_idx) & 1) == 1; - let p = tree.predict(bit, bit_idx); - if p.is_finite() && p > 0.0 { - logp += p.ln(); - } else { - logp = f64::NEG_INFINITY; - } - tree.update_predicted(bit, bit_idx); - } - if logp.is_finite() { - logp.max(min_prob.ln()) - } else { - min_prob.ln() - } -} - -fn fill_fac_tree_log_probs( - tree: &mut FacContextTree, - bits_per_symbol: usize, - msb_first: bool, - min_logp: f64, - out: &mut [f64; 256], -) { - struct RecParams { - bits: usize, - msb_first: bool, - log_before: f64, - min_logp: f64, - } - - let bits = bits_per_symbol.clamp(1, 8); - let patterns = 1usize << bits; - let mut pattern_logps = [f64::NEG_INFINITY; 256]; - let params = RecParams { - bits, - msb_first, - log_before: tree.get_log_block_probability(), - min_logp, - }; - - fn rec( - tree: &mut FacContextTree, - depth: usize, - params: &RecParams, - symbol_acc: u8, - pattern_logps: &mut [f64; 256], - ) { - if depth == params.bits { - let pat = symbol_acc as usize; - let logp = (tree.get_log_block_probability() - params.log_before).max(params.min_logp); - pattern_logps[pat] = logp; - return; - } - - for bit in [false, true] { - tree.update(bit, depth); - let mut next_symbol = symbol_acc; - if params.msb_first { - let shift = 7usize.saturating_sub(depth); - if bit { - next_symbol |= 1u8 << shift; - } - } else if bit { - next_symbol |= 1u8 << depth; - } - rec(tree, depth + 1, params, next_symbol, pattern_logps); - tree.revert(depth); - } - } - - rec(tree, 0, ¶ms, 0, &mut pattern_logps); - - if bits == 8 { - out.copy_from_slice(&pattern_logps); - } else { - let aliases = 1usize << (8 - bits); - let alias_ln = (aliases as f64).ln(); - let mask = patterns - 1; - for byte in 0..256usize { - out[byte] = pattern_logps[byte & mask] - alias_ln; - } - } -} - -/// A concrete online predictor backed by a `RateBackend` configuration. -#[allow(clippy::large_enum_variant)] -#[derive(Clone)] -pub enum RateBackendPredictor { - /// ROSA-Plus online suffix automaton. - Rosa { - /// ROSA model state. - model: RosaPlus, - /// Probability floor for numeric stability. - min_prob: f64, - }, - /// Local contiguous match predictor. - Match { - /// Match model state. - model: MatchModel, - /// Probability floor for numeric stability. - min_prob: f64, - }, - /// Sparse/gapped local match predictor. - SparseMatch { - /// Sparse-match model state. - model: SparseMatchModel, - /// Probability floor for numeric stability. - min_prob: f64, - }, - /// Bounded-memory PPMD-style predictor. - Ppmd { - /// PPMD model state. - model: PpmdModel, - /// Probability floor for numeric stability. - min_prob: f64, - }, - /// Exact online Sequitur grammar backend with predictive suffix contexts. - Sequitur { - /// Sequitur model state. - model: SequiturModel, - /// Probability floor for numeric stability. - min_prob: f64, - }, - /// Byte-wise CTW implemented as 8 factorized bit trees (MSB-first). - Ctw { - /// FAC-CTW tree stack (8 bits per byte). - tree: FacContextTree, - /// Probability floor for numeric stability. - min_prob: f64, - }, - /// Factorized CTW with configurable bit-encoding (LSB-first). - FacCtw { - /// FAC-CTW tree stack for configured bit width. - tree: FacContextTree, - /// Active bit-width per symbol. - bits_per_symbol: usize, - /// Probability floor for numeric stability. - min_prob: f64, - }, - /// RWKV-7 neural predictor. - #[cfg(feature = "backend-rwkv")] - Rwkv7 { - /// RWKV compressor/runtime state. - compressor: rwkvzip::Compressor, - /// Whether the first-token distribution has been primed. - primed: bool, - /// Scratch copy used for update API that borrows immutable PDF. - pdf_scratch: Vec, - /// Probability floor for numeric stability. - min_prob: f64, - }, - /// Mamba-1 neural predictor. - #[cfg(feature = "backend-mamba")] - Mamba { - /// Mamba compressor/runtime state. - compressor: mambazip::Compressor, - /// Whether the first-token distribution has been primed. - primed: bool, - /// Scratch copy used for update API that borrows immutable PDF. - pdf_scratch: Vec, - /// Probability floor for numeric stability. - min_prob: f64, - }, - /// ZPAQ streaming rate model. - Zpaq { - /// ZPAQ rate model state. - model: ZpaqRateModel, - }, - /// Online mixture over experts (Bayes, fading Bayes, switching, MDL). - Mixture { - /// Active mixture runtime. - runtime: MixtureRuntime, - }, - /// Particle-latent filter ensemble. - Particle { - /// Particle runtime. - runtime: crate::particle::ParticleRuntime, - }, - /// Calibrated wrapper around another predictor. - Calibrated { - /// Wrapped predictor whose PDF is calibrated. - base: Box, - /// Online calibrator state and context features. - core: CalibratorCore, - /// Cached calibrated PDF. - pdf: [f64; 256], - /// Whether `pdf` currently matches wrapped state. - valid: bool, - /// Probability floor used for numerical stability. - min_prob: f64, - }, -} - -#[derive(Clone)] -/// Checkpoint snapshot used for temporary predictor rollback. -/// -/// Most backends use a full cloned predictor snapshot. Sequitur uses a compact -/// internal checkpoint to avoid cloning its full state. -pub enum RateBackendPredictorCheckpoint { - /// Full predictor clone for backends without specialized checkpointing. - Full(RateBackendPredictor), - /// Compact Sequitur undo marker for [`RateBackendPredictor::Sequitur`]. - Sequitur(SequiturCheckpoint), -} - -impl RateBackendPredictor { - /// Create a new online predictor from a rate backend configuration. - pub fn from_backend(backend: RateBackend, max_order: i64, min_prob: f64) -> Self { - match backend { - RateBackend::RosaPlus => { - let mut model = RosaPlus::new(max_order, false, 0, 42); - model.build_lm_full_bytes_no_finalize_endpos(); - Self::Rosa { model, min_prob } - } - RateBackend::Match { - hash_bits, - min_len, - max_len, - base_mix, - confidence_scale, - } => Self::Match { - model: MatchModel::new_contiguous( - hash_bits, - min_len, - max_len, - base_mix, - confidence_scale, - ), - min_prob, - }, - RateBackend::SparseMatch { - hash_bits, - min_len, - max_len, - gap_min, - gap_max, - base_mix, - confidence_scale, - } => Self::SparseMatch { - model: SparseMatchModel::new( - hash_bits, - min_len, - max_len, - gap_min, - gap_max, - base_mix, - confidence_scale, - ), - min_prob, - }, - RateBackend::Ppmd { order, memory_mb } => Self::Ppmd { - model: PpmdModel::new(order, memory_mb), - min_prob, - }, - RateBackend::Sequitur { context_bytes } => Self::Sequitur { - model: SequiturModel::new(context_bytes), - min_prob, - }, - RateBackend::Ctw { depth } => { - let tree = FacContextTree::new(depth, 8); - Self::Ctw { tree, min_prob } - } - RateBackend::FacCtw { - base_depth, - num_percept_bits: _, - encoding_bits, - } => { - let bits_per_symbol = encoding_bits.clamp(1, 8); - let tree = FacContextTree::new(base_depth, bits_per_symbol); - Self::FacCtw { - tree, - bits_per_symbol, - min_prob, - } - } - #[cfg(feature = "backend-rwkv")] - RateBackend::Rwkv7 { model } => { - let mut compressor = rwkvzip::Compressor::new_from_model(model); - compressor.reset_and_prime(); - Self::Rwkv7 { - pdf_scratch: vec![0.0; compressor.pdf_buffer.len()], - compressor, - primed: true, - min_prob, - } - } - #[cfg(feature = "backend-rwkv")] - RateBackend::Rwkv7Method { method } => { - let mut compressor = rwkvzip::Compressor::new_from_method(&method) - .unwrap_or_else(|e| panic!("invalid rwkv method '{method}': {e}")); - compressor.reset_and_prime(); - Self::Rwkv7 { - pdf_scratch: vec![0.0; compressor.pdf_buffer.len()], - compressor, - primed: true, - min_prob, - } - } - #[cfg(feature = "backend-mamba")] - RateBackend::Mamba { model } => { - let mut compressor = mambazip::Compressor::new_from_model(model); - let bias = compressor.online_bias_snapshot(); - let logits = - compressor - .model - .forward(&mut compressor.scratch, 0, &mut compressor.state); - mambazip::Compressor::logits_to_pdf( - logits, - bias.as_deref(), - &mut compressor.pdf_buffer, - ); - Self::Mamba { - pdf_scratch: vec![0.0; compressor.pdf_buffer.len()], - compressor, - primed: true, - min_prob, - } - } - #[cfg(feature = "backend-mamba")] - RateBackend::MambaMethod { method } => { - let mut compressor = mambazip::Compressor::new_from_method(&method) - .unwrap_or_else(|e| panic!("invalid mamba method '{method}': {e}")); - let bias = compressor.online_bias_snapshot(); - let logits = - compressor - .model - .forward(&mut compressor.scratch, 0, &mut compressor.state); - mambazip::Compressor::logits_to_pdf( - logits, - bias.as_deref(), - &mut compressor.pdf_buffer, - ); - Self::Mamba { - pdf_scratch: vec![0.0; compressor.pdf_buffer.len()], - compressor, - primed: true, - min_prob, - } - } - RateBackend::Zpaq { method } => { - let model = ZpaqRateModel::new(method, min_prob); - Self::Zpaq { model } - } - RateBackend::Mixture { spec } => { - let experts = spec.build_experts(); - let runtime = build_mixture_runtime(spec.as_ref(), &experts) - .unwrap_or_else(|e| panic!("MixtureSpec invalid: {e}")); - Self::Mixture { runtime } - } - RateBackend::Particle { spec } => { - let runtime = crate::particle::ParticleRuntime::new(spec.as_ref()); - Self::Particle { runtime } - } - RateBackend::Calibrated { spec } => Self::Calibrated { - base: Box::new(Self::from_backend(spec.base.clone(), max_order, min_prob)), - core: build_calibrator(spec.as_ref()), - pdf: [1.0 / 256.0; 256], - valid: false, - min_prob, - }, - } - } - - /// Human-readable default name for a backend + config. - pub fn default_name(backend: &RateBackend, max_order: i64) -> String { - match backend { - RateBackend::RosaPlus => format!("rosa(mo={})", max_order), - RateBackend::Match { .. } => "match".to_string(), - RateBackend::SparseMatch { .. } => "sparse-match".to_string(), - RateBackend::Ppmd { order, memory_mb } => { - format!("ppmd(o={},m={}MiB)", order, memory_mb) - } - RateBackend::Sequitur { context_bytes } => { - format!("sequitur(ctx={context_bytes})") - } - RateBackend::Ctw { depth } => format!("ctw(d={})", depth), - RateBackend::FacCtw { - base_depth, - encoding_bits, - .. - } => format!("fac-ctw(d={},b={})", base_depth, encoding_bits), - #[cfg(feature = "backend-rwkv")] - RateBackend::Rwkv7 { .. } => "rwkv7".to_string(), - #[cfg(feature = "backend-rwkv")] - RateBackend::Rwkv7Method { method } => format!("rwkv7({method})"), - #[cfg(feature = "backend-mamba")] - RateBackend::Mamba { .. } => "mamba".to_string(), - #[cfg(feature = "backend-mamba")] - RateBackend::MambaMethod { method } => format!("mamba({method})"), - RateBackend::Zpaq { method } => format!("zpaq(m={})", method), - RateBackend::Mixture { spec } => { - let kind = match spec.kind { - MixtureKind::Bayes => "bayes", - MixtureKind::FadingBayes => "fading", - MixtureKind::Switching => "switch", - MixtureKind::Convex => "convex", - MixtureKind::Mdl => "mdl", - MixtureKind::Neural => "neural", - }; - format!("mix({})", kind) - } - RateBackend::Particle { spec } => { - format!("particle(n={},c={})", spec.num_particles, spec.num_cells) - } - RateBackend::Calibrated { spec } => { - format!("calibrated({})", Self::default_name(&spec.base, max_order)) - } - } - } - - pub(crate) fn checkpoint(&mut self) -> RateBackendPredictorCheckpoint { - match self { - RateBackendPredictor::Sequitur { model, .. } => { - RateBackendPredictorCheckpoint::Sequitur(model.checkpoint()) - } - _ => RateBackendPredictorCheckpoint::Full(self.clone()), - } - } - - pub(crate) fn restore_checkpoint(&mut self, checkpoint: &RateBackendPredictorCheckpoint) { - match (self, checkpoint) { - ( - RateBackendPredictor::Sequitur { model, .. }, - RateBackendPredictorCheckpoint::Sequitur(ck), - ) => { - model.restore(ck); - } - (slot, RateBackendPredictorCheckpoint::Full(state)) => { - *slot = state.clone(); - } - (_, RateBackendPredictorCheckpoint::Sequitur(_)) => { - panic!("mismatched RateBackendPredictor checkpoint variant") - } - } - } - - pub(crate) fn clear_checkpoints_if_supported(&mut self) { - if let RateBackendPredictor::Sequitur { model, .. } = self { - model.clear_checkpoints(); - } - } -} - -impl OnlineBytePredictor for RateBackendPredictor { - fn begin_stream(&mut self, total_symbols: Option) -> Result<(), String> { - self.finish_stream()?; - match self { - RateBackendPredictor::Rosa { model, .. } => { - if let Some(total) = total_symbols { - let reserve = usize::try_from(total).unwrap_or(usize::MAX / 4); - model.reserve_for_stream(reserve); - } - Ok(()) - } - RateBackendPredictor::Match { .. } - | RateBackendPredictor::SparseMatch { .. } - | RateBackendPredictor::Ppmd { .. } => Ok(()), - RateBackendPredictor::Sequitur { model, .. } => { - model.begin_stream(total_symbols); - Ok(()) - } - RateBackendPredictor::Ctw { .. } - | RateBackendPredictor::FacCtw { .. } - | RateBackendPredictor::Zpaq { .. } - | RateBackendPredictor::Particle { .. } => Ok(()), - #[cfg(feature = "backend-rwkv")] - RateBackendPredictor::Rwkv7 { compressor, .. } => compressor - .begin_online_policy_stream(total_symbols) - .map_err(|e| e.to_string()), - #[cfg(feature = "backend-mamba")] - RateBackendPredictor::Mamba { compressor, .. } => compressor - .begin_online_policy_stream(total_symbols) - .map_err(|e| e.to_string()), - RateBackendPredictor::Mixture { runtime } => runtime.begin_stream(total_symbols), - RateBackendPredictor::Calibrated { base, .. } => base.begin_stream(total_symbols), - } - } - - fn finish_stream(&mut self) -> Result<(), String> { - match self { - RateBackendPredictor::Rosa { .. } - | RateBackendPredictor::Match { .. } - | RateBackendPredictor::SparseMatch { .. } - | RateBackendPredictor::Ppmd { .. } - | RateBackendPredictor::Ctw { .. } - | RateBackendPredictor::FacCtw { .. } - | RateBackendPredictor::Zpaq { .. } - | RateBackendPredictor::Particle { .. } => Ok(()), - RateBackendPredictor::Sequitur { model, .. } => { - model.finish_stream(); - Ok(()) - } - #[cfg(feature = "backend-rwkv")] - RateBackendPredictor::Rwkv7 { compressor, .. } => compressor - .finish_online_policy_stream() - .map_err(|e| e.to_string()), - #[cfg(feature = "backend-mamba")] - RateBackendPredictor::Mamba { .. } => Ok(()), - RateBackendPredictor::Mixture { runtime } => runtime.finish_stream(), - RateBackendPredictor::Calibrated { base, .. } => base.finish_stream(), - } - } - - fn log_prob(&mut self, symbol: u8) -> f64 { - match self { - RateBackendPredictor::Rosa { model, min_prob } => { - let p = clamp_prob(model.prob_for_last(symbol as u32), *min_prob); - p.ln() - } - RateBackendPredictor::Match { model, min_prob } => model.log_prob(symbol, *min_prob), - RateBackendPredictor::SparseMatch { model, min_prob } => { - model.log_prob(symbol, *min_prob) - } - RateBackendPredictor::Ppmd { model, min_prob } => model.log_prob(symbol, *min_prob), - RateBackendPredictor::Sequitur { model, min_prob } => model.log_prob(symbol, *min_prob), - RateBackendPredictor::Ctw { tree, min_prob } => { - let log_before = tree.get_log_block_probability(); - for bit_idx in 0..8 { - let bit = ((symbol >> (7 - bit_idx)) & 1) == 1; - tree.update(bit, bit_idx); - } - let log_after = tree.get_log_block_probability(); - for bit_idx in (0..8).rev() { - tree.revert(bit_idx); - } - let logp = log_after - log_before; - if logp.is_finite() { - logp.max(min_prob.ln()) - } else { - min_prob.ln() - } - } - RateBackendPredictor::FacCtw { - tree, - bits_per_symbol, - min_prob, - } => { - let log_before = tree.get_log_block_probability(); - for i in 0..*bits_per_symbol { - let bit = ((symbol >> i) & 1) == 1; - tree.update(bit, i); - } - let log_after = tree.get_log_block_probability(); - for i in (0..*bits_per_symbol).rev() { - tree.revert(i); - } - let logp = log_after - log_before; - if logp.is_finite() { - logp.max(min_prob.ln()) - } else { - min_prob.ln() - } - } - #[cfg(feature = "backend-rwkv")] - RateBackendPredictor::Rwkv7 { - compressor, - primed, - min_prob, - .. - } => { - ensure_rwkv_primed(compressor, primed); - let p = clamp_prob(compressor.pdf_buffer[symbol as usize], *min_prob); - p.ln() - } - #[cfg(feature = "backend-mamba")] - RateBackendPredictor::Mamba { - compressor, - primed, - min_prob, - .. - } => { - if !*primed { - let bias = compressor.online_bias_snapshot(); - let logits = - compressor - .model - .forward(&mut compressor.scratch, 0, &mut compressor.state); - mambazip::Compressor::logits_to_pdf( - logits, - bias.as_deref(), - &mut compressor.pdf_buffer, - ); - *primed = true; - } - let p = clamp_prob(compressor.pdf_buffer[symbol as usize], *min_prob); - p.ln() - } - RateBackendPredictor::Zpaq { model } => model.log_prob(symbol), - RateBackendPredictor::Mixture { runtime } => runtime.peek_log_prob(symbol), - RateBackendPredictor::Particle { runtime } => runtime.peek_log_prob(symbol), - RateBackendPredictor::Calibrated { - base, - core, - pdf, - valid, - min_prob, - } => { - if !*valid { - let mut base_logps = [0.0; 256]; - base.fill_log_probs(&mut base_logps); - let mut base_pdf = [0.0; 256]; - for (dst, &lp) in base_pdf.iter_mut().zip(base_logps.iter()) { - *dst = clamp_prob(lp.exp(), *min_prob); - } - core.apply_pdf(&base_pdf, pdf); - *valid = true; - } - pdf[symbol as usize].max(*min_prob).ln() - } - } - } - - fn fill_log_probs(&mut self, out: &mut [f64; 256]) { - match self { - RateBackendPredictor::Rosa { model, min_prob } => { - model.fill_probs_for_last_bytes(out); - for slot in out.iter_mut() { - *slot = clamp_prob(*slot, *min_prob).ln(); - } - } - RateBackendPredictor::Match { model, min_prob } => { - let mut pdf = [0.0; 256]; - model.fill_pdf(&mut pdf); - for (slot, &p) in out.iter_mut().zip(pdf.iter()) { - *slot = clamp_prob(p, *min_prob).ln(); - } - } - RateBackendPredictor::SparseMatch { model, min_prob } => { - let mut pdf = [0.0; 256]; - model.fill_pdf(&mut pdf); - for (slot, &p) in out.iter_mut().zip(pdf.iter()) { - *slot = clamp_prob(p, *min_prob).ln(); - } - } - RateBackendPredictor::Ppmd { model, min_prob } => { - let mut pdf = [0.0; 256]; - model.fill_pdf(&mut pdf); - for (slot, &p) in out.iter_mut().zip(pdf.iter()) { - *slot = clamp_prob(p, *min_prob).ln(); - } - } - RateBackendPredictor::Sequitur { model, min_prob } => { - let mut pdf = [0.0; 256]; - model.fill_pdf(&mut pdf); - for (slot, &p) in out.iter_mut().zip(pdf.iter()) { - *slot = clamp_prob(p, *min_prob).ln(); - } - } - RateBackendPredictor::Ctw { tree, min_prob } => { - fill_fac_tree_log_probs(tree, 8, true, min_prob.ln(), out); - } - RateBackendPredictor::FacCtw { - tree, - bits_per_symbol, - min_prob, - } => { - fill_fac_tree_log_probs(tree, *bits_per_symbol, false, min_prob.ln(), out); - } - #[cfg(feature = "backend-rwkv")] - RateBackendPredictor::Rwkv7 { - compressor, - primed, - min_prob, - .. - } => { - ensure_rwkv_primed(compressor, primed); - for (slot, &p_raw) in out - .iter_mut() - .take(256) - .zip(compressor.pdf_buffer.iter().take(256)) - { - let p = clamp_prob(p_raw, *min_prob); - *slot = p.ln(); - } - } - #[cfg(feature = "backend-mamba")] - RateBackendPredictor::Mamba { - compressor, - primed, - min_prob, - .. - } => { - if !*primed { - let bias = compressor.online_bias_snapshot(); - let logits = - compressor - .model - .forward(&mut compressor.scratch, 0, &mut compressor.state); - mambazip::Compressor::logits_to_pdf( - logits, - bias.as_deref(), - &mut compressor.pdf_buffer, - ); - *primed = true; - } - for (slot, &p_raw) in out - .iter_mut() - .take(256) - .zip(compressor.pdf_buffer.iter().take(256)) - { - let p = clamp_prob(p_raw, *min_prob); - *slot = p.ln(); - } - } - RateBackendPredictor::Zpaq { model } => { - model.fill_log_probs(out); - } - RateBackendPredictor::Mixture { runtime } => { - runtime.fill_log_probs(out); - } - RateBackendPredictor::Particle { runtime } => { - runtime.fill_log_probs_cached(out); - } - RateBackendPredictor::Calibrated { - base, - core, - pdf, - valid, - min_prob, - } => { - if !*valid { - let mut base_logps = [0.0; 256]; - base.fill_log_probs(&mut base_logps); - let mut base_pdf = [0.0; 256]; - for (dst, &lp) in base_pdf.iter_mut().zip(base_logps.iter()) { - *dst = clamp_prob(lp.exp(), *min_prob); - } - core.apply_pdf(&base_pdf, pdf); - *valid = true; - } - for (slot, &p) in out.iter_mut().zip(pdf.iter()) { - *slot = clamp_prob(p, *min_prob).ln(); - } - } - } - } - - fn update(&mut self, symbol: u8) { - match self { - RateBackendPredictor::Rosa { model, .. } => { - model.train_byte(symbol); - } - RateBackendPredictor::Match { model, .. } => { - model.update(symbol); - } - RateBackendPredictor::SparseMatch { model, .. } => { - model.update(symbol); - } - RateBackendPredictor::Ppmd { model, .. } => { - model.update(symbol); - } - RateBackendPredictor::Sequitur { model, .. } => { - model.update(symbol); - } - RateBackendPredictor::Ctw { tree, .. } => { - for bit_idx in 0..8 { - let bit = ((symbol >> (7 - bit_idx)) & 1) == 1; - tree.update(bit, bit_idx); - } - } - RateBackendPredictor::FacCtw { - tree, - bits_per_symbol, - .. - } => { - for i in 0..*bits_per_symbol { - let bit = ((symbol >> i) & 1) == 1; - tree.update(bit, i); - } - } - #[cfg(feature = "backend-rwkv")] - RateBackendPredictor::Rwkv7 { - compressor, primed, .. - } => { - ensure_rwkv_primed(compressor, primed); - compressor - .observe_symbol_from_current_pdf(symbol) - .unwrap_or_else(|e| panic!("rwkv online update failed: {e}")); - } - #[cfg(feature = "backend-mamba")] - RateBackendPredictor::Mamba { - compressor, - primed, - pdf_scratch, - .. - } => { - if !*primed { - let bias = compressor.online_bias_snapshot(); - let logits = - compressor - .model - .forward(&mut compressor.scratch, 0, &mut compressor.state); - mambazip::Compressor::logits_to_pdf( - logits, - bias.as_deref(), - &mut compressor.pdf_buffer, - ); - *primed = true; - } - if pdf_scratch.len() != compressor.pdf_buffer.len() { - pdf_scratch.resize(compressor.pdf_buffer.len(), 0.0); - } - pdf_scratch.copy_from_slice(&compressor.pdf_buffer); - compressor - .online_update_from_pdf(symbol, pdf_scratch) - .unwrap_or_else(|e| panic!("mamba online update failed: {e}")); - let bias = compressor.online_bias_snapshot(); - let logits = compressor.model.forward( - &mut compressor.scratch, - symbol as u32, - &mut compressor.state, - ); - mambazip::Compressor::logits_to_pdf( - logits, - bias.as_deref(), - &mut compressor.pdf_buffer, - ); - } - RateBackendPredictor::Zpaq { model } => { - model.update(symbol); - } - RateBackendPredictor::Mixture { runtime } => { - let _ = runtime.step(symbol); - } - RateBackendPredictor::Particle { runtime } => { - runtime.step(symbol); - } - RateBackendPredictor::Calibrated { - base, - core, - pdf, - valid, - .. - } => { - if !*valid { - let mut base_logps = [0.0; 256]; - base.fill_log_probs(&mut base_logps); - let mut base_pdf = [0.0; 256]; - for (dst, &lp) in base_pdf.iter_mut().zip(base_logps.iter()) { - *dst = clamp_prob(lp.exp(), DEFAULT_MIN_PROB); - } - core.apply_pdf(&base_pdf, pdf); - } - core.update(symbol, pdf); - base.update(symbol); - *valid = false; - } - } - } - - fn log_prob_update(&mut self, symbol: u8) -> f64 { - match self { - RateBackendPredictor::Rosa { model, min_prob } => { - let p = clamp_prob(model.prob_for_last(symbol as u32), *min_prob); - model.train_byte(symbol); - p.ln() - } - RateBackendPredictor::Ctw { tree, min_prob } => { - ctw_log_prob_update_msb(tree, symbol, *min_prob) - } - RateBackendPredictor::FacCtw { - tree, - bits_per_symbol, - min_prob, - } => ctw_log_prob_update_lsb(tree, symbol, *bits_per_symbol, *min_prob), - _ => { - let logp = self.log_prob(symbol); - self.update(symbol); - logp - } - } - } - - fn reset_frozen(&mut self, total_symbols: Option) -> Result<(), String> { - self.finish_stream()?; - match self { - RateBackendPredictor::Rosa { model, .. } => { - if let Some(total) = total_symbols { - let reserve = usize::try_from(total).unwrap_or(usize::MAX / 4); - model.reserve_for_stream(reserve); - } - model.build_lm_full_bytes_no_finalize_endpos(); - model.reset_conditioning_cursor(); - Ok(()) - } - RateBackendPredictor::Match { model, .. } => { - model.reset_history(); - Ok(()) - } - RateBackendPredictor::SparseMatch { model, .. } => { - model.reset_history(); - Ok(()) - } - RateBackendPredictor::Ppmd { model, .. } => { - model.reset_history(); - Ok(()) - } - RateBackendPredictor::Sequitur { model, .. } => { - model.reset_frozen(); - Ok(()) - } - RateBackendPredictor::Ctw { tree, .. } => { - tree.reset_history_only(); - Ok(()) - } - RateBackendPredictor::FacCtw { tree, .. } => { - tree.reset_history_only(); - Ok(()) - } - #[cfg(feature = "backend-rwkv")] - RateBackendPredictor::Rwkv7 { - compressor, primed, .. - } => { - compressor.reset_and_prime(); - *primed = true; - Ok(()) - } - #[cfg(feature = "backend-mamba")] - RateBackendPredictor::Mamba { - compressor, primed, .. - } => { - compressor.reset_and_prime(); - *primed = true; - Ok(()) - } - RateBackendPredictor::Zpaq { .. } => { - Err("plugin entropy is not supported for zpaq rate backends in 1.1.1".to_string()) - } - RateBackendPredictor::Mixture { runtime } => runtime.reset_frozen(total_symbols), - RateBackendPredictor::Particle { runtime } => { - runtime.reset_frozen_state(); - Ok(()) - } - RateBackendPredictor::Calibrated { - base, - core, - pdf, - valid, - .. - } => { - base.reset_frozen(total_symbols)?; - core.reset_context(); - pdf.fill(1.0 / 256.0); - *valid = false; - Ok(()) - } - } - } - - fn update_frozen(&mut self, symbol: u8) { - match self { - RateBackendPredictor::Rosa { model, .. } => { - model.advance_conditioning_byte(symbol); - } - RateBackendPredictor::Match { model, .. } => { - model.update_history_only(symbol); - } - RateBackendPredictor::SparseMatch { model, .. } => { - model.update_history_only(symbol); - } - RateBackendPredictor::Ppmd { model, .. } => { - model.update_history_only(symbol); - } - RateBackendPredictor::Sequitur { model, .. } => { - model.update_frozen(symbol); - } - RateBackendPredictor::Ctw { tree, .. } => { - let mut bits = [false; 8]; - for (bit_idx, slot) in bits.iter_mut().enumerate() { - *slot = ((symbol >> (7 - bit_idx)) & 1) == 1; - } - tree.update_history(&bits); - } - RateBackendPredictor::FacCtw { - tree, - bits_per_symbol, - .. - } => { - let bits = (*bits_per_symbol).clamp(1, 8); - let mut history_bits = [false; 8]; - for (idx, slot) in history_bits.iter_mut().enumerate().take(bits) { - *slot = ((symbol >> idx) & 1) == 1; - } - tree.update_history(&history_bits[..bits]); - } - #[cfg(feature = "backend-rwkv")] - RateBackendPredictor::Rwkv7 { - compressor, primed, .. - } => { - if !*primed { - compressor.reset_and_prime(); - *primed = true; - } - compressor.forward_to_internal_pdf(symbol as u32); - } - #[cfg(feature = "backend-mamba")] - RateBackendPredictor::Mamba { - compressor, primed, .. - } => { - if !*primed { - compressor.reset_and_prime(); - *primed = true; - } - let bias = compressor.online_bias_snapshot(); - let logits = compressor.model.forward( - &mut compressor.scratch, - symbol as u32, - &mut compressor.state, - ); - mambazip::Compressor::logits_to_pdf( - logits, - bias.as_deref(), - &mut compressor.pdf_buffer, - ); - } - RateBackendPredictor::Zpaq { model } => { - model.update(symbol); - } - RateBackendPredictor::Mixture { runtime } => { - runtime.update_frozen(symbol); - } - RateBackendPredictor::Particle { runtime } => { - runtime.update_frozen(symbol); - } - RateBackendPredictor::Calibrated { - base, - core, - pdf, - valid, - .. - } => { - if !*valid { - let mut base_logps = [0.0; 256]; - base.fill_log_probs(&mut base_logps); - let mut base_pdf = [0.0; 256]; - for (dst, &lp) in base_pdf.iter_mut().zip(base_logps.iter()) { - *dst = clamp_prob(lp.exp(), DEFAULT_MIN_PROB); - } - core.apply_pdf(&base_pdf, pdf); - *valid = true; - } - base.update_frozen(symbol); - core.update_context_only(symbol); - *valid = false; - } - } - } -} - -/// Configuration for a mixture expert. -#[derive(Clone)] -pub struct ExpertConfig { - /// Human-readable expert identifier. - pub name: String, - /// Log prior weight (natural log). Uniform priors can be `0.0`. - pub log_prior: f64, - builder: Arc Box + Send + Sync>, -} - -impl ExpertConfig { - /// Create a new expert config from a builder closure. - pub fn new( - name: impl Into, - log_prior: f64, - builder: impl Fn() -> Box + Send + Sync + 'static, - ) -> Self { - Self { - name: name.into(), - log_prior, - builder: Arc::new(builder), - } - } - - /// Uniform prior helper. - pub fn uniform( - name: impl Into, - builder: impl Fn() -> Box + Send + Sync + 'static, - ) -> Self { - Self::new(name, 0.0, builder) - } - - /// Expert from a `RateBackend` configuration. `max_order` applies to ROSA. - pub fn from_rate_backend( - name: Option, - log_prior: f64, - backend: RateBackend, - max_order: i64, - ) -> Self { - let name = name.unwrap_or_else(|| RateBackendPredictor::default_name(&backend, max_order)); - Self::new(name, log_prior, move || { - Box::new(RateBackendPredictor::from_backend( - backend.clone(), - max_order, - DEFAULT_MIN_PROB, - )) - }) - } - - /// ROSA expert (uniform prior). - pub fn rosa(name: impl Into, max_order: i64) -> Self { - let name = name.into(); - Self::uniform(name, move || { - Box::new(RateBackendPredictor::from_backend( - RateBackend::RosaPlus, - max_order, - DEFAULT_MIN_PROB, - )) - }) - } - - /// CTW expert (uniform prior). - pub fn ctw(name: impl Into, depth: usize) -> Self { - let name = name.into(); - Self::uniform(name, move || { - Box::new(RateBackendPredictor::from_backend( - RateBackend::Ctw { depth }, - -1, - DEFAULT_MIN_PROB, - )) - }) - } - - /// FAC-CTW expert (uniform prior). - pub fn fac_ctw(name: impl Into, base_depth: usize, encoding_bits: usize) -> Self { - let name = name.into(); - Self::uniform(name, move || { - Box::new(RateBackendPredictor::from_backend( - RateBackend::FacCtw { - base_depth, - num_percept_bits: encoding_bits, - encoding_bits, - }, - -1, - DEFAULT_MIN_PROB, - )) - }) - } - - /// RWKV-7 expert (uniform prior). - #[cfg(feature = "backend-rwkv")] - pub fn rwkv(name: impl Into, model: Arc) -> Self { - let name = name.into(); - Self::uniform(name, move || { - Box::new(RateBackendPredictor::from_backend( - RateBackend::Rwkv7 { - model: model.clone(), - }, - -1, - DEFAULT_MIN_PROB, - )) - }) - } - - /// Mamba expert (uniform prior). - #[cfg(feature = "backend-mamba")] - pub fn mamba(name: impl Into, model: Arc) -> Self { - let name = name.into(); - Self::uniform(name, move || { - Box::new(RateBackendPredictor::from_backend( - RateBackend::Mamba { - model: model.clone(), - }, - -1, - DEFAULT_MIN_PROB, - )) - }) - } - - /// ZPAQ expert (uniform prior). - pub fn zpaq(name: impl Into, method: impl Into) -> Self { - let name = name.into(); - let method = method.into(); - Self::uniform(name, move || { - Box::new(RateBackendPredictor::from_backend( - RateBackend::Zpaq { - method: method.clone(), - }, - -1, - DEFAULT_MIN_PROB, - )) - }) - } - - /// Expert name. - pub fn name(&self) -> &str { - &self.name - } - - /// Log prior weight (unnormalized). - pub fn log_prior(&self) -> f64 { - self.log_prior - } - - /// Build a fresh predictor instance for evaluation or analysis. - pub fn build_predictor(&self) -> Box { - (self.builder)() - } - - fn build(&self) -> ExpertState { - ExpertState { - name: self.name.clone(), - log_weight: self.log_prior, - log_prior: self.log_prior, - predictor: (self.builder)(), - cum_log_loss: 0.0, - } - } -} - -#[derive(Clone)] -struct ExpertState { - name: String, - log_weight: f64, - log_prior: f64, - predictor: Box, - cum_log_loss: f64, -} - -impl ExpertState { - #[inline] - fn begin_stream(&mut self, total_symbols: Option) -> Result<(), String> { - self.predictor.begin_stream(total_symbols) - } - - #[inline] - fn finish_stream(&mut self) -> Result<(), String> { - self.predictor.finish_stream() - } - - #[inline] - fn log_prob(&mut self, symbol: u8) -> f64 { - self.predictor.log_prob(symbol) - } - - #[inline] - fn log_prob_update(&mut self, symbol: u8) -> f64 { - self.predictor.log_prob_update(symbol) - } - - #[inline] - fn update(&mut self, symbol: u8) { - self.predictor.update(symbol); - } - - #[inline] - fn reset_frozen(&mut self, total_symbols: Option) -> Result<(), String> { - self.predictor.reset_frozen(total_symbols) - } - - #[inline] - fn update_frozen(&mut self, symbol: u8) { - self.predictor.update_frozen(symbol); - } -} - -/// Exponential-weights Bayes mixture (log-loss Hedge). -#[derive(Clone)] -pub struct BayesMixture { - experts: Vec, - scratch_logps: Vec, - scratch_mix: Vec, - cached_symbol: u8, - cached_log_mix: f64, - cache_valid: bool, - total_log_loss: f64, -} - -impl BayesMixture { - /// Construct a normalized Bayes mixture from expert configs. - pub fn new(configs: &[ExpertConfig]) -> Self { - let mut experts: Vec = configs.iter().map(|c| c.build()).collect(); - let log_priors: Vec = experts.iter().map(|e| e.log_prior).collect(); - let norm = logsumexp(&log_priors); - for e in &mut experts { - e.log_weight -= norm; - } - Self { - experts, - scratch_logps: vec![0.0; configs.len()], - scratch_mix: vec![0.0; configs.len()], - cached_symbol: 0, - cached_log_mix: f64::NEG_INFINITY, - cache_valid: false, - total_log_loss: 0.0, - } - } - - /// Log-probability (natural log) of the mixture for `symbol`, then update. - pub fn step(&mut self, symbol: u8) -> f64 { - if self.experts.is_empty() { - return f64::NEG_INFINITY; - } - let log_mix = if self.cache_valid && self.cached_symbol == symbol { - for (i, expert) in self.experts.iter_mut().enumerate() { - expert.cum_log_loss -= self.scratch_logps[i]; - expert.update(symbol); - } - self.cached_log_mix - } else { - for (i, expert) in self.experts.iter_mut().enumerate() { - self.scratch_logps[i] = expert.log_prob_update(symbol); - self.scratch_mix[i] = expert.log_weight + self.scratch_logps[i]; - expert.cum_log_loss -= self.scratch_logps[i]; - } - logsumexp(&self.scratch_mix) - }; - for (i, expert) in self.experts.iter_mut().enumerate() { - expert.log_weight = expert.log_weight + self.scratch_logps[i] - log_mix; - } - self.cache_valid = false; - self.total_log_loss -= log_mix; - log_mix - } - - fn predict_log_prob(&mut self, symbol: u8) -> f64 { - if self.experts.is_empty() { - return f64::NEG_INFINITY; - } - for (i, expert) in self.experts.iter_mut().enumerate() { - self.scratch_logps[i] = expert.log_prob(symbol); - self.scratch_mix[i] = expert.log_weight + self.scratch_logps[i]; - } - let log_mix = logsumexp(&self.scratch_mix); - self.cached_symbol = symbol; - self.cached_log_mix = log_mix; - self.cache_valid = true; - log_mix - } - - fn fill_log_probs(&mut self, out: &mut [f64; 256]) { - if self.experts.is_empty() { - out.fill(f64::NEG_INFINITY); - return; - } - out.fill(f64::NEG_INFINITY); - let norm = logsumexp_weights(&self.experts); - let mut row = [0.0f64; 256]; - for expert in &mut self.experts { - expert.predictor.fill_log_probs(&mut row); - let lw = expert.log_weight - norm; - for b in 0..256 { - out[b] = logsumexp2(out[b], lw + row[b]); - } - } - } - - /// Posterior weights (normalized) over experts. - pub fn posterior(&self) -> Vec { - let norm = logsumexp_weights(&self.experts); - self.experts - .iter() - .map(|e| (e.log_weight - norm).exp()) - .collect() - } - - /// Index and log-loss (nats) of the current best expert. - pub fn min_expert_log_loss(&self) -> (usize, f64) { - let mut best_idx = 0usize; - let mut best_loss = f64::INFINITY; - for (i, e) in self.experts.iter().enumerate() { - if e.cum_log_loss < best_loss { - best_loss = e.cum_log_loss; - best_idx = i; - } - } - (best_idx, best_loss) - } - - /// Index and posterior mass of the most likely expert. - pub fn max_posterior(&self) -> (usize, f64) { - let norm = logsumexp_weights(&self.experts); - let mut best_idx = 0usize; - let mut best_p = 0.0; - for (i, e) in self.experts.iter().enumerate() { - let p = (e.log_weight - norm).exp(); - if p > best_p { - best_p = p; - best_idx = i; - } - } - (best_idx, best_p) - } - - /// Total log-loss of the mixture so far (nats). - pub fn total_log_loss(&self) -> f64 { - self.total_log_loss - } - - /// Expert cumulative log-losses (nats) and names. - pub fn expert_log_losses(&self) -> Vec<(String, f64)> { - self.experts - .iter() - .map(|e| (e.name.clone(), e.cum_log_loss)) - .collect() - } - - /// Expert names in order. - pub fn expert_names(&self) -> Vec { - self.experts.iter().map(|e| e.name.clone()).collect() - } - - fn reset_frozen(&mut self, total_symbols: Option) -> Result<(), String> { - for expert in &mut self.experts { - expert.reset_frozen(total_symbols)?; - } - self.cache_valid = false; - self.total_log_loss = 0.0; - Ok(()) - } - - fn update_frozen(&mut self, symbol: u8) { - for expert in &mut self.experts { - expert.update_frozen(symbol); - } - self.cache_valid = false; - } -} - -/// Exponential-weights Bayes mixture with exponential forgetting on weights. -/// -/// This is a non-stationary control: weights are discounted each step by `decay`. -#[derive(Clone)] -pub struct FadingBayesMixture { - experts: Vec, - decay: f64, - scratch_logps: Vec, - scratch_mix: Vec, - cached_symbol: u8, - cached_log_predictive: f64, - cached_log_evidence: f64, - cache_valid: bool, - total_log_loss: f64, -} - -impl FadingBayesMixture { - /// Construct a fading Bayes mixture with decay in `[0, 1]`. - pub fn new(configs: &[ExpertConfig], decay: f64) -> Self { - let mut experts: Vec = configs.iter().map(|c| c.build()).collect(); - let log_priors: Vec = experts.iter().map(|e| e.log_prior).collect(); - let norm = logsumexp(&log_priors); - for e in &mut experts { - e.log_weight -= norm; - } - let decay = decay.clamp(0.0, 1.0); - Self { - experts, - decay, - scratch_logps: vec![0.0; configs.len()], - scratch_mix: vec![0.0; configs.len()], - cached_symbol: 0, - cached_log_predictive: f64::NEG_INFINITY, - cached_log_evidence: f64::NEG_INFINITY, - cache_valid: false, - total_log_loss: 0.0, - } - } - - /// Log-probability (natural log) of the fading mixture for `symbol`, then update. - pub fn step(&mut self, symbol: u8) -> f64 { - if self.experts.is_empty() { - return f64::NEG_INFINITY; - } - let (log_predictive, log_evidence) = if self.cache_valid && self.cached_symbol == symbol { - for (i, expert) in self.experts.iter_mut().enumerate() { - expert.cum_log_loss -= self.scratch_logps[i]; - expert.update(symbol); - } - (self.cached_log_predictive, self.cached_log_evidence) - } else { - for (i, expert) in self.experts.iter_mut().enumerate() { - self.scratch_logps[i] = expert.log_prob_update(symbol); - self.scratch_mix[i] = self.decay * expert.log_weight; - } - let log_prior_norm = logsumexp(&self.scratch_mix); - for (i, expert) in self.experts.iter_mut().enumerate() { - self.scratch_mix[i] += self.scratch_logps[i]; - expert.cum_log_loss -= self.scratch_logps[i]; - } - let log_evidence = logsumexp(&self.scratch_mix); - (log_evidence - log_prior_norm, log_evidence) - }; - for (i, expert) in self.experts.iter_mut().enumerate() { - let decayed = self.decay * expert.log_weight; - expert.log_weight = decayed + self.scratch_logps[i] - log_evidence; - } - self.cache_valid = false; - self.total_log_loss -= log_predictive; - log_predictive - } - - fn predict_log_prob(&mut self, symbol: u8) -> f64 { - if self.experts.is_empty() { - return f64::NEG_INFINITY; - } - for (i, expert) in self.experts.iter_mut().enumerate() { - self.scratch_logps[i] = expert.log_prob(symbol); - self.scratch_mix[i] = self.decay * expert.log_weight; - } - let log_prior_norm = logsumexp(&self.scratch_mix); - for i in 0..self.experts.len() { - self.scratch_mix[i] += self.scratch_logps[i]; - } - let log_evidence = logsumexp(&self.scratch_mix); - let log_predictive = log_evidence - log_prior_norm; - self.cached_symbol = symbol; - self.cached_log_predictive = log_predictive; - self.cached_log_evidence = log_evidence; - self.cache_valid = true; - log_predictive - } - - fn fill_log_probs(&mut self, out: &mut [f64; 256]) { - if self.experts.is_empty() { - out.fill(f64::NEG_INFINITY); - return; - } - out.fill(f64::NEG_INFINITY); - let mut decayed = Vec::with_capacity(self.experts.len()); - for expert in &self.experts { - decayed.push(self.decay * expert.log_weight); - } - let norm = logsumexp(&decayed); - let mut row = [0.0f64; 256]; - for (i, expert) in self.experts.iter_mut().enumerate() { - expert.predictor.fill_log_probs(&mut row); - let lw = decayed[i] - norm; - for b in 0..256 { - out[b] = logsumexp2(out[b], lw + row[b]); - } - } - } - - /// Posterior weights (normalized) over experts. - pub fn posterior(&self) -> Vec { - let norm = logsumexp_weights(&self.experts); - self.experts - .iter() - .map(|e| (e.log_weight - norm).exp()) - .collect() - } - - /// Index and log-loss (nats) of the current best expert (non-discounted loss). - pub fn min_expert_log_loss(&self) -> (usize, f64) { - let mut best_idx = 0usize; - let mut best_loss = f64::INFINITY; - for (i, e) in self.experts.iter().enumerate() { - if e.cum_log_loss < best_loss { - best_loss = e.cum_log_loss; - best_idx = i; - } - } - (best_idx, best_loss) - } - - /// Total log-loss of the mixture so far (nats). - pub fn total_log_loss(&self) -> f64 { - self.total_log_loss - } - - /// Expert names in order. - pub fn expert_names(&self) -> Vec { - self.experts.iter().map(|e| e.name.clone()).collect() - } - - fn reset_frozen(&mut self, total_symbols: Option) -> Result<(), String> { - for expert in &mut self.experts { - expert.reset_frozen(total_symbols)?; - } - self.cache_valid = false; - self.total_log_loss = 0.0; - Ok(()) - } - - fn update_frozen(&mut self, symbol: u8) { - for expert in &mut self.experts { - expert.update_frozen(symbol); - } - self.cache_valid = false; - } -} - -/// Switching mixture: allows occasional switches between experts. -#[derive(Clone)] -pub struct SwitchingMixture { - experts: Vec, - prior: Vec, - alpha: f64, - schedule: MixtureScheduleMode, - scratch_logps: Vec, - scratch_joint: Vec, - scratch_weights: Vec, - cached_symbol: u8, - cached_log_mix: f64, - cache_valid: bool, - total_log_loss: f64, - update_count: u64, -} - -impl SwitchingMixture { - /// Construct a switching mixture. - pub fn new(configs: &[ExpertConfig], alpha: f64, schedule: MixtureScheduleMode) -> Self { - let mut experts: Vec = configs.iter().map(|c| c.build()).collect(); - let prior = normalized_prior_weights(configs); - set_log_weights_from_linear(&mut experts, &prior); - Self { - experts, - prior, - alpha, - schedule, - scratch_logps: vec![0.0; configs.len()], - scratch_joint: vec![0.0; configs.len()], - scratch_weights: vec![0.0; configs.len()], - cached_symbol: 0, - cached_log_mix: f64::NEG_INFINITY, - cache_valid: false, - total_log_loss: 0.0, - update_count: 0, - } - } - - /// Log-probability (natural log) of the switching mixture for `symbol`, then update. - pub fn step(&mut self, symbol: u8) -> f64 { - if self.experts.is_empty() { - return f64::NEG_INFINITY; - } - let log_mix = if self.cache_valid && self.cached_symbol == symbol { - for (i, expert) in self.experts.iter_mut().enumerate() { - expert.cum_log_loss -= self.scratch_logps[i]; - expert.update(symbol); - } - self.cached_log_mix - } else { - for (i, expert) in self.experts.iter_mut().enumerate() { - self.scratch_logps[i] = expert.log_prob_update(symbol); - expert.cum_log_loss -= self.scratch_logps[i]; - self.scratch_joint[i] = expert.log_weight + self.scratch_logps[i]; - } - logsumexp(&self.scratch_joint) - }; - - for i in 0..self.experts.len() { - self.scratch_weights[i] = (self.scratch_joint[i] - log_mix).exp(); - } - - let alpha = switching_alpha_for_update(self.schedule, self.alpha, self.update_count); - self.update_count = self.update_count.saturating_add(1); - - if self.experts.len() == 1 || alpha <= 0.0 { - set_log_weights_from_linear(&mut self.experts, &self.scratch_weights); - } else { - let mut switch_out_sum = 0.0; - let mut num_switch_targets = 0usize; - for &prior in &self.prior { - if prior < 1.0 { - num_switch_targets += 1; - } - } - - if num_switch_targets <= 1 { - set_log_weights_from_linear(&mut self.experts, &self.scratch_weights); - } else { - for i in 0..self.experts.len() { - let denom = 1.0 - self.prior[i]; - if denom > 0.0 { - switch_out_sum += self.scratch_weights[i] / denom; - } - } - - for i in 0..self.experts.len() { - let stay = (1.0 - alpha) * self.scratch_weights[i]; - let switch_in = if self.prior[i] > 0.0 { - let denom = 1.0 - self.prior[i]; - let switchable_mass = if denom > 0.0 { - switch_out_sum - self.scratch_weights[i] / denom - } else { - 0.0 - }; - alpha * self.prior[i] * switchable_mass - } else { - 0.0 - }; - self.scratch_joint[i] = stay + switch_in; - } - normalize_simplex_weights(&mut self.scratch_joint); - set_log_weights_from_linear(&mut self.experts, &self.scratch_joint); - } - } - self.cache_valid = false; - self.total_log_loss -= log_mix; - log_mix - } - - fn predict_log_prob(&mut self, symbol: u8) -> f64 { - if self.experts.is_empty() { - return f64::NEG_INFINITY; - } - for i in 0..self.experts.len() { - let lp = self.experts[i].log_prob(symbol); - self.scratch_logps[i] = lp; - self.scratch_joint[i] = self.experts[i].log_weight + lp; - } - let log_mix = logsumexp(&self.scratch_joint); - self.cached_symbol = symbol; - self.cached_log_mix = log_mix; - self.cache_valid = true; - log_mix - } - - fn fill_log_probs(&mut self, out: &mut [f64; 256]) { - if self.experts.is_empty() { - out.fill(f64::NEG_INFINITY); - return; - } - out.fill(f64::NEG_INFINITY); - let norm = logsumexp_weights(&self.experts); - let mut row = [0.0f64; 256]; - for expert in &mut self.experts { - expert.predictor.fill_log_probs(&mut row); - let lw = expert.log_weight - norm; - for b in 0..256 { - out[b] = logsumexp2(out[b], lw + row[b]); - } - } - } - - /// Posterior weights (normalized) over experts. - pub fn posterior(&self) -> Vec { - let norm = logsumexp_weights(&self.experts); - self.experts - .iter() - .map(|e| (e.log_weight - norm).exp()) - .collect() - } - - /// Index and log-loss (nats) of the current best expert. - pub fn min_expert_log_loss(&self) -> (usize, f64) { - let mut best_idx = 0usize; - let mut best_loss = f64::INFINITY; - for (i, e) in self.experts.iter().enumerate() { - if e.cum_log_loss < best_loss { - best_loss = e.cum_log_loss; - best_idx = i; - } - } - (best_idx, best_loss) - } - - /// Index and posterior mass of the most likely expert. - pub fn max_posterior(&self) -> (usize, f64) { - let norm = logsumexp_weights(&self.experts); - let mut best_idx = 0usize; - let mut best_p = 0.0; - for (i, e) in self.experts.iter().enumerate() { - let p = (e.log_weight - norm).exp(); - if p > best_p { - best_p = p; - best_idx = i; - } - } - (best_idx, best_p) - } - - /// Total log-loss of the mixture so far (nats). - pub fn total_log_loss(&self) -> f64 { - self.total_log_loss - } - - /// Expert cumulative log-losses (nats) and names. - pub fn expert_log_losses(&self) -> Vec<(String, f64)> { - self.experts - .iter() - .map(|e| (e.name.clone(), e.cum_log_loss)) - .collect() - } - - /// Expert names in order. - pub fn expert_names(&self) -> Vec { - self.experts.iter().map(|e| e.name.clone()).collect() - } - - fn reset_frozen(&mut self, total_symbols: Option) -> Result<(), String> { - for expert in &mut self.experts { - expert.reset_frozen(total_symbols)?; - } - self.cache_valid = false; - self.total_log_loss = 0.0; - self.update_count = 0; - Ok(()) - } - - fn update_frozen(&mut self, symbol: u8) { - for expert in &mut self.experts { - expert.update_frozen(symbol); - } - self.cache_valid = false; - } -} - -/// Convex mixture with projected-simplex online updates. -#[derive(Clone)] -pub struct ConvexMixture { - experts: Vec, - alpha: f64, - schedule: MixtureScheduleMode, - lambda: Vec, - scratch_logps: Vec, - projection_scratch: Vec, - cached_symbol: u8, - cached_log_mix: f64, - cache_valid: bool, - total_log_loss: f64, - update_count: u64, -} - -impl ConvexMixture { - /// Construct a convex mixture with prior-derived initial weights. - pub fn new(configs: &[ExpertConfig], alpha: f64, schedule: MixtureScheduleMode) -> Self { - Self { - experts: configs.iter().map(|c| c.build()).collect(), - alpha, - schedule, - lambda: normalized_prior_weights(configs), - scratch_logps: vec![0.0; configs.len()], - projection_scratch: Vec::with_capacity(configs.len()), - cached_symbol: 0, - cached_log_mix: f64::NEG_INFINITY, - cache_valid: false, - total_log_loss: 0.0, - update_count: 0, - } - } - - fn mix_log_prob(&self, logps: &[f64]) -> f64 { - let mut mix = 0.0; - for (weight, &logp) in self.lambda.iter().zip(logps.iter()) { - if *weight > 0.0 { - mix += *weight * logp.exp(); - } - } - clamp_prob(mix, DEFAULT_MIN_PROB).ln() - } - - /// Log-probability (natural log) of the convex mixture for `symbol`, then update. - pub fn step(&mut self, symbol: u8) -> f64 { - if self.experts.is_empty() { - return f64::NEG_INFINITY; - } - - let log_mix = if self.cache_valid && self.cached_symbol == symbol { - for (i, expert) in self.experts.iter_mut().enumerate() { - expert.cum_log_loss -= self.scratch_logps[i]; - expert.update(symbol); - } - self.cached_log_mix - } else { - for (i, expert) in self.experts.iter_mut().enumerate() { - self.scratch_logps[i] = expert.log_prob_update(symbol); - expert.cum_log_loss -= self.scratch_logps[i]; - } - self.mix_log_prob(&self.scratch_logps) - }; - - self.update_count = self.update_count.saturating_add(1); - let step_size = convex_step_size_for_update(self.schedule, self.alpha, self.update_count); - for (weight, &logp) in self.lambda.iter_mut().zip(self.scratch_logps.iter()) { - let grad = -(logp - log_mix).exp(); - *weight -= step_size * grad; - } - project_simplex_with_scratch(&mut self.lambda, &mut self.projection_scratch); - self.cache_valid = false; - self.total_log_loss -= log_mix; - log_mix - } - - fn predict_log_prob(&mut self, symbol: u8) -> f64 { - if self.experts.is_empty() { - return f64::NEG_INFINITY; - } - for (i, expert) in self.experts.iter_mut().enumerate() { - self.scratch_logps[i] = expert.log_prob(symbol); - } - let log_mix = self.mix_log_prob(&self.scratch_logps); - self.cached_symbol = symbol; - self.cached_log_mix = log_mix; - self.cache_valid = true; - log_mix - } - - fn fill_log_probs(&mut self, out: &mut [f64; 256]) { - if self.experts.is_empty() { - out.fill(f64::NEG_INFINITY); - return; - } - out.fill(f64::NEG_INFINITY); - let mut row = [0.0f64; 256]; - for (index, expert) in self.experts.iter_mut().enumerate() { - expert.predictor.fill_log_probs(&mut row); - let weight = self.lambda.get(index).copied().unwrap_or(0.0); - if weight <= 0.0 { - continue; - } - let log_weight = weight.ln(); - for byte in 0..256 { - out[byte] = logsumexp2(out[byte], log_weight + row[byte]); - } - } - } - - fn reset_frozen(&mut self, total_symbols: Option) -> Result<(), String> { - for expert in &mut self.experts { - expert.reset_frozen(total_symbols)?; - } - self.cache_valid = false; - self.total_log_loss = 0.0; - self.update_count = 0; - Ok(()) - } - - fn update_frozen(&mut self, symbol: u8) { - for expert in &mut self.experts { - expert.update_frozen(symbol); - } - self.cache_valid = false; - } -} - -/// MDL-style selector: predicts with the current best expert (by cumulative loss). -#[derive(Clone)] -pub struct MdlSelector { - experts: Vec, - scratch_logps: Vec, - total_log_loss: f64, - last_best: usize, - cached_symbol: u8, - cached_best_idx: usize, - cached_best_logp: f64, - cache_valid: bool, -} - -/// Bytewise neural mixer inspired by fx2-cmix online adaptation. -/// -/// This model is a context-conditioned two-stage gating network trained online -/// from per-symbol expert likelihoods: -/// 1) context-local first-stage expert gates, -/// 2) context-local second-stage meta-gate over stage-1 outputs, -/// 3) per-symbol SGD updates with optional tiny-error skip. -#[derive(Clone)] -pub struct NeuralMixture { - experts: Vec, - neural: NeuralMixCore, - analyzer: TextContextAnalyzer, - min_prob: f64, - scratch_expert_logps: Vec, - scratch_mix_weights: Vec, - eval_cache_valid: bool, - eval_cache_full_valid: bool, - eval_cache_history: NeuralHistoryState, - eval_cache_symbol: u8, - eval_cache_logp: f64, - eval_cache_mix_logps: [f64; 256], - eval_cache_expert_logps: Vec<[f64; 256]>, - total_log_loss: f64, -} - -impl NeuralMixture { - /// Construct a neural mixture. `learning_rate` is taken from `MixtureSpec.alpha`. - pub fn new(configs: &[ExpertConfig], learning_rate: f64) -> Self { - let mut experts: Vec = configs.iter().map(|c| c.build()).collect(); - let n = experts.len(); - - let mut prior_weights = vec![0.0; n]; - if n > 0 { - let log_priors: Vec = experts.iter().map(|e| e.log_prior).collect(); - let norm = logsumexp(&log_priors); - for (i, e) in experts.iter_mut().enumerate() { - let p = (e.log_prior - norm).exp(); - prior_weights[i] = p; - } - } - - let base_lr = if learning_rate.is_finite() { - learning_rate.abs().clamp(1e-6, 1.0) - } else { - 0.03 - }; - let effective_lr = (base_lr * 25.0).clamp(1e-6, 1.0); - let analyzer = TextContextAnalyzer::new(); - let mut neural = - NeuralMixCore::new(n, &prior_weights, effective_lr * 0.5, effective_lr, 1e-5); - neural.set_context_state(analyzer.state()); - let eval_cache_history = neural.history_state(); - - Self { - experts, - neural, - analyzer, - min_prob: DEFAULT_MIN_PROB, - scratch_expert_logps: vec![0.0; n], - scratch_mix_weights: vec![0.0; n], - eval_cache_valid: false, - eval_cache_full_valid: false, - eval_cache_history, - eval_cache_symbol: 0, - eval_cache_logp: f64::NEG_INFINITY, - eval_cache_mix_logps: [f64::NEG_INFINITY; 256], - eval_cache_expert_logps: vec![[f64::NEG_INFINITY; 256]; n], - total_log_loss: 0.0, - } - } - - #[inline] - fn invalidate_eval_cache(&mut self) { - self.eval_cache_valid = false; - self.eval_cache_full_valid = false; - } - - fn sync_history_state(&mut self) -> NeuralHistoryState { - let history = self.analyzer.state(); - if self.neural.history_state() != history { - self.neural.set_context_state(history); - } - if self.eval_cache_history != history { - self.invalidate_eval_cache(); - self.eval_cache_history = history; - } - history - } - - fn ensure_full_evaluation(&mut self) { - self.sync_history_state(); - if self.eval_cache_full_valid { - return; - } - - self.neural.evaluate_expert_weights(); - self.scratch_mix_weights - .copy_from_slice(self.neural.expert_weights()); - let mut mix_pdf = [0.0f64; 256]; - for i in 0..self.experts.len() { - let row = &mut self.eval_cache_expert_logps[i]; - self.experts[i].predictor.fill_log_probs(row); - let w = self.scratch_mix_weights[i]; - for (dst, &lp) in mix_pdf.iter_mut().zip(row.iter()) { - *dst += w * clamp_prob(lp.exp(), self.min_prob); - } - } - - let sum: f64 = mix_pdf.iter().sum(); - if !sum.is_finite() || sum <= 0.0 { - let uniform = (1.0f64 / 256.0).ln(); - self.eval_cache_mix_logps.fill(uniform); - } else { - let inv = 1.0 / sum; - for (dst, &p_raw) in self.eval_cache_mix_logps.iter_mut().zip(mix_pdf.iter()) { - let p = clamp_unit_prob(p_raw * inv, self.min_prob); - *dst = p.ln(); - } - } - - self.eval_cache_full_valid = true; - } - - fn evaluate_symbol(&mut self, symbol: u8) -> f64 { - let history = self.sync_history_state(); - if self.eval_cache_valid - && self.eval_cache_history == history - && self.eval_cache_symbol == symbol - { - return self.eval_cache_logp; - } - - if self.eval_cache_full_valid && self.eval_cache_history == history { - for (dst, row) in self - .scratch_expert_logps - .iter_mut() - .zip(self.eval_cache_expert_logps.iter()) - { - *dst = row[symbol as usize]; - } - let logp = self.eval_cache_mix_logps[symbol as usize]; - self.eval_cache_valid = true; - self.eval_cache_symbol = symbol; - self.eval_cache_logp = logp; - return logp; - } - - let expert_count = self.experts.len(); - for i in 0..expert_count { - self.scratch_expert_logps[i] = self.experts[i].log_prob(symbol); - } - let p = self - .neural - .evaluate_symbol(&self.scratch_expert_logps, self.min_prob); - let logp = clamp_unit_prob(p, self.min_prob).ln(); - self.eval_cache_valid = true; - self.eval_cache_history = history; - self.eval_cache_symbol = symbol; - self.eval_cache_logp = logp; - logp - } - - fn predict_log_prob(&mut self, symbol: u8) -> f64 { - if self.experts.is_empty() { - return f64::NEG_INFINITY; - } - if self.experts.len() == 1 { - return self.experts[0].log_prob(symbol); - } - self.evaluate_symbol(symbol) - } - - fn fill_log_probs(&mut self, out: &mut [f64; 256]) { - if self.experts.is_empty() { - out.fill(f64::NEG_INFINITY); - return; - } - if self.experts.len() == 1 { - self.experts[0].predictor.fill_log_probs(out); - return; - } - self.ensure_full_evaluation(); - out.copy_from_slice(&self.eval_cache_mix_logps); - } - - /// Log-probability (natural log) of the neural mixture for `symbol`, then update. - pub fn step(&mut self, symbol: u8) -> f64 { - if self.experts.is_empty() { - return f64::NEG_INFINITY; - } - - if self.experts.len() == 1 { - let expert = &mut self.experts[0]; - let logp = expert.log_prob_update(symbol); - expert.cum_log_loss -= logp; - self.total_log_loss -= logp; - self.analyzer.update(symbol); - self.neural.set_context_state(self.analyzer.state()); - self.invalidate_eval_cache(); - return logp; - } - - let history = self.sync_history_state(); - let logp = if self.eval_cache_valid - && self.eval_cache_history == history - && self.eval_cache_symbol == symbol - { - let logp = self.eval_cache_logp; - for i in 0..self.experts.len() { - let expert = &mut self.experts[i]; - expert.cum_log_loss -= self.scratch_expert_logps[i]; - expert.update(symbol); - } - logp - } else if self.eval_cache_full_valid && self.eval_cache_history == history { - for i in 0..self.experts.len() { - self.scratch_expert_logps[i] = self.eval_cache_expert_logps[i][symbol as usize]; - } - let logp = self.eval_cache_mix_logps[symbol as usize]; - for i in 0..self.experts.len() { - let expert = &mut self.experts[i]; - expert.cum_log_loss -= self.scratch_expert_logps[i]; - expert.update(symbol); - } - logp - } else { - for i in 0..self.experts.len() { - let expert = &mut self.experts[i]; - self.scratch_expert_logps[i] = expert.log_prob_update(symbol); - expert.cum_log_loss -= self.scratch_expert_logps[i]; - } - let p = self - .neural - .evaluate_symbol(&self.scratch_expert_logps, self.min_prob); - clamp_unit_prob(p, self.min_prob).ln() - }; - self.neural - .update_weights_symbol(&self.scratch_expert_logps, self.min_prob); - self.total_log_loss -= logp; - self.analyzer.update(symbol); - self.neural.set_context_state(self.analyzer.state()); - self.invalidate_eval_cache(); - logp - } - - /// Total log-loss of the mixture so far (nats). - pub fn total_log_loss(&self) -> f64 { - self.total_log_loss - } - - fn reset_frozen(&mut self, total_symbols: Option) -> Result<(), String> { - for expert in &mut self.experts { - expert.reset_frozen(total_symbols)?; - } - self.analyzer = TextContextAnalyzer::new(); - self.neural.set_context_state(self.analyzer.state()); - self.invalidate_eval_cache(); - self.eval_cache_history = self.neural.history_state(); - self.total_log_loss = 0.0; - Ok(()) - } - - fn update_frozen(&mut self, symbol: u8) { - for expert in &mut self.experts { - expert.update_frozen(symbol); - } - self.analyzer.update(symbol); - self.neural.set_context_state(self.analyzer.state()); - self.invalidate_eval_cache(); - self.eval_cache_history = self.neural.history_state(); - } -} - -impl MdlSelector { - /// Construct an MDL-style expert selector. - pub fn new(configs: &[ExpertConfig]) -> Self { - let experts: Vec = configs.iter().map(|c| c.build()).collect(); - let last_best = 0usize; - Self { - experts, - scratch_logps: vec![0.0; configs.len()], - total_log_loss: 0.0, - last_best, - cached_symbol: 0, - cached_best_idx: 0, - cached_best_logp: f64::NEG_INFINITY, - cache_valid: false, - } - } - - /// Log-probability (natural log) of the MDL selector for `symbol`, then update. - pub fn step(&mut self, symbol: u8) -> f64 { - if self.experts.is_empty() { - return f64::NEG_INFINITY; - } - let used_cache = self.cache_valid && self.cached_symbol == symbol; - let best_idx = if used_cache { - self.scratch_logps[self.cached_best_idx] = self.cached_best_logp; - for (i, expert) in self.experts.iter_mut().enumerate() { - if i == self.cached_best_idx { - continue; - } - self.scratch_logps[i] = expert.log_prob(symbol); - } - self.cached_best_idx - } else { - for (i, expert) in self.experts.iter_mut().enumerate() { - self.scratch_logps[i] = expert.log_prob_update(symbol); - } - let mut best_idx = 0usize; - let mut best_loss = f64::INFINITY; - for (i, expert) in self.experts.iter().enumerate() { - if expert.cum_log_loss < best_loss { - best_loss = expert.cum_log_loss; - best_idx = i; - } - } - best_idx - }; - let logp = self.scratch_logps[best_idx]; - self.cache_valid = false; - for (i, expert) in self.experts.iter_mut().enumerate() { - expert.cum_log_loss -= self.scratch_logps[i]; - if used_cache { - expert.update(symbol); - } - } - self.total_log_loss -= logp; - self.last_best = best_idx; - logp - } - - fn predict_log_prob(&mut self, symbol: u8) -> f64 { - if self.experts.is_empty() { - return f64::NEG_INFINITY; - } - let mut best_idx = 0usize; - let mut best_loss = f64::INFINITY; - for (i, expert) in self.experts.iter().enumerate() { - if expert.cum_log_loss < best_loss { - best_loss = expert.cum_log_loss; - best_idx = i; - } - } - let logp = self.experts[best_idx].log_prob(symbol); - self.cached_symbol = symbol; - self.cached_best_idx = best_idx; - self.cached_best_logp = logp; - self.cache_valid = true; - logp - } - - fn fill_log_probs(&mut self, out: &mut [f64; 256]) { - if self.experts.is_empty() { - out.fill(f64::NEG_INFINITY); - return; - } - let mut best_idx = 0usize; - let mut best_loss = f64::INFINITY; - for (i, expert) in self.experts.iter().enumerate() { - if expert.cum_log_loss < best_loss { - best_loss = expert.cum_log_loss; - best_idx = i; - } - } - self.experts[best_idx].predictor.fill_log_probs(out); - } - - /// Index of the current best expert. - pub fn best_index(&self) -> usize { - self.last_best - } - - /// Index and log-loss (nats) of the current best expert. - pub fn min_expert_log_loss(&self) -> (usize, f64) { - let mut best_idx = 0usize; - let mut best_loss = f64::INFINITY; - for (i, e) in self.experts.iter().enumerate() { - if e.cum_log_loss < best_loss { - best_loss = e.cum_log_loss; - best_idx = i; - } - } - (best_idx, best_loss) - } - - /// Total log-loss of the selector so far (nats). - pub fn total_log_loss(&self) -> f64 { - self.total_log_loss - } - - /// Expert cumulative log-losses (nats) and names. - pub fn expert_log_losses(&self) -> Vec<(String, f64)> { - self.experts - .iter() - .map(|e| (e.name.clone(), e.cum_log_loss)) - .collect() - } - - /// Expert names in order. - pub fn expert_names(&self) -> Vec { - self.experts.iter().map(|e| e.name.clone()).collect() - } - - fn reset_frozen(&mut self, total_symbols: Option) -> Result<(), String> { - for expert in &mut self.experts { - expert.reset_frozen(total_symbols)?; - } - self.cache_valid = false; - self.total_log_loss = 0.0; - Ok(()) - } - - fn update_frozen(&mut self, symbol: u8) { - for expert in &mut self.experts { - expert.update_frozen(symbol); - } - self.cache_valid = false; - } -} - -// ============================================================================= -// Mixture Runtime Helper (for RateBackend::Mixture) -// ============================================================================= - -/// Runtime wrapper over concrete mixture strategies. -#[allow(clippy::large_enum_variant)] -#[derive(Clone)] -pub enum MixtureRuntime { - /// Bayes mixture. - Bayes(BayesMixture), - /// Fading Bayes mixture. - Fading(FadingBayesMixture), - /// Switching mixture. - Switching(SwitchingMixture), - /// Convex mixture. - Convex(ConvexMixture), - /// MDL selector. - Mdl(MdlSelector), - /// Bytewise neural logistic mixer. - Neural(NeuralMixture), -} - -impl MixtureRuntime { - pub(crate) fn begin_stream(&mut self, total_symbols: Option) -> Result<(), String> { - match self { - MixtureRuntime::Bayes(m) => begin_expert_stream(&mut m.experts, total_symbols), - MixtureRuntime::Fading(m) => begin_expert_stream(&mut m.experts, total_symbols), - MixtureRuntime::Switching(m) => begin_expert_stream(&mut m.experts, total_symbols), - MixtureRuntime::Convex(m) => begin_expert_stream(&mut m.experts, total_symbols), - MixtureRuntime::Mdl(m) => begin_expert_stream(&mut m.experts, total_symbols), - MixtureRuntime::Neural(m) => begin_expert_stream(&mut m.experts, total_symbols), - } - } - - pub(crate) fn finish_stream(&mut self) -> Result<(), String> { - match self { - MixtureRuntime::Bayes(m) => finish_expert_stream(&mut m.experts), - MixtureRuntime::Fading(m) => finish_expert_stream(&mut m.experts), - MixtureRuntime::Switching(m) => finish_expert_stream(&mut m.experts), - MixtureRuntime::Convex(m) => finish_expert_stream(&mut m.experts), - MixtureRuntime::Mdl(m) => finish_expert_stream(&mut m.experts), - MixtureRuntime::Neural(m) => finish_expert_stream(&mut m.experts), - } - } - - pub(crate) fn reset_frozen(&mut self, total_symbols: Option) -> Result<(), String> { - match self { - MixtureRuntime::Bayes(m) => m.reset_frozen(total_symbols), - MixtureRuntime::Fading(m) => m.reset_frozen(total_symbols), - MixtureRuntime::Switching(m) => m.reset_frozen(total_symbols), - MixtureRuntime::Convex(m) => m.reset_frozen(total_symbols), - MixtureRuntime::Mdl(m) => m.reset_frozen(total_symbols), - MixtureRuntime::Neural(m) => m.reset_frozen(total_symbols), - } - } - - /// Non-mutating log-probability (nats) for `symbol` at current state. - pub(crate) fn peek_log_prob(&mut self, symbol: u8) -> f64 { - match self { - MixtureRuntime::Bayes(m) => m.predict_log_prob(symbol), - MixtureRuntime::Fading(m) => m.predict_log_prob(symbol), - MixtureRuntime::Switching(m) => m.predict_log_prob(symbol), - MixtureRuntime::Convex(m) => m.predict_log_prob(symbol), - MixtureRuntime::Mdl(m) => m.predict_log_prob(symbol), - MixtureRuntime::Neural(m) => m.predict_log_prob(symbol), - } - } - - /// Step the mixture and return log-probability (nats). - pub(crate) fn step(&mut self, symbol: u8) -> f64 { - match self { - MixtureRuntime::Bayes(m) => m.step(symbol), - MixtureRuntime::Fading(m) => m.step(symbol), - MixtureRuntime::Switching(m) => m.step(symbol), - MixtureRuntime::Convex(m) => m.step(symbol), - MixtureRuntime::Mdl(m) => m.step(symbol), - MixtureRuntime::Neural(m) => m.step(symbol), - } - } - - pub(crate) fn update_frozen(&mut self, symbol: u8) { - match self { - MixtureRuntime::Bayes(m) => m.update_frozen(symbol), - MixtureRuntime::Fading(m) => m.update_frozen(symbol), - MixtureRuntime::Switching(m) => m.update_frozen(symbol), - MixtureRuntime::Convex(m) => m.update_frozen(symbol), - MixtureRuntime::Mdl(m) => m.update_frozen(symbol), - MixtureRuntime::Neural(m) => m.update_frozen(symbol), - } - } - - pub(crate) fn fill_log_probs(&mut self, out: &mut [f64; 256]) { - match self { - MixtureRuntime::Bayes(m) => m.fill_log_probs(out), - MixtureRuntime::Fading(m) => m.fill_log_probs(out), - MixtureRuntime::Switching(m) => m.fill_log_probs(out), - MixtureRuntime::Convex(m) => m.fill_log_probs(out), - MixtureRuntime::Mdl(m) => m.fill_log_probs(out), - MixtureRuntime::Neural(m) => m.fill_log_probs(out), - } - } -} - -fn begin_expert_stream( - experts: &mut [ExpertState], - total_symbols: Option, -) -> Result<(), String> { - for expert in experts { - expert.begin_stream(total_symbols)?; - } - Ok(()) -} - -fn finish_expert_stream(experts: &mut [ExpertState]) -> Result<(), String> { - for expert in experts { - expert.finish_stream()?; - } - Ok(()) -} - -pub(crate) fn build_mixture_runtime( - spec: &MixtureSpec, - experts: &[ExpertConfig], -) -> Result { - spec.validate()?; - match spec.kind { - MixtureKind::Bayes => Ok(MixtureRuntime::Bayes(BayesMixture::new(experts))), - MixtureKind::FadingBayes => { - let decay = spec - .decay - .ok_or_else(|| "fading Bayes mixture requires decay".to_string())?; - Ok(MixtureRuntime::Fading(FadingBayesMixture::new( - experts, decay, - ))) - } - MixtureKind::Switching => Ok(MixtureRuntime::Switching(SwitchingMixture::new( - experts, - spec.alpha, - spec.schedule, - ))), - MixtureKind::Convex => Ok(MixtureRuntime::Convex(ConvexMixture::new( - experts, - spec.alpha, - spec.schedule, - ))), - MixtureKind::Mdl => Ok(MixtureRuntime::Mdl(MdlSelector::new(experts))), - MixtureKind::Neural => Ok(MixtureRuntime::Neural(NeuralMixture::new( - experts, spec.alpha, - ))), - } -} - -#[cfg(test)] -mod tests { - use super::*; - use std::sync::{ - Arc, - atomic::{AtomicU64, AtomicUsize, Ordering}, - }; - - #[derive(Clone)] - struct AlwaysPredict { - byte: u8, - } - - impl OnlineBytePredictor for AlwaysPredict { - fn log_prob(&mut self, symbol: u8) -> f64 { - if symbol == self.byte { - 0.0 - } else { - f64::NEG_INFINITY - } - } - - fn update(&mut self, _symbol: u8) {} - } - - #[derive(Clone)] - struct FixedProbPredict { - prob_zero: f64, - } - - impl OnlineBytePredictor for FixedProbPredict { - fn log_prob(&mut self, symbol: u8) -> f64 { - let p = if symbol == 0 { - self.prob_zero - } else { - 1.0 - self.prob_zero - }; - p.ln() - } - - fn update(&mut self, _symbol: u8) {} - } - - fn weighted_cfg(name: &'static str, weight: f64, prob_zero: f64) -> ExpertConfig { - ExpertConfig::new(name, weight.ln(), move || { - Box::new(FixedProbPredict { prob_zero }) - }) - } - - #[test] - fn bayes_mixture_prefers_correct_expert() { - let configs = vec![ - ExpertConfig::uniform("zero", || Box::new(AlwaysPredict { byte: 0 })), - ExpertConfig::uniform("one", || Box::new(AlwaysPredict { byte: 1 })), - ]; - let mut mix = BayesMixture::new(&configs); - for _ in 0..10 { - mix.step(0); - } - let post = mix.posterior(); - assert!(post[0] > 0.999); - assert!(post[1] < 1e-6); - } - - fn counting_cfg(name: &'static str, calls: Arc) -> ExpertConfig { - ExpertConfig::uniform(name, move || { - Box::new(CountingPredict { - calls: calls.clone(), - }) - }) - } - - #[test] - fn bayes_predict_then_step_reuses_cached_log_probs() { - let c0 = Arc::new(AtomicUsize::new(0)); - let c1 = Arc::new(AtomicUsize::new(0)); - let mut mix = BayesMixture::new(&[ - counting_cfg("c0", c0.clone()), - counting_cfg("c1", c1.clone()), - ]); - let _ = mix.predict_log_prob(0); - let after_predict = c0.load(Ordering::Relaxed) + c1.load(Ordering::Relaxed); - assert_eq!(after_predict, 2); - let _ = mix.step(0); - let after_step = c0.load(Ordering::Relaxed) + c1.load(Ordering::Relaxed); - assert_eq!(after_step, after_predict); - } - - #[test] - fn fading_predict_then_step_reuses_cached_log_probs() { - let c0 = Arc::new(AtomicUsize::new(0)); - let c1 = Arc::new(AtomicUsize::new(0)); - let mut mix = FadingBayesMixture::new( - &[ - counting_cfg("c0", c0.clone()), - counting_cfg("c1", c1.clone()), - ], - 0.95, - ); - let _ = mix.predict_log_prob(0); - let after_predict = c0.load(Ordering::Relaxed) + c1.load(Ordering::Relaxed); - assert_eq!(after_predict, 2); - let _ = mix.step(0); - let after_step = c0.load(Ordering::Relaxed) + c1.load(Ordering::Relaxed); - assert_eq!(after_step, after_predict); - } - - #[test] - fn switching_predict_then_step_reuses_cached_log_probs() { - let c0 = Arc::new(AtomicUsize::new(0)); - let c1 = Arc::new(AtomicUsize::new(0)); - let mut mix = SwitchingMixture::new( - &[ - counting_cfg("c0", c0.clone()), - counting_cfg("c1", c1.clone()), - ], - 0.05, - MixtureScheduleMode::Default, - ); - let _ = mix.predict_log_prob(0); - let after_predict = c0.load(Ordering::Relaxed) + c1.load(Ordering::Relaxed); - assert_eq!(after_predict, 2); - let _ = mix.step(0); - let after_step = c0.load(Ordering::Relaxed) + c1.load(Ordering::Relaxed); - assert_eq!(after_step, after_predict); - } - - #[test] - fn switching_mixture_matches_fixed_share_update_for_uniform_prior() { - let configs = vec![weighted_cfg("a", 0.5, 0.8), weighted_cfg("b", 0.5, 0.3)]; - let alpha = 0.2; - let mut mix = SwitchingMixture::new(&configs, alpha, MixtureScheduleMode::Default); - - let predicted = mix.predict_log_prob(0).exp(); - assert!((predicted - 0.55).abs() < 1e-12, "predicted={predicted}"); - - let observed = mix.step(0).exp(); - assert!((observed - 0.55).abs() < 1e-12, "observed={observed}"); - - let post = mix.posterior(); - let posterior_a = 0.5 * 0.8 / 0.55; - let posterior_b = 0.5 * 0.3 / 0.55; - let expected_a = (1.0 - alpha) * posterior_a + alpha * posterior_b; - let expected_b = (1.0 - alpha) * posterior_b + alpha * posterior_a; - assert!( - (post[0] - expected_a).abs() < 1e-12 && (post[1] - expected_b).abs() < 1e-12, - "expected [{expected_a}, {expected_b}], got {:?}", - post - ); - } - - #[test] - fn switching_mixture_switches_according_to_prior_over_other_experts() { - let configs = vec![ - weighted_cfg("a", 0.5, 0.75), - weighted_cfg("b", 0.3, 0.25), - weighted_cfg("c", 0.2, 0.60), - ]; - let alpha = 0.15; - let mut mix = SwitchingMixture::new(&configs, alpha, MixtureScheduleMode::Default); - - let _ = mix.step(0); - let post = mix.posterior(); - - let current = [0.5_f64, 0.3, 0.2]; - let likelihood = [0.75_f64, 0.25, 0.60]; - let mix_prob = current - .iter() - .zip(likelihood.iter()) - .map(|(w, p)| w * p) - .sum::(); - let posterior = [ - current[0] * likelihood[0] / mix_prob, - current[1] * likelihood[1] / mix_prob, - current[2] * likelihood[2] / mix_prob, - ]; - let prior = [0.5_f64, 0.3, 0.2]; - let mut expected = [0.0_f64; 3]; - for j in 0..3 { - let stay = (1.0 - alpha) * posterior[j]; - let switch_in = alpha - * prior[j] - * (0..3) - .filter(|&k| k != j) - .map(|k| posterior[k] / (1.0 - prior[k])) - .sum::(); - expected[j] = stay + switch_in; - } - - for i in 0..3 { - assert!( - (post[i] - expected[i]).abs() < 1e-12, - "expert {i}: expected {} got {}", - expected[i], - post[i] - ); - } - } - - #[test] - fn switching_theorem_schedule_uses_one_over_t() { - assert!( - (switching_alpha_for_update(MixtureScheduleMode::Theorem, 0.99, 0) - 0.5).abs() < 1e-12 - ); - assert!( - (switching_alpha_for_update(MixtureScheduleMode::Theorem, 0.99, 1) - (1.0 / 3.0)).abs() - < 1e-12 - ); - - let configs = vec![weighted_cfg("a", 0.5, 0.8), weighted_cfg("b", 0.5, 0.3)]; - let mut mix = SwitchingMixture::new(&configs, 0.99, MixtureScheduleMode::Theorem); - let _ = mix.step(0); - let post = mix.posterior(); - let posterior_a = 0.5 * 0.8 / 0.55; - let posterior_b = 0.5 * 0.3 / 0.55; - let expected_a = 0.5 * posterior_a + 0.5 * posterior_b; - let expected_b = expected_a; - assert!((post[0] - expected_a).abs() < 1e-12); - assert!((post[1] - expected_b).abs() < 1e-12); - } - - #[test] - fn convex_theorem_schedule_uses_paper_step_size() { - let eta = convex_step_size_for_update(MixtureScheduleMode::Theorem, 9.0, 1); - assert!((eta - DEFAULT_MIN_PROB).abs() < 1e-18); - - let configs = vec![weighted_cfg("a", 0.5, 0.8), weighted_cfg("b", 0.5, 0.3)]; - let mut mix = ConvexMixture::new(&configs, 9.0, MixtureScheduleMode::Theorem); - let observed = mix.step(0).exp(); - assert!((observed - 0.55).abs() < 1e-12, "observed={observed}"); - - let expected = [ - 0.5 + eta * ((0.8 / 0.55) - 1.0), - 0.5 + eta * ((0.3 / 0.55) - 1.0), - ]; - assert!((mix.lambda[0] - expected[0]).abs() < 1e-12); - assert!((mix.lambda[1] - expected[1]).abs() < 1e-12); - } - - #[test] - fn mdl_predict_then_step_reuses_best_expert_log_prob() { - let c0 = Arc::new(AtomicUsize::new(0)); - let c1 = Arc::new(AtomicUsize::new(0)); - let mut mdl = MdlSelector::new(&[ - counting_cfg("c0", c0.clone()), - counting_cfg("c1", c1.clone()), - ]); - let _ = mdl.predict_log_prob(0); - let after_predict = c0.load(Ordering::Relaxed) + c1.load(Ordering::Relaxed); - assert_eq!(after_predict, 1); - let _ = mdl.step(0); - let after_step = c0.load(Ordering::Relaxed) + c1.load(Ordering::Relaxed); - assert_eq!(after_step, 2); - } - - #[test] - fn neural_mixture_adapts_to_correct_symbol() { - let configs = vec![ - ExpertConfig::uniform("zero", || Box::new(AlwaysPredict { byte: 0 })), - ExpertConfig::uniform("one", || Box::new(AlwaysPredict { byte: 1 })), - ]; - let mut mix = NeuralMixture::new(&configs, 0.05); - - let mut early = 0.0; - let mut late = 0.0; - for t in 0..200 { - let lp = mix.step(0); - if t < 20 { - early -= lp; - } - if t >= 180 { - late -= lp; - } - } - - let early_avg = early / 20.0; - let late_avg = late / 20.0; - assert!( - late_avg < early_avg, - "late_avg={late_avg} early_avg={early_avg}" - ); - assert!(late_avg < 0.35, "late_avg={late_avg}"); - } - - #[derive(Clone)] - struct CountingPredict { - calls: Arc, - } - - impl OnlineBytePredictor for CountingPredict { - fn log_prob(&mut self, symbol: u8) -> f64 { - self.calls.fetch_add(1, Ordering::Relaxed); - if symbol == 0 { 0.0 } else { -20.0 } - } - - fn update(&mut self, _symbol: u8) {} - } - - #[derive(Clone)] - struct CountingFillPredict { - log_calls: Arc, - fill_calls: Arc, - } - - impl OnlineBytePredictor for CountingFillPredict { - fn log_prob(&mut self, symbol: u8) -> f64 { - self.log_calls.fetch_add(1, Ordering::Relaxed); - if symbol == 0 { 0.0 } else { -20.0 } - } - - fn fill_log_probs(&mut self, out: &mut [f64; 256]) { - self.fill_calls.fetch_add(1, Ordering::Relaxed); - out.fill(-20.0); - out[0] = 0.0; - } - - fn update(&mut self, _symbol: u8) {} - } - - #[derive(Clone)] - struct BeginAwarePredict { - seen_total: Arc, - began: bool, - } - - impl OnlineBytePredictor for BeginAwarePredict { - fn begin_stream(&mut self, total_symbols: Option) -> Result<(), String> { - let total = total_symbols.ok_or_else(|| "missing total symbols".to_string())?; - self.seen_total.store(total, Ordering::Relaxed); - self.began = true; - Ok(()) - } - - fn log_prob(&mut self, _symbol: u8) -> f64 { - if self.began { 0.0 } else { f64::NEG_INFINITY } - } - - fn update(&mut self, _symbol: u8) {} - } - - fn assert_log_prob_update_matches_separate(label: &str, backend: RateBackend) { - let mut separate = - RateBackendPredictor::from_backend(backend.clone(), -1, DEFAULT_MIN_PROB); - let mut combined = RateBackendPredictor::from_backend(backend, -1, DEFAULT_MIN_PROB); - let data = b"combined step check data"; - - for &b in data { - let logp_separate = separate.log_prob(b); - separate.update(b); - let logp_combined = combined.log_prob_update(b); - let diff = (logp_separate - logp_combined).abs(); - assert!( - diff <= 1e-12, - "[{label}] symbol={b} separate={logp_separate} combined={logp_combined} diff={diff}" - ); - - let mut sep_row = [0.0; 256]; - let mut combo_row = [0.0; 256]; - separate.fill_log_probs(&mut sep_row); - combined.fill_log_probs(&mut combo_row); - for i in 0..256 { - let diff = (sep_row[i] - combo_row[i]).abs(); - assert!( - diff <= 1e-12, - "row mismatch at {i}: {} vs {}", - sep_row[i], - combo_row[i] - ); - } - } - } - - fn assert_fill_matches_symbol_queries(label: &str, backend: RateBackend) { - let mut bulk = RateBackendPredictor::from_backend(backend.clone(), -1, DEFAULT_MIN_PROB); - let mut queried = RateBackendPredictor::from_backend(backend, -1, DEFAULT_MIN_PROB); - let data = b"continuation consistency prompt"; - - bulk.begin_stream(Some(data.len() as u64)) - .expect("bulk begin"); - queried - .begin_stream(Some(data.len() as u64)) - .expect("query begin"); - for &b in data { - bulk.update(b); - queried.update(b); - } - - let mut bulk_row = [0.0; 256]; - bulk.fill_log_probs(&mut bulk_row); - for (sym, &bulk_logp) in bulk_row.iter().enumerate() { - let queried_logp = queried.log_prob(sym as u8); - let diff = (bulk_logp - queried_logp).abs(); - assert!( - diff <= 1e-12, - "[{label}] sym={sym} bulk={bulk_logp} queried={queried_logp} diff={diff}" - ); - } - } - - fn assert_fill_matches_symbol_queries_after_frozen_conditioning( - label: &str, - backend: RateBackend, - ) { - let fit = b"If a frog is green, dogs are red.\nIf a toad is green, cats are red.\n"; - let condition = b"If a cat is red, toads are \n"; - let total = (fit.len() + condition.len()) as u64; - - let mut bulk = RateBackendPredictor::from_backend(backend.clone(), -1, DEFAULT_MIN_PROB); - let mut queried = RateBackendPredictor::from_backend(backend, -1, DEFAULT_MIN_PROB); - - bulk.begin_stream(Some(total)).expect("bulk begin"); - queried.begin_stream(Some(total)).expect("query begin"); - for &b in fit { - bulk.update(b); - queried.update(b); - } - bulk.reset_frozen(Some(condition.len() as u64)) - .expect("bulk reset frozen"); - queried - .reset_frozen(Some(condition.len() as u64)) - .expect("query reset frozen"); - for &b in condition { - bulk.update_frozen(b); - queried.update_frozen(b); - } - - let mut bulk_row = [0.0; 256]; - bulk.fill_log_probs(&mut bulk_row); - for (sym, &bulk_logp) in bulk_row.iter().enumerate() { - let queried_logp = queried.log_prob(sym as u8); - let diff = (bulk_logp - queried_logp).abs(); - assert!( - diff <= 1e-12, - "[{label}] frozen sym={sym} bulk={bulk_logp} queried={queried_logp} diff={diff}" - ); - } - } - - #[test] - fn predictor_log_prob_update_matches_separate_update_for_rosa_backend() { - assert_log_prob_update_matches_separate("rosa", RateBackend::RosaPlus); - } - - #[test] - fn predictor_log_prob_update_matches_separate_update_for_ctw_backend() { - assert_log_prob_update_matches_separate("ctw", RateBackend::Ctw { depth: 6 }); - } - - #[test] - fn predictor_log_prob_update_matches_separate_update_for_fac_ctw_backend() { - assert_log_prob_update_matches_separate( - "fac-ctw", - RateBackend::FacCtw { - base_depth: 6, - num_percept_bits: 8, - encoding_bits: 8, - }, - ); - } - - #[test] - fn predictor_fill_matches_symbol_queries_for_rosa_backend() { - assert_fill_matches_symbol_queries("rosa", RateBackend::RosaPlus); - } - - #[test] - fn predictor_fill_matches_symbol_queries_for_ctw_backend() { - assert_fill_matches_symbol_queries("ctw", RateBackend::Ctw { depth: 6 }); - } - - #[test] - fn predictor_fill_matches_symbol_queries_for_match_backend() { - assert_fill_matches_symbol_queries( - "match", - RateBackend::Match { - hash_bits: 18, - min_len: 4, - max_len: 64, - base_mix: 0.02, - confidence_scale: 1.0, - }, - ); - } - - #[test] - fn predictor_fill_matches_symbol_queries_for_ppmd_backend() { - assert_fill_matches_symbol_queries( - "ppmd", - RateBackend::Ppmd { - order: 8, - memory_mb: 8, - }, - ); - } - - #[cfg(feature = "backend-rwkv")] - #[test] - fn predictor_fill_matches_symbol_queries_for_rwkv_backend() { - assert_fill_matches_symbol_queries( - "rwkv7", - RateBackend::Rwkv7Method { - method: "cfg:hidden=64,layers=1,intermediate=64,decay_rank=8,a_rank=8,v_rank=8,g_rank=8,seed=31,train=none,lr=0.0,stride=1;policy:schedule=0..100:infer".to_string(), - }, - ); - } - - #[test] - fn predictor_fill_matches_symbol_queries_for_rosa_backend_after_frozen_conditioning() { - assert_fill_matches_symbol_queries_after_frozen_conditioning("rosa", RateBackend::RosaPlus); - } - - #[test] - fn predictor_frozen_conditioning_reuses_match_fit_corpus() { - let mut predictor = RateBackendPredictor::from_backend( - RateBackend::Match { - hash_bits: 20, - min_len: 3, - max_len: 32, - base_mix: 0.02, - confidence_scale: 1.0, - }, - -1, - DEFAULT_MIN_PROB, - ); - - for &b in b"abcabcX" { - predictor.update(b); - } - predictor - .reset_frozen(Some(6)) - .expect("reset frozen for match backend"); - for &b in b"abcabc" { - predictor.update_frozen(b); - } - let p_x = predictor.log_prob(b'X').exp(); - assert!( - p_x > 0.01, - "frozen conditioning should preserve fit corpus for match backend; p_x={p_x}" - ); - } - - #[test] - fn predictor_frozen_conditioning_reuses_sparse_match_fit_corpus() { - let mut predictor = RateBackendPredictor::from_backend( - RateBackend::SparseMatch { - hash_bits: 20, - min_len: 3, - max_len: 32, - gap_min: 0, - gap_max: 2, - base_mix: 0.02, - confidence_scale: 1.0, - }, - -1, - DEFAULT_MIN_PROB, - ); - - for &b in b"abcabcX" { - predictor.update(b); - } - predictor - .reset_frozen(Some(6)) - .expect("reset frozen for sparse-match backend"); - for &b in b"abcabc" { - predictor.update_frozen(b); - } - let p_x = predictor.log_prob(b'X').exp(); - assert!( - p_x > 0.01, - "frozen conditioning should preserve fit corpus for sparse-match backend; p_x={p_x}" - ); - } - - #[test] - fn neural_predict_then_step_reuses_evaluation_cache() { - let c0 = Arc::new(AtomicUsize::new(0)); - let c1 = Arc::new(AtomicUsize::new(0)); - let cfg0 = { - let c = c0.clone(); - ExpertConfig::uniform("c0", move || Box::new(CountingPredict { calls: c.clone() })) - }; - let cfg1 = { - let c = c1.clone(); - ExpertConfig::uniform("c1", move || Box::new(CountingPredict { calls: c.clone() })) - }; - let mut mix = NeuralMixture::new(&[cfg0, cfg1], 0.03); - - let _ = mix.predict_log_prob(0); - let after_predict = c0.load(Ordering::Relaxed) + c1.load(Ordering::Relaxed); - assert_eq!(after_predict, 2); - - let _ = mix.step(0); - let after_step = c0.load(Ordering::Relaxed) + c1.load(Ordering::Relaxed); - assert_eq!(after_step, after_predict); - } - - #[test] - fn neural_predict_multiple_symbols_reuses_single_evaluation() { - let c0 = Arc::new(AtomicUsize::new(0)); - let c1 = Arc::new(AtomicUsize::new(0)); - let cfg0 = { - let c = c0.clone(); - ExpertConfig::uniform("c0", move || Box::new(CountingPredict { calls: c.clone() })) - }; - let cfg1 = { - let c = c1.clone(); - ExpertConfig::uniform("c1", move || Box::new(CountingPredict { calls: c.clone() })) - }; - let mut mix = NeuralMixture::new(&[cfg0, cfg1], 0.03); - - let _ = mix.predict_log_prob(0); - let after_first = c0.load(Ordering::Relaxed) + c1.load(Ordering::Relaxed); - assert_eq!(after_first, 2); - - let _ = mix.predict_log_prob(1); - let after_second = c0.load(Ordering::Relaxed) + c1.load(Ordering::Relaxed); - assert_eq!(after_second, after_first + 2); - } - - #[test] - fn neural_fill_then_step_reuses_cached_full_rows() { - let log0 = Arc::new(AtomicUsize::new(0)); - let log1 = Arc::new(AtomicUsize::new(0)); - let fill0 = Arc::new(AtomicUsize::new(0)); - let fill1 = Arc::new(AtomicUsize::new(0)); - let cfg0 = { - let log_calls = log0.clone(); - let fill_calls = fill0.clone(); - ExpertConfig::uniform("c0", move || { - Box::new(CountingFillPredict { - log_calls: log_calls.clone(), - fill_calls: fill_calls.clone(), - }) - }) - }; - let cfg1 = { - let log_calls = log1.clone(); - let fill_calls = fill1.clone(); - ExpertConfig::uniform("c1", move || { - Box::new(CountingFillPredict { - log_calls: log_calls.clone(), - fill_calls: fill_calls.clone(), - }) - }) - }; - let mut mix = NeuralMixture::new(&[cfg0, cfg1], 0.03); - - let mut row = [0.0; 256]; - mix.fill_log_probs(&mut row); - assert_eq!(fill0.load(Ordering::Relaxed), 1); - assert_eq!(fill1.load(Ordering::Relaxed), 1); - assert_eq!(log0.load(Ordering::Relaxed), 0); - assert_eq!(log1.load(Ordering::Relaxed), 0); - - let _ = mix.step(0); - assert_eq!(fill0.load(Ordering::Relaxed), 1); - assert_eq!(fill1.load(Ordering::Relaxed), 1); - assert_eq!(log0.load(Ordering::Relaxed), 0); - assert_eq!(log1.load(Ordering::Relaxed), 0); - } - - #[test] - fn runtime_begin_stream_propagates_to_experts() { - let seen_total = Arc::new(AtomicU64::new(0)); - let cfg = { - let seen_total = seen_total.clone(); - ExpertConfig::uniform("begin-aware", move || { - Box::new(BeginAwarePredict { - seen_total: seen_total.clone(), - began: false, - }) - }) - }; - - let spec = MixtureSpec::new( - MixtureKind::Bayes, - vec![crate::MixtureExpertSpec { - name: Some("begin-aware".to_string()), - log_prior: 0.0, - max_order: -1, - backend: RateBackend::Ctw { depth: 1 }, - }], - ); - let mut runtime = build_mixture_runtime(&spec, &[cfg]).expect("runtime"); - runtime.begin_stream(Some(123)).expect("begin stream"); - let _ = runtime.step(0); - assert_eq!(seen_total.load(Ordering::Relaxed), 123); - } - - #[test] - fn zpaq_fill_log_probs_does_not_drift_history() { - let backend = RateBackend::Zpaq { - method: "1".to_string(), - }; - let mut baseline = - RateBackendPredictor::from_backend(backend.clone(), -1, DEFAULT_MIN_PROB); - let mut probe = RateBackendPredictor::from_backend(backend, -1, DEFAULT_MIN_PROB); - - let history = b"history for zpaq predictor"; - for &b in history { - baseline.update(b); - probe.update(b); - } - - let mut row = [0.0f64; 256]; - probe.fill_log_probs(&mut row); - - let sym = b'k'; - let lp_base = baseline.log_prob(sym); - let lp_probe = probe.log_prob(sym); - assert!((lp_base - lp_probe).abs() < 1e-9); - assert!((row[sym as usize] - lp_base).abs() < 1e-9); - - baseline.update(sym); - probe.update(sym); - let next = b'q'; - let next_base = baseline.log_prob(next); - let next_probe = probe.log_prob(next); - assert!((next_base - next_probe).abs() < 1e-9); - } - - fn assert_predictor_log_probs_normalize_to_one(backend: RateBackend) { - let mut predictor = RateBackendPredictor::from_backend(backend, -1, DEFAULT_MIN_PROB); - for &b in b"normalization corpus for ctw/fac predictor checks" { - predictor.update(b); - } - let mut sum = 0.0f64; - for sym in 0u8..=255u8 { - sum += predictor.log_prob(sym).exp(); - } - assert!( - (sum - 1.0).abs() <= 1e-10, - "probability mass drift: sum={sum}" - ); - } - - #[test] - fn ctw_predictor_symbol_probs_normalize() { - assert_predictor_log_probs_normalize_to_one(RateBackend::Ctw { depth: 7 }); - } - - #[test] - fn fac_ctw_predictor_symbol_probs_normalize() { - assert_predictor_log_probs_normalize_to_one(RateBackend::FacCtw { - base_depth: 7, - num_percept_bits: 8, - encoding_bits: 8, - }); - } -} diff --git a/src/search.rs b/src/search.rs deleted file mode 100644 index 4398cfc5..00000000 --- a/src/search.rs +++ /dev/null @@ -1,792 +0,0 @@ -use crate::rosaplus::RosaPlus; -use crate::{InfotheoryCtx, RateBackend, cross_entropy_bytes, marginal_entropy_bytes}; -use rayon::prelude::*; -use std::collections::hash_map::DefaultHasher; -use std::fs; -use std::hash::{Hash, Hasher}; -use std::path::{Path, PathBuf}; - -#[derive(Debug, Clone)] -/// One scored retrieval unit returned by code search. -pub struct Snippet { - /// Source file containing the match/candidate. - pub path: PathBuf, - /// 1-based inclusive start line for snippet display. - pub start_line: usize, - /// 1-based inclusive end line for snippet display. - pub end_line: usize, - /// Raw candidate bytes used for entropy/rerank scoring. - pub content: Vec, - /// Final ranking score (larger is better). - pub score: f64, -} - -fn stage0_prefilter( - query_bytes: &[u8], - mut candidates: Vec, - opts: &SearchOptions, - debug: bool, -) -> Vec { - let n = candidates.len(); - if n == 0 { - return candidates; - } - - let frac = opts.stage0_keep_frac.clamp(0.0, 1.0); - if frac >= 1.0 { - return candidates; - } - - // Option A: Unigram (i.i.d.) likelihood-gain proxy. - // score0(x) = H0(Q) - H0(Q|X) - // where H0(Q|X) is computed as cross-entropy of Q under X's unigram model. - let h0_q = marginal_entropy_bytes(query_bytes); - candidates.par_iter_mut().for_each(|s| { - let h0_q_x = cross_entropy_bytes(query_bytes, &s.content, 0); - s.score = h0_q - h0_q_x; - }); - - let mut keep = ((n as f64) * frac).ceil() as usize; - keep = keep.max(opts.top_k).min(n); - if keep < n { - let nth = keep.saturating_sub(1); - candidates.select_nth_unstable_by(nth, |a, b| { - b.score - .partial_cmp(&a.score) - .unwrap_or(std::cmp::Ordering::Equal) - }); - candidates.truncate(keep); - } - - if debug { - println!( - "Stage-0 prefilter kept {}/{} candidates (frac={:.4})", - candidates.len(), - n, - frac - ); - } - - candidates -} - -#[derive(Debug, Clone, Copy, Eq, PartialEq)] -/// Candidate unit granularity for stage-0/1 collection. -pub enum SearchGranularity { - /// Split files into hashed line windows/snippets. - Snippet, - /// Treat each file as a single candidate. - File, -} - -#[derive(Debug, Clone, Copy, Eq, PartialEq)] -/// Stage-2 prior handling strategy for KMI reranking. -pub enum Stage2PriorMode { - /// Use the (full or summarized) universal prior as a prefix for compression metrics. - Use, - /// Do NOT use the universal prior in Stage 2 (pure NCD/KMI rerank on Stage-1-filtered set). - Disable, - /// Summarize the universal prior via an inner prior-less search over the prior corpus. - Summarize, -} - -#[derive(Clone)] -/// Tunables for the three-stage information-theoretic search pipeline. -pub struct SearchOptions { - /// Candidate granularity at collection time. - pub granularity: SearchGranularity, - /// Universal prior corpus path (file or directory). If set: - /// - Stage 1 always uses it. - /// - Stage 2 uses it by default (unless Stage2PriorMode::Disable). - pub universal_prior: Option, - /// Whether/how Stage-2 reranking uses universal prior context. - pub stage2_prior_mode: Stage2PriorMode, - /// Maximum model order used by entropy-rate estimators. - pub max_order: i64, - /// Number of final results to keep. - pub top_k: usize, - /// Fraction of candidates retained by the unigram prefilter. - pub stage0_keep_frac: f64, - /// Fully configured information-theory context/backend bundle. - pub ctx: InfotheoryCtx, -} - -impl Default for SearchOptions { - fn default() -> Self { - Self { - granularity: SearchGranularity::Snippet, - universal_prior: None, - stage2_prior_mode: Stage2PriorMode::Use, - max_order: 8, - top_k: 50, - stage0_keep_frac: 0.2, - ctx: InfotheoryCtx::with_zpaq("5"), - } - } -} - -/// Run search with default options and print top shell extraction commands. -pub fn run_search(query: &str, target_path: &str) { - run_search_with_options(query, target_path, &SearchOptions::default()); -} - -/// Run search with explicit options and print top shell extraction commands. -pub fn run_search_with_options(query: &str, target_path: &str, opts: &SearchOptions) { - let debug = std::env::var("DEBUG_SEARCH").is_ok(); - let results = search_with_options(query, target_path, opts); - for (i, snippet) in results.iter().take(5).enumerate() { - if debug { - println!( - "Rank {}: Score={:.6}, Path={}", - i + 1, - snippet.score, - snippet.path.display() - ); - } - println!( - "sed -n '{},{}p' {}", - snippet.start_line, - snippet.end_line, - snippet.path.display() - ); - } -} - -/// Run the full 3-stage search pipeline and return ranked results. -/// -/// The returned `Vec` is sorted by descending score, truncated -/// to `opts.top_k` entries. Each snippet carries its file path, line -/// range, content bytes, and final KMI-reranked score. -pub fn search_with_options(query: &str, target_path: &str, opts: &SearchOptions) -> Vec { - let debug = std::env::var("DEBUG_SEARCH").is_ok(); - let query_bytes = resolve_query_bytes(query); - if query_bytes.is_empty() { - eprintln!("Error: Query is empty."); - return Vec::new(); - } - - if debug { - println!( - "Scanning target: {} (granularity={:?}, prior={}, stage2_prior_mode={:?})", - target_path, - opts.granularity, - opts.universal_prior.as_deref().unwrap_or(""), - opts.stage2_prior_mode - ); - } - - let candidates = collect_candidates(target_path, opts.granularity); - if candidates.is_empty() { - eprintln!("No accessible files found in target '{}'.", target_path); - return Vec::new(); - } - - let candidates = stage0_prefilter(query_bytes.as_slice(), candidates, opts, debug); - if candidates.is_empty() { - eprintln!("No candidates remain after Stage-0 prefilter."); - return Vec::new(); - } - if debug { - println!("Found {} candidates. Filtering...", candidates.len()); - } - - // Stage 1: Filter - let mut scored_candidates = if let Some(prior_path) = opts.universal_prior.as_deref() { - stage1_filter_with_universal_prior(&query_bytes, prior_path, candidates, opts) - } else { - stage1_filter_no_prior(&query_bytes, candidates, opts) - }; - - let top_k_size = opts.top_k.min(scored_candidates.len()); - if top_k_size < scored_candidates.len() { - let nth = top_k_size.saturating_sub(1); - scored_candidates.select_nth_unstable_by(nth, |a, b| { - b.score - .partial_cmp(&a.score) - .unwrap_or(std::cmp::Ordering::Equal) - }); - scored_candidates.truncate(top_k_size); - } - - scored_candidates.sort_by(|a, b| { - b.score - .partial_cmp(&a.score) - .unwrap_or(std::cmp::Ordering::Equal) - }); - - let top_candidates = &mut scored_candidates[..top_k_size]; - if debug { - println!( - "Reranking top {} candidates with Kolmogorov Mutual Information...", - top_k_size - ); - } - - // Stage 2: Rerank - stage2_rerank_kmi(&query_bytes, top_candidates, opts); - top_candidates.sort_by(|a, b| { - b.score - .partial_cmp(&a.score) - .unwrap_or(std::cmp::Ordering::Equal) - }); - - scored_candidates -} - -fn resolve_query_bytes(query: &str) -> Vec { - let p = Path::new(query); - if p.exists() && fs::metadata(p).map(|m| m.is_file()).unwrap_or(false) { - fs::read(p).unwrap_or_else(|_| query.as_bytes().to_vec()) - } else { - query.as_bytes().to_vec() - } -} - -fn stage1_filter_no_prior( - query_bytes: &[u8], - candidates: Vec, - opts: &SearchOptions, -) -> Vec { - let h_q = opts.ctx.entropy_rate_bytes(query_bytes, opts.max_order); - - let scored: Vec = candidates - .into_par_iter() - .map(|mut snippet| { - let h_q_x = - opts.ctx - .cross_entropy_rate_bytes(query_bytes, &snippet.content, opts.max_order); - snippet.score = h_q - h_q_x; - snippet - }) - .collect(); - - // Keep equivalence with old behavior by not clamping. - scored -} - -fn stage1_filter_with_universal_prior( - query_bytes: &[u8], - prior_path: &str, - candidates: Vec, - opts: &SearchOptions, -) -> Vec { - #[cfg(feature = "backend-rwkv")] - if let Some((mut base, prior_snapshot)) = rwkv_prior_snapshot(opts, prior_path) { - let h_u_q = { - base.restore_runtime(&prior_snapshot); - base.cross_entropy_from_current(query_bytes).unwrap_or(0.0) - }; - return candidates - .into_par_iter() - .map_init( - || base.clone(), - |m: &mut crate::rwkvzip::Compressor, mut snippet| { - m.restore_runtime(&prior_snapshot); - let _ = m.absorb_chain(&[snippet.content.as_slice()]); - let h_ux_q = m.cross_entropy_from_current(query_bytes).unwrap_or(0.0); - snippet.score = h_u_q - h_ux_q; - snippet - }, - ) - .collect(); - } - - if !matches!(opts.ctx.rate_backend, RateBackend::RosaPlus) { - let prior_prefix = corpus_bytes(prior_path, SearchGranularity::File); - let h_u_q = opts - .ctx - .cross_entropy_conditional_chain(&[prior_prefix.as_slice()], query_bytes); - return candidates - .into_par_iter() - .map(|mut snippet| { - let h_ux_q = opts.ctx.cross_entropy_conditional_chain( - &[prior_prefix.as_slice(), snippet.content.as_slice()], - query_bytes, - ); - snippet.score = h_u_q - h_ux_q; - snippet - }) - .collect(); - } - - // PERFORMANCE NOTE: - // Training the prior using snippet-level windows would duplicate overlapping content - // and explode runtime. We *always* train/load the prior at file granularity. - let mut base = load_or_train_prior_model(prior_path, opts); - // For true conditional updates we require the fixed 256-byte alphabet LM. - // This ensures symbol indices remain stable across incremental updates. - base.ensure_lm_built_no_finalize_endpos(); - // Reduce the cost of cloning `base` per worker. - base.shrink_aux_buffers(); - - // Precompute query codepoints once (cross_entropy() would allocate this per call). - let query_cps: Vec = query_bytes.iter().map(|&b| b as u32).collect(); - let h_u_q = base.cross_entropy_cps(&query_cps); - - // True conditional update: - // score(x) = H_U(q) - H_{U+x}(q) - // by applying a reversible candidate update to the *full* prior model. - // - // MEMORY NOTE: - // `map_init(|| base.clone(), ...)` clones the model once per Rayon worker. - // For large priors this can blow up RSS. We cap worker count based on an estimate - // of model bytes and best-effort available memory (Linux). - let model_bytes = base.estimated_size_bytes().max(1); - let threads = memory_aware_threads(model_bytes); - let pool = rayon::ThreadPoolBuilder::new() - .num_threads(threads) - .build() - .expect("failed to build rayon pool"); - - pool.install(|| { - candidates - .into_par_iter() - .map_init( - || base.clone(), - |m, mut snippet| { - let mut tx = m.begin_tx(); - m.train_example_tx(&mut tx, &snippet.content); - let h_ux_q = m.cross_entropy_cps(&query_cps); - m.rollback_tx(tx); - snippet.score = h_u_q - h_ux_q; - snippet - }, - ) - .collect() - }) -} - -#[cfg(feature = "backend-rwkv")] -fn rwkv_prior_snapshot( - opts: &SearchOptions, - prior_path: &str, -) -> Option<(crate::rwkvzip::Compressor, crate::rwkvzip::RuntimeSnapshot)> { - let mut compressor = match &opts.ctx.rate_backend { - RateBackend::Rwkv7 { model } => crate::rwkvzip::Compressor::new_from_model(model.clone()), - RateBackend::Rwkv7Method { method } => { - crate::rwkvzip::Compressor::new_from_method(method).ok()? - } - _ => return None, - }; - - let prior_prefix = corpus_bytes(prior_path, SearchGranularity::File); - compressor.reset_and_prime(); - let _ = compressor.absorb_chain(&[prior_prefix.as_slice()]); - let snapshot = compressor.snapshot_runtime(); - Some((compressor, snapshot)) -} - -fn memory_aware_threads(model_bytes: usize) -> usize { - let hw = num_cpus::get().max(1); - let avail = linux_mem_available_bytes().unwrap_or(0); - if avail == 0 { - return hw; - } - - // Heuristic: allow up to 25% of available memory for (worker clones + overhead). - let budget = (avail / 4).max(model_bytes as u64); - let max_by_mem = (budget / (model_bytes as u64)).max(1) as usize; - hw.min(max_by_mem).max(1) -} - -fn linux_mem_available_bytes() -> Option { - // Linux-only best-effort. If parsing fails, fall back to unconstrained. - let s = std::fs::read_to_string("/proc/meminfo").ok()?; - for line in s.lines() { - if let Some(rest) = line.strip_prefix("MemAvailable:") { - let parts: Vec<&str> = rest.split_whitespace().collect(); - if parts.is_empty() { - return None; - } - let kb: u64 = parts[0].parse().ok()?; - return Some(kb.saturating_mul(1024)); - } - } - None -} - -fn stage2_rerank_kmi(query_bytes: &[u8], top_candidates: &mut [Snippet], opts: &SearchOptions) { - let prior_prefix: Option> = - match (opts.universal_prior.as_deref(), opts.stage2_prior_mode) { - (None, _) => None, - (Some(_), Stage2PriorMode::Disable) => None, - (Some(prior_path), Stage2PriorMode::Use) => { - Some(corpus_bytes(prior_path, SearchGranularity::File)) - } - (Some(prior_path), Stage2PriorMode::Summarize) => { - Some(summarize_prior_for_query(query_bytes, prior_path, opts)) - } - }; - - let cq = if let Some(prefix) = prior_prefix.as_deref() { - opts.ctx.compress_size_chain(&[prefix, query_bytes]) - } else { - opts.ctx.compress_size_chain(&[query_bytes]) - }; - - top_candidates.par_iter_mut().for_each(|snippet| { - let cx = if let Some(prefix) = prior_prefix.as_deref() { - opts.ctx - .compress_size_chain(&[prefix, snippet.content.as_slice()]) - } else { - opts.ctx.compress_size_chain(&[snippet.content.as_slice()]) - }; - - let c1 = if let Some(prefix) = prior_prefix.as_deref() { - opts.ctx - .compress_size_chain(&[prefix, snippet.content.as_slice(), query_bytes]) - } else { - opts.ctx - .compress_size_chain(&[snippet.content.as_slice(), query_bytes]) - }; - - let c2 = if let Some(prefix) = prior_prefix.as_deref() { - opts.ctx - .compress_size_chain(&[prefix, query_bytes, snippet.content.as_slice()]) - } else { - opts.ctx - .compress_size_chain(&[query_bytes, snippet.content.as_slice()]) - }; - - let c_joint = c1.min(c2); - snippet.score = if c_joint == u64::MAX { - 0.0 - } else { - (cq as f64 + cx as f64 - c_joint as f64).max(0.0) - }; - }); -} - -fn summarize_prior_for_query( - query_bytes: &[u8], - prior_path: &str, - opts: &SearchOptions, -) -> Vec { - // Prior-less search inside the prior corpus itself. - // We approximate K(q|x) via conditional compression: min(C(xq),C(qx)) - C(x), and select the MIN. - let candidates = collect_candidates(prior_path, opts.granularity); - if candidates.is_empty() { - return Vec::new(); - } - - let cq = opts.ctx.compress_size_chain(&[query_bytes]); - - let mut best: Option<(f64, Vec)> = None; - for c in candidates { - let cx = opts.ctx.compress_size_chain(&[c.content.as_slice()]); - - let cxq = opts - .ctx - .compress_size_chain(&[c.content.as_slice(), query_bytes]); - let cqx = opts - .ctx - .compress_size_chain(&[query_bytes, c.content.as_slice()]); - let c_joint = cxq.min(cqx); - if c_joint == u64::MAX { - continue; - } - // Conditional complexity proxy. - let k_q_given_x = (c_joint as f64 - cx as f64).max(0.0); - // Tie-breaker: if equal, prefer smaller candidate. - let candidate_key = (k_q_given_x, cx as f64, cq as f64); - let is_better = match &best { - None => true, - Some((best_k, best_bytes)) => { - let best_cx = opts.ctx.compress_size(best_bytes) as f64; - (candidate_key.0, candidate_key.1) < (*best_k, best_cx) - } - }; - if is_better { - best = Some((k_q_given_x, c.content)); - } - } - - best.map(|(_, b)| b).unwrap_or_default() -} - -fn train_rosa_on_corpus(m: &mut RosaPlus, corpus_path: &str, granularity: SearchGranularity) { - // Train incrementally on each candidate to avoid giant concatenations. - for c in collect_candidates(corpus_path, granularity) { - if !c.content.is_empty() { - m.train_example(&c.content); - } - } -} - -fn prior_cache_path(prior_path: &str, max_order: i64) -> Option { - let home = std::env::var("XDG_CACHE_HOME") - .ok() - .or_else(|| std::env::var("HOME").ok().map(|h| format!("{}/.cache", h))); - let cache_root = match home { - Some(h) => PathBuf::from(h).join("infotheory").join("rosa_prior"), - None => return None, - }; - - let mut hasher = DefaultHasher::new(); - // Cache format/version (bump when training or serialization semantics change). - (4u32).hash(&mut hasher); - prior_path.hash(&mut hasher); - max_order.hash(&mut hasher); - // file-granularity is baked into the cache key (we always use it for prior training) - ("file" as &str).hash(&mut hasher); - let key = hasher.finish(); - Some(cache_root.join(format!("prior_{:016x}.rosa", key))) -} - -fn load_or_train_prior_model(prior_path: &str, opts: &SearchOptions) -> RosaPlus { - // Load cached prior model if present. - if let Some(cache_path) = prior_cache_path(prior_path, opts.max_order) { - if let Some(parent) = cache_path.parent() { - let _ = fs::create_dir_all(parent); - } - if cache_path.exists() - && let Ok(mut m) = RosaPlus::load(cache_path.to_string_lossy().as_ref()) - { - // Ensure fixed 256-byte alphabet LM for incremental conditional updates. - if m.lm_alpha_n() != 256 { - m.build_lm_full_bytes_no_finalize_endpos(); - let _ = m.save(cache_path.to_string_lossy().as_ref()); - } - return m; - } - - // Train + save. - let mut m = RosaPlus::new(opts.max_order, false, 0, 42); - train_rosa_on_corpus(&mut m, prior_path, SearchGranularity::File); - // Build a fixed-byte alphabet LM once so the saved model is the full state. - m.build_lm_full_bytes_no_finalize_endpos(); - let _ = m.save(cache_path.to_string_lossy().as_ref()); - return m; - } - - // Fallback: no cache location available. - let mut m = RosaPlus::new(opts.max_order, false, 0, 42); - train_rosa_on_corpus(&mut m, prior_path, SearchGranularity::File); - m -} - -fn corpus_bytes(corpus_path: &str, granularity: SearchGranularity) -> Vec { - // Compression prior prefix requires a concrete byte buffer. - // We join candidates with a simple delimiter to preserve boundaries. - let mut out = Vec::new(); - for c in collect_candidates(corpus_path, granularity) { - if c.content.is_empty() { - continue; - } - out.extend_from_slice(&c.content); - out.extend_from_slice(b"\n\n"); - } - out -} - -fn collect_candidates(target: &str, granularity: SearchGranularity) -> Vec { - let mut snippets = Vec::new(); - let path = Path::new(target); - - if path.exists() { - if path.is_file() { - snippets.extend(file_to_candidates(path, granularity)); - } else if path.is_dir() { - visit_dirs(path, &mut snippets, granularity); - } - } - - snippets -} - -fn visit_dirs(dir: &Path, snippets: &mut Vec, granularity: SearchGranularity) { - if let Ok(entries) = fs::read_dir(dir) { - for entry in entries.flatten() { - let path = entry.path(); - if path.is_dir() { - if let Some(name_str) = path.file_name().and_then(|n| n.to_str()) - && !name_str.starts_with('.') - { - visit_dirs(&path, snippets, granularity); - } - } else { - snippets.extend(file_to_candidates(&path, granularity)); - } - } - } -} - -fn file_to_candidates(path: &Path, granularity: SearchGranularity) -> Vec { - let mut snippets = Vec::new(); - - // Only process text files - if let Some(ext) = path.extension() { - let ext_str = ext.to_string_lossy(); - if matches!( - ext_str.as_ref(), - "o" | "a" | "so" | "dll" | "exe" | "bin" | "png" | "jpg" | "zip" | "gz" - ) { - return snippets; - } - } - - match granularity { - SearchGranularity::File => { - if let Ok(bytes) = fs::read(path) - && !bytes.is_empty() - { - // Best-effort line count for `sed` output. - let lines = bytes.iter().filter(|&&b| b == b'\n').count() + 1; - snippets.push(Snippet { - path: path.to_path_buf(), - start_line: 1, - end_line: lines.max(1), - content: bytes, - score: 0.0, - }); - } - } - SearchGranularity::Snippet => { - if let Ok(bytes) = fs::read(path) { - if bytes.is_empty() { - return snippets; - } - - let window = 50usize; - let stride = 20usize; - - let mut line_starts: Vec = Vec::new(); - line_starts.push(0); - for (i, &b) in bytes.iter().enumerate() { - if b == b'\n' { - let next = i + 1; - if next < bytes.len() { - line_starts.push(next); - } - } - } - - if line_starts.is_empty() { - return snippets; - } - - let mut i = 0usize; - while i < line_starts.len() { - let end = (i + window).min(line_starts.len()); - let start_b = line_starts[i]; - let end_b = if end >= line_starts.len() { - bytes.len() - } else { - line_starts[end] - }; - - if end_b > start_b { - let content = bytes[start_b..end_b].to_vec(); - if content.len() > 50 { - snippets.push(Snippet { - path: path.to_path_buf(), - start_line: i + 1, - end_line: end, - content, - score: 0.0, - }); - } - } - - if end == line_starts.len() { - break; - } - i += stride; - } - } - } - } - snippets -} - -#[cfg(test)] -mod tests { - use super::*; - use std::time::{SystemTime, UNIX_EPOCH}; - - fn temp_path(prefix: &str) -> PathBuf { - let nanos = SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("clock before epoch") - .as_nanos(); - std::env::temp_dir().join(format!("infotheory-search-{prefix}-{nanos}")) - } - - #[test] - fn resolve_query_bytes_prefers_file_contents() { - let path = temp_path("query"); - fs::write(&path, b"query-from-file").expect("write query file"); - let got = resolve_query_bytes(path.to_string_lossy().as_ref()); - assert_eq!(got, b"query-from-file"); - let _ = fs::remove_file(path); - } - - #[test] - fn file_to_candidates_skips_binary_extensions() { - let path = temp_path("binary").with_extension("png"); - fs::write(&path, b"not-actually-image").expect("write pseudo-binary"); - let out = file_to_candidates(&path, SearchGranularity::File); - assert!(out.is_empty(), "binary extension should be skipped"); - let _ = fs::remove_file(path); - } - - #[test] - fn file_to_candidates_generates_snippets() { - let path = temp_path("snippet").with_extension("txt"); - let mut text = String::new(); - for i in 0..120 { - text.push_str(&format!("line-{i:03}\n")); - } - fs::write(&path, text.as_bytes()).expect("write snippet file"); - let out = file_to_candidates(&path, SearchGranularity::Snippet); - assert!(!out.is_empty(), "expected snippet candidates"); - assert!(out.iter().all(|s| s.end_line >= s.start_line)); - let _ = fs::remove_file(path); - } - - #[test] - fn collect_candidates_skips_hidden_directories() { - let root = temp_path("tree"); - let hidden = root.join(".hidden"); - let visible = root.join("visible"); - fs::create_dir_all(&hidden).expect("create hidden dir"); - fs::create_dir_all(&visible).expect("create visible dir"); - fs::write(hidden.join("secret.txt"), b"hidden").expect("write hidden file"); - fs::write(visible.join("public.txt"), b"visible\ntext\n").expect("write visible file"); - - let out = collect_candidates(root.to_string_lossy().as_ref(), SearchGranularity::File); - assert_eq!(out.len(), 1, "only visible file should be collected"); - assert!( - out[0].path.to_string_lossy().contains("public.txt"), - "unexpected collected file path: {}", - out[0].path.display() - ); - - let _ = fs::remove_dir_all(root); - } - - #[test] - fn stage0_prefilter_respects_topk_floor() { - let mut candidates = Vec::new(); - for i in 0..10 { - candidates.push(Snippet { - path: PathBuf::from(format!("f{i}.txt")), - start_line: 1, - end_line: 1, - content: format!("candidate-{i}").into_bytes(), - score: 0.0, - }); - } - let opts = SearchOptions { - top_k: 4, - stage0_keep_frac: 0.1, - ..SearchOptions::default() - }; - let kept = stage0_prefilter(b"candidate", candidates, &opts, false); - assert!( - kept.len() >= 4, - "stage0 must keep at least top_k candidates, got {}", - kept.len() - ); - } -} diff --git a/src/simd_math.rs b/src/simd_math.rs deleted file mode 100644 index 7053330d..00000000 --- a/src/simd_math.rs +++ /dev/null @@ -1,117 +0,0 @@ -use wide::f64x4; - -#[inline] -pub(crate) fn dot_wide(lhs: &[f64], rhs: &[f64]) -> f64 { - let n = lhs.len().min(rhs.len()); - let mut acc = f64x4::ZERO; - let mut i = 0usize; - while i + 4 <= n { - let a = f64x4::new([lhs[i], lhs[i + 1], lhs[i + 2], lhs[i + 3]]); - let b = f64x4::new([rhs[i], rhs[i + 1], rhs[i + 2], rhs[i + 3]]); - acc += a * b; - i += 4; - } - let lanes = acc.to_array(); - let mut out = lanes[0] + lanes[1] + lanes[2] + lanes[3]; - while i < n { - out += lhs[i] * rhs[i]; - i += 1; - } - out -} - -#[inline] -pub(crate) fn max_wide(xs: &[f64]) -> f64 { - if xs.is_empty() { - return f64::NEG_INFINITY; - } - let mut i = 0usize; - let mut max4 = f64x4::splat(f64::NEG_INFINITY); - while i + 4 <= xs.len() { - let v = f64x4::new([xs[i], xs[i + 1], xs[i + 2], xs[i + 3]]); - max4 = max4.max(v); - i += 4; - } - let lanes = max4.to_array(); - let mut max_v = lanes[0].max(lanes[1]).max(lanes[2]).max(lanes[3]); - while i < xs.len() { - if xs[i] > max_v { - max_v = xs[i]; - } - i += 1; - } - max_v -} - -#[inline] -pub(crate) fn logsumexp_wide(xs: &[f64]) -> f64 { - let max_v = max_wide(xs); - if !max_v.is_finite() { - return max_v; - } - let mut sum = 0.0; - for &v in xs { - sum += (v - max_v).exp(); - } - max_v + sum.ln() -} - -#[inline] -pub(crate) fn axpy_wide(dst: &mut [f64], alpha: f64, src: &[f64]) { - let n = dst.len().min(src.len()); - let mut i = 0usize; - let a4 = f64x4::splat(alpha); - while i + 4 <= n { - let d = f64x4::new([dst[i], dst[i + 1], dst[i + 2], dst[i + 3]]); - let s = f64x4::new([src[i], src[i + 1], src[i + 2], src[i + 3]]); - let r = d + a4 * s; - let lanes = r.to_array(); - dst[i] = lanes[0]; - dst[i + 1] = lanes[1]; - dst[i + 2] = lanes[2]; - dst[i + 3] = lanes[3]; - i += 4; - } - while i < n { - dst[i] += alpha * src[i]; - i += 1; - } -} - -#[allow(dead_code)] -#[inline] -pub(crate) fn affine3_wide( - dst: &mut [f64], - bias: &[f64], - weights: [f64; 3], - src0: &[f64], - src1: &[f64], - src2: &[f64], -) { - let n = dst.len(); - assert!(bias.len() >= n, "bias shorter than dst"); - assert!(src0.len() >= n, "src0 shorter than dst"); - assert!(src1.len() >= n, "src1 shorter than dst"); - assert!(src2.len() >= n, "src2 shorter than dst"); - let mut i = 0usize; - let w0 = f64x4::splat(weights[0]); - let w1 = f64x4::splat(weights[1]); - let w2 = f64x4::splat(weights[2]); - while i + 4 <= n { - let b = f64x4::new([bias[i], bias[i + 1], bias[i + 2], bias[i + 3]]); - let x0 = f64x4::new([src0[i], src0[i + 1], src0[i + 2], src0[i + 3]]); - let x1 = f64x4::new([src1[i], src1[i + 1], src1[i + 2], src1[i + 3]]); - let x2 = f64x4::new([src2[i], src2[i + 1], src2[i + 2], src2[i + 3]]); - let r = b + w0 * x0 + w1 * x1 + w2 * x2; - let lanes = r.to_array(); - dst[i] = lanes[0]; - dst[i + 1] = lanes[1]; - dst[i + 2] = lanes[2]; - dst[i + 3] = lanes[3]; - i += 4; - } - while i < n { - dst[i] = bias[i] + weights[0] * src0[i] + weights[1] * src1[i] + weights[2] * src2[i]; - i += 1; - } -} diff --git a/tests/aiqi_validation.rs b/tests/aiqi_validation.rs deleted file mode 100644 index 9ed39e36..00000000 --- a/tests/aiqi_validation.rs +++ /dev/null @@ -1,297 +0,0 @@ -//! AIQI validation tests. - -use infotheory::aixi::aiqi::{AiqiAgent, AiqiConfig}; -use infotheory::aixi::environment::{CoinFlip, CtwTest, Environment}; -use infotheory::aixi::model::RateBackendBitPredictor; -use infotheory::{MixtureKind, MixtureSpec, RateBackend}; -use std::sync::Arc; - -fn base_config() -> AiqiConfig { - AiqiConfig { - algorithm: "ac-ctw".to_string(), - ct_depth: 8, - observation_bits: 1, - observation_stream_len: 1, - reward_bits: 1, - agent_actions: 2, - min_reward: 0, - max_reward: 1, - reward_offset: 0, - discount_gamma: 0.99, - return_horizon: 2, - return_bins: 8, - augmentation_period: 2, - history_prune_keep_steps: None, - baseline_exploration: 0.01, - random_seed: Some(11), - rate_backend: None, - rate_backend_max_order: 20, - rwkv_model_path: None, - rosa_max_order: None, - zpaq_method: None, - } -} - -#[test] -fn aiqi_config_rejects_period_shorter_than_horizon() { - let mut cfg = base_config(); - cfg.return_horizon = 3; - cfg.augmentation_period = 2; - let err = cfg.validate().expect_err("N < H must be rejected"); - assert!(err.contains("augmentation_period")); -} - -#[test] -fn aiqi_config_rejects_non_power_of_two_return_bins() { - let mut cfg = base_config(); - cfg.return_bins = 3; - let err = cfg - .validate() - .expect_err("non-power-of-two return_bins must be rejected"); - assert!(err.contains("power of two")); -} - -#[test] -fn aiqi_config_rejects_zpaq_algorithm_in_strict_mode() { - let mut cfg = base_config(); - cfg.algorithm = "zpaq".to_string(); - let err = cfg - .validate() - .expect_err("strict AIQI should reject zpaq algorithm mode"); - assert!(err.contains("strict mode")); -} - -#[test] -fn aiqi_config_allows_unknown_algorithm_when_rate_backend_overrides() { - let mut cfg = base_config(); - cfg.algorithm = "unknown-backend-name".to_string(); - cfg.rate_backend = Some(RateBackend::Match { - hash_bits: 16, - min_len: 2, - max_len: 16, - base_mix: 0.05, - confidence_scale: 1.0, - }); - cfg.validate() - .expect("rate_backend override should make algorithm non-binding"); -} - -#[test] -fn aiqi_config_allows_algorithm_zpaq_when_rate_backend_overrides() { - let mut cfg = base_config(); - cfg.algorithm = "zpaq".to_string(); - cfg.rate_backend = Some(RateBackend::RosaPlus); - cfg.validate() - .expect("rate_backend override should ignore algorithm=zpaq"); -} - -#[test] -fn aiqi_config_rejects_zpaq_rate_backend_in_strict_mode() { - let mut cfg = base_config(); - cfg.rate_backend = Some(RateBackend::Zpaq { - method: "1".to_string(), - }); - let err = cfg - .validate() - .expect_err("strict AIQI should reject zpaq rate backend"); - assert!(err.contains("strict frozen conditioning")); -} - -#[test] -fn aiqi_config_rejects_invalid_programmatic_mixture_rate_backend() { - let mut cfg = base_config(); - cfg.rate_backend = Some(RateBackend::Mixture { - spec: Arc::new(MixtureSpec::new(MixtureKind::Bayes, vec![])), - }); - let err = cfg - .validate() - .expect_err("empty mixture backend should be rejected"); - assert!(err.contains("invalid rate_backend")); - assert!(err.contains("must include at least one expert")); -} - -#[test] -fn aiqi_coinflip_smoke_runs() { - let mut agent = AiqiAgent::new(base_config()).expect("valid AIQI config"); - let mut env = CoinFlip::new(0.8); - - let mut total_reward = 0i64; - for _ in 0..64 { - let action = agent.get_planned_action(); - env.perform_action(action); - let obs_stream = env.drain_observations(); - let rew = env.get_reward(); - agent - .observe_transition(action, &obs_stream, rew) - .expect("transition must be accepted"); - total_reward += rew; - } - - assert!(total_reward >= 0); -} - -#[test] -fn aiqi_learns_ctw_test_pattern() { - let mut cfg = base_config(); - cfg.discount_gamma = 0.7; - cfg.return_horizon = 4; - cfg.augmentation_period = 4; - cfg.ct_depth = 10; - cfg.baseline_exploration = 1e-6; - - let mut agent = AiqiAgent::new(cfg).expect("valid AIQI config"); - let mut env = CtwTest::new(); - - let mut total_reward = 0i64; - for _ in 0..120 { - let action = agent.get_planned_action(); - env.perform_action(action); - let obs_stream = env.drain_observations(); - let rew = env.get_reward(); - agent - .observe_transition(action, &obs_stream, rew) - .expect("transition must be accepted"); - total_reward += rew; - } - - assert!( - total_reward > 50, - "AIQI failed to learn CtwTest pattern; total_reward={total_reward}" - ); -} - -#[test] -fn aiqi_with_generic_rate_backend_smoke_runs() { - let mut cfg = base_config(); - cfg.rate_backend = Some(RateBackend::Match { - hash_bits: 16, - min_len: 2, - max_len: 16, - base_mix: 0.05, - confidence_scale: 1.0, - }); - cfg.rate_backend_max_order = 8; - - let mut agent = AiqiAgent::new(cfg).expect("valid AIQI config"); - let mut env = CoinFlip::new(0.7); - - for _ in 0..24 { - let action = agent.get_planned_action(); - env.perform_action(action); - let obs_stream = env.drain_observations(); - let rew = env.get_reward(); - agent - .observe_transition(action, &obs_stream, rew) - .expect("transition must be accepted"); - } - - assert!(agent.steps_observed() >= 24); -} - -#[test] -fn aiqi_with_rosa_generic_planner_smoke_runs() { - let mut cfg = base_config(); - cfg.algorithm = "rosa".to_string(); - cfg.rosa_max_order = Some(8); - - let mut agent = AiqiAgent::new(cfg).expect("valid AIQI config"); - let mut env = CoinFlip::new(0.7); - - for _ in 0..24 { - let action = agent.get_planned_action(); - env.perform_action(action); - let obs_stream = env.drain_observations(); - let rew = env.get_reward(); - agent - .observe_transition(action, &obs_stream, rew) - .expect("transition must be accepted"); - } - - assert!(agent.steps_observed() >= 24); -} - -#[test] -fn aiqi_optional_history_pruning_smoke_runs() { - let mut cfg = base_config(); - cfg.return_horizon = 3; - cfg.augmentation_period = 4; - cfg.history_prune_keep_steps = Some(16); - - let mut agent = AiqiAgent::new(cfg).expect("valid AIQI config"); - let mut env = CoinFlip::new(0.7); - - for _ in 0..128 { - let action = agent.get_planned_action(); - env.perform_action(action); - let obs_stream = env.drain_observations(); - let rew = env.get_reward(); - agent - .observe_transition(action, &obs_stream, rew) - .expect("transition must be accepted"); - } - - assert_eq!(agent.steps_observed(), 128); -} - -#[test] -fn aiqi_seeded_policy_is_reproducible() { - let mut cfg = base_config(); - cfg.baseline_exploration = 0.35; - cfg.random_seed = Some(987654321); - - let mut a = AiqiAgent::new(cfg.clone()).expect("valid AIQI config"); - let mut b = AiqiAgent::new(cfg).expect("valid AIQI config"); - - for step in 0..128usize { - let act_a = a.get_planned_action(); - let act_b = b.get_planned_action(); - assert_eq!(act_a, act_b, "action mismatch at step {step}"); - - let obs = [(step % 2) as u64]; - let rew = (step % 2) as i64; - a.observe_transition(act_a, &obs, rew) - .expect("transition should be accepted"); - b.observe_transition(act_b, &obs, rew) - .expect("transition should be accepted"); - } -} - -#[test] -fn rate_backend_bit_predictor_rejects_zpaq_backend() { - let err = match RateBackendBitPredictor::new( - RateBackend::Zpaq { - method: "1".to_string(), - }, - 8, - ) { - Ok(_) => panic!("zpaq must be rejected in RateBackendBitPredictor"), - Err(err) => err, - }; - assert!(err.contains("does not support zpaq backends")); -} - -#[cfg(feature = "backend-rwkv")] -#[test] -fn aiqi_config_rejects_rwkv_without_model_path_when_no_rate_backend() { - let mut cfg = base_config(); - cfg.algorithm = "rwkv".to_string(); - cfg.rwkv_model_path = None; - cfg.rate_backend = None; - - let err = cfg - .validate() - .expect_err("algorithm=rwkv without path and without rate_backend override must fail"); - assert!(err.contains("rwkv_model_path")); -} - -#[cfg(feature = "backend-rwkv")] -#[test] -fn aiqi_config_allows_rwkv_without_model_path_with_rate_backend_override() { - let mut cfg = base_config(); - cfg.algorithm = "rwkv".to_string(); - cfg.rwkv_model_path = None; - cfg.rate_backend = Some(RateBackend::RosaPlus); - - cfg.validate() - .expect("rate_backend override should avoid requiring rwkv_model_path"); -} diff --git a/tests/aixi_discounting.rs b/tests/aixi_discounting.rs deleted file mode 100644 index 90b5cae6..00000000 --- a/tests/aixi_discounting.rs +++ /dev/null @@ -1,91 +0,0 @@ -use infotheory::aixi::agent::{Agent, AgentConfig}; -use infotheory::aixi::common::ObservationKeyMode; -use infotheory::aixi::mcts::AgentSimulator; - -fn approx_eq(a: f64, b: f64, eps: f64) { - assert!( - (a - b).abs() <= eps, - "expected {a} ≈ {b} (|diff|={})", - (a - b).abs() - ); -} - -fn mk_agent(discount_gamma: f64, horizon: usize, min_reward: i64, max_reward: i64) -> Agent { - Agent::new(AgentConfig { - algorithm: "ctw".to_string(), - ct_depth: 8, - agent_horizon: horizon, - observation_bits: 1, - observation_stream_len: 1, - observation_key_mode: ObservationKeyMode::FullStream, - reward_bits: 8, - agent_actions: 2, - num_simulations: 1, - exploration_exploitation_ratio: 1.0, - discount_gamma, - min_reward, - max_reward, - reward_offset: (-min_reward).max(0), - random_seed: Some(13), - rate_backend: None, - rate_backend_max_order: 20, - rwkv_model_path: None, - rwkv_method: None, - mamba_model_path: None, - mamba_method: None, - rosa_max_order: None, - zpaq_method: None, - }) -} - -#[test] -fn norm_reward_undiscounted_hits_endpoints() { - let horizon = 5; - let min = -2; - let max = 6; - let agent = mk_agent(1.0, horizon, min, max); - - let sum = horizon as f64; - let min_cum = (min as f64) * sum; - let max_cum = (max as f64) * sum; - - let z0 = agent.norm_reward(min_cum); - let z1 = agent.norm_reward(max_cum); - - approx_eq(z0, 0.0, 1e-12); - approx_eq(z1, 1.0, 1e-12); -} - -#[test] -fn norm_reward_discounted_hits_endpoints() { - let horizon = 10; - let min = -1; - let max = 3; - let gamma = 0.7; - let agent = mk_agent(gamma, horizon, min, max); - - let sum = (1.0 - gamma.powi(horizon as i32)) / (1.0 - gamma); - let min_cum = (min as f64) * sum; - let max_cum = (max as f64) * sum; - - let z0 = agent.norm_reward(min_cum); - let z1 = agent.norm_reward(max_cum); - - approx_eq(z0, 0.0, 1e-10); - approx_eq(z1, 1.0, 1e-10); -} - -#[test] -fn norm_reward_midpoint_is_half() { - let horizon = 7; - let min = -4; - let max = 4; - let gamma = 0.5; - let agent = mk_agent(gamma, horizon, min, max); - - let sum = (1.0 - gamma.powi(horizon as i32)) / (1.0 - gamma); - let mid_cum = ((min + max) as f64 / 2.0) * sum; - - let z = agent.norm_reward(mid_cum); - approx_eq(z, 0.5, 1e-10); -} diff --git a/tests/aixi_validation.rs b/tests/aixi_validation.rs deleted file mode 100644 index e0ceea28..00000000 --- a/tests/aixi_validation.rs +++ /dev/null @@ -1,614 +0,0 @@ -//! AIXI Module Validation Tests -//! -//! Tests for predictors, environments, and agents. - -use infotheory::aixi::agent::{Agent, AgentConfig}; -use infotheory::aixi::common::{Action, ObservationKeyMode}; -use infotheory::aixi::environment::{CoinFlip, CtwTest, Environment}; -use infotheory::aixi::model::{CtwPredictor, Predictor, RateBackendBitPredictor, RosaPredictor}; -use infotheory::{MAX_MIXTURE_NESTING, MixtureExpertSpec, MixtureKind, MixtureSpec, RateBackend}; -use std::sync::Arc; - -// ============================================================================ -// Predictor Consistency Tests -// ============================================================================ - -fn test_predictor_sum_to_one(mut predictor: Box, name: &str) { - // Feed some history - for &sym in &[true, false, true, true, false] { - predictor.update(sym); - } - - let p_true = predictor.predict_prob(true); - let p_false = predictor.predict_prob(false); - - // Check they sum to 1.0 (binary predictor) - let sum = p_true + p_false; - println!("{name}: P(1)={p_true:.6}, P(0)={p_false:.6}, Sum={sum:.6}"); - assert!( - (sum - 1.0).abs() < 1e-6, - "{name}: Probabilities must sum to 1.0, got {p_true} + {p_false} = {sum}" - ); - - // Check range - assert!( - (0.0..=1.0).contains(&p_true), - "{name}: Prob out of range: {p_true}" - ); -} - -#[test] -fn ctw_probabilities_valid() { - test_predictor_sum_to_one(Box::new(CtwPredictor::new(8)), "CTW"); -} - -#[test] -fn rosa_probabilities_valid() { - test_predictor_sum_to_one(Box::new(RosaPredictor::new(8)), "ROSA"); -} - -fn test_predictor_revert(mut predictor: Box, name: &str) { - let history = [true, false, true, true, false, false, true]; - - // Update all - for &sym in &history { - predictor.update(sym); - } - let prob_after_updates = predictor.predict_prob(true); - - // Revert all - for _ in &history { - predictor.revert(); - } - - // Should be back to initial state (approx 0.5 for uniform prior) - let prob_reverted = predictor.predict_prob(true); - - println!("{name}: After full revert, p(1) = {prob_reverted}"); - assert!( - (prob_reverted - 0.5).abs() < 0.1, - "{name}: Reverted predictor should be roughly uninformed (0.5), got {prob_reverted}" - ); - - // Re-apply and check we get same result as before - for &sym in &history { - predictor.update(sym); - } - let prob_redo = predictor.predict_prob(true); - assert!( - (prob_redo - prob_after_updates).abs() < 1e-9, - "{name}: Deterministic replay failed. {prob_redo} != {prob_after_updates}" - ); -} - -#[test] -fn ctw_update_revert_consistency() { - test_predictor_revert(Box::new(CtwPredictor::new(8)), "CTW"); -} - -#[test] -fn rosa_update_revert_consistency() { - test_predictor_revert(Box::new(RosaPredictor::new(8)), "ROSA"); -} - -fn nested_generic_backend() -> RateBackend { - let inner = MixtureSpec::new( - MixtureKind::Bayes, - vec![ - MixtureExpertSpec { - name: Some("ctw".to_string()), - log_prior: 0.0, - max_order: -1, - backend: RateBackend::Ctw { depth: 6 }, - }, - MixtureExpertSpec { - name: Some("match".to_string()), - log_prior: 0.0, - max_order: -1, - backend: RateBackend::Match { - hash_bits: 18, - min_len: 2, - max_len: 32, - base_mix: 0.05, - confidence_scale: 1.0, - }, - }, - ], - ) - .with_alpha(0.03); - let outer = MixtureSpec::new( - MixtureKind::Convex, - vec![ - MixtureExpertSpec { - name: Some("nested".to_string()), - log_prior: 0.0, - max_order: -1, - backend: RateBackend::Mixture { - spec: Arc::new(inner), - }, - }, - MixtureExpertSpec { - name: Some("ppmd".to_string()), - log_prior: 0.0, - max_order: -1, - backend: RateBackend::Ppmd { - order: 4, - memory_mb: 8, - }, - }, - ], - ) - .with_alpha(1.25); - RateBackend::Mixture { - spec: Arc::new(outer), - } -} - -fn predictor_snapshot(predictor: &mut dyn Predictor) -> (f64, f64) { - (predictor.predict_prob(false), predictor.predict_prob(true)) -} - -fn assert_snapshot_eq(actual: (f64, f64), expected: (f64, f64), label: &str) { - assert!( - (actual.0 - expected.0).abs() < 1e-12 && (actual.1 - expected.1).abs() < 1e-12, - "{label}: expected {:?}, got {:?}", - expected, - actual - ); -} - -#[test] -fn rate_backend_bit_predictor_roundtrips_nested_mixtures() { - let mut predictor = - RateBackendBitPredictor::new(nested_generic_backend(), 8).expect("valid predictor"); - - let initial = predictor_snapshot(&mut predictor); - - predictor.update(true); - let after_update = predictor_snapshot(&mut predictor); - predictor.revert(); - assert_snapshot_eq( - predictor_snapshot(&mut predictor), - initial, - "revert after update", - ); - - predictor.update(true); - assert_snapshot_eq( - predictor_snapshot(&mut predictor), - after_update, - "redo after update", - ); - - predictor.update_history(false); - let after_frozen = predictor_snapshot(&mut predictor); - predictor.pop_history(); - assert_snapshot_eq( - predictor_snapshot(&mut predictor), - after_update, - "pop_history after frozen update", - ); - - predictor.update_history(false); - assert_snapshot_eq( - predictor_snapshot(&mut predictor), - after_frozen, - "redo after frozen update", - ); -} - -#[test] -fn rate_backend_bit_predictor_roundtrips_sequitur_backend() { - let mut predictor = - RateBackendBitPredictor::new(RateBackend::Sequitur { context_bytes: 32 }, 8) - .expect("valid sequitur predictor"); - - let initial = predictor_snapshot(&mut predictor); - - predictor.update(true); - let after_update = predictor_snapshot(&mut predictor); - predictor.revert(); - assert_snapshot_eq( - predictor_snapshot(&mut predictor), - initial, - "sequitur revert after update", - ); - - predictor.update(true); - assert_snapshot_eq( - predictor_snapshot(&mut predictor), - after_update, - "sequitur redo after update", - ); - - predictor.update_history(false); - let after_frozen = predictor_snapshot(&mut predictor); - predictor.pop_history(); - assert_snapshot_eq( - predictor_snapshot(&mut predictor), - after_update, - "sequitur pop_history after frozen update", - ); - - predictor.update_history(false); - assert_snapshot_eq( - predictor_snapshot(&mut predictor), - after_frozen, - "sequitur redo after frozen update", - ); -} - -// ============================================================================ -// Environment Tests -// ============================================================================ - -#[test] -fn ctw_test_env_is_deterministic() { - let mut env1 = CtwTest::new(); - let mut env2 = CtwTest::new(); - - for i in 0..50 { - let action = (i % 2) as Action; - env1.perform_action(action); - env2.perform_action(action); - - assert_eq!( - env1.get_observation(), - env2.get_observation(), - "Obs mismatch at step {i}" - ); - assert_eq!( - env1.get_reward(), - env2.get_reward(), - "Reward mismatch at step {i}" - ); - } -} - -// ============================================================================ -// Agent / MCTS Tests -// ============================================================================ - -fn run_agent_env(agent: &mut Agent, mut env: T, cycles: usize) -> f64 { - let mut total_reward = 0.0; - let mut obs_stream = env.drain_observations(); - let mut prev_rew = env.get_reward(); - let mut prev_act = 0; - - for _ in 0..cycles { - agent.model_update_percept_stream(&obs_stream, prev_rew); - let action = agent.get_planned_action(&obs_stream, prev_rew, prev_act); - - // Update model with chosen action (so model sees: ...p a p a p a...) - agent.model_update_action_external(action); - - env.perform_action(action); - - obs_stream = env.drain_observations(); - let rew = env.get_reward(); - - // Update model with observed percept stream - agent.model_update_percept_stream(&obs_stream, rew); - - total_reward += rew as f64; - prev_rew = rew; - prev_act = action; - - if env.is_finished() { - break; - } - } - total_reward -} - -fn generic_agent_config(rate_backend: RateBackend) -> AgentConfig { - AgentConfig { - algorithm: "ignored-by-rate-backend".into(), - ct_depth: 8, - agent_horizon: 5, - observation_bits: 1, - observation_stream_len: 1, - observation_key_mode: ObservationKeyMode::FullStream, - reward_bits: 1, - agent_actions: 2, - num_simulations: 60, - exploration_exploitation_ratio: 1.4, - discount_gamma: 1.0, - min_reward: 0, - max_reward: 1, - reward_offset: 0, - random_seed: Some(2026), - rate_backend: Some(rate_backend), - rate_backend_max_order: 8, - rwkv_model_path: None, - rwkv_method: None, - mamba_model_path: None, - mamba_method: None, - rosa_max_order: Some(8), - zpaq_method: None, - } -} - -fn mixture_backend(kind: MixtureKind) -> RateBackend { - let experts = vec![ - MixtureExpertSpec { - name: Some("ctw".to_string()), - log_prior: 0.0, - max_order: -1, - backend: RateBackend::Ctw { depth: 8 }, - }, - MixtureExpertSpec { - name: Some("rosa".to_string()), - log_prior: 0.0, - max_order: 8, - backend: RateBackend::RosaPlus, - }, - ]; - let alpha = match kind { - MixtureKind::Switching => 0.05, - MixtureKind::Convex => 1.25, - _ => 0.03, - }; - RateBackend::Mixture { - spec: Arc::new(MixtureSpec::new(kind, experts).with_alpha(alpha)), - } -} - -fn deeply_nested_bayes_backend(depth: usize) -> RateBackend { - let mut backend = RateBackend::Ctw { depth: 4 }; - for level in 0..depth { - backend = RateBackend::Mixture { - spec: Arc::new(MixtureSpec::new( - MixtureKind::Bayes, - vec![MixtureExpertSpec { - name: Some(format!("level-{level}")), - log_prior: 0.0, - max_order: -1, - backend, - }], - )), - }; - } - backend -} - -#[test] -fn agent_solves_ctw_test_environment() { - let config = AgentConfig { - algorithm: "ctw".into(), - ct_depth: 8, - agent_horizon: 8, // Increased from 4 - observation_bits: 1, - observation_stream_len: 1, - observation_key_mode: infotheory::aixi::common::ObservationKeyMode::FullStream, - reward_bits: 1, - agent_actions: 2, - num_simulations: 200, // Increased from 50 - exploration_exploitation_ratio: 2.0, - discount_gamma: 1.0, - min_reward: 0, - max_reward: 1, - reward_offset: 0, - random_seed: Some(17), - rate_backend: None, - rate_backend_max_order: 20, - rwkv_model_path: None, - rwkv_method: None, - mamba_model_path: None, - mamba_method: None, - rosa_max_order: None, - zpaq_method: None, - }; - - let mut agent = Agent::new(config); - let env = CtwTest::new(); - - let cycles = 100; - let total_reward = run_agent_env(&mut agent, env, cycles); - - println!( - "Agent Total Reward on CtwTest (100 cycles): {}", - total_reward - ); - - // Agent should learn pattern and get reasonable reward - assert!( - total_reward > 50.0, - "Agent failed to learn CtwTest pattern. Reward: {total_reward}" - ); -} - -#[test] -fn agent_regret_sublinear_coinflip() { - let config = AgentConfig { - algorithm: "ctw".into(), - ct_depth: 4, - agent_horizon: 4, // Increased from 2 - observation_bits: 1, - observation_stream_len: 1, - observation_key_mode: infotheory::aixi::common::ObservationKeyMode::FullStream, - reward_bits: 1, - agent_actions: 2, - num_simulations: 100, // Increased from 20 - exploration_exploitation_ratio: 1.0, - discount_gamma: 1.0, - min_reward: 0, - max_reward: 1, - reward_offset: 0, - random_seed: Some(23), - rate_backend: None, - rate_backend_max_order: 20, - rwkv_model_path: None, - rwkv_method: None, - mamba_model_path: None, - mamba_method: None, - rosa_max_order: None, - zpaq_method: None, - }; - - let mut agent = Agent::new(config); - let env = CoinFlip::new(0.8); - - let cycles = 500; - let total_reward = run_agent_env(&mut agent, env, cycles); - - let expected_optimal = 0.8 * cycles as f64; - let regret = expected_optimal - total_reward; - let regret_per_step = regret / cycles as f64; - - println!( - "CoinFlip(0.8): Reward={total_reward}, Opt={expected_optimal}, Regret/step={regret_per_step:.4}" - ); - - // Regret should be reasonable (< 0.25 per step) - assert!(regret_per_step < 0.25, "Regret too high: {regret_per_step}"); -} - -#[test] -fn agent_seeded_policy_is_reproducible_on_deterministic_env() { - let config = AgentConfig { - algorithm: "ctw".into(), - ct_depth: 8, - agent_horizon: 6, - observation_bits: 1, - observation_stream_len: 1, - observation_key_mode: infotheory::aixi::common::ObservationKeyMode::FullStream, - reward_bits: 1, - agent_actions: 2, - num_simulations: 80, - exploration_exploitation_ratio: 1.4, - discount_gamma: 1.0, - min_reward: 0, - max_reward: 1, - reward_offset: 0, - random_seed: Some(12345), - rate_backend: None, - rate_backend_max_order: 20, - rwkv_model_path: None, - rwkv_method: None, - mamba_model_path: None, - mamba_method: None, - rosa_max_order: None, - zpaq_method: None, - }; - - let mut a = Agent::new(config.clone()); - let mut b = Agent::new(config); - let mut env_a = CtwTest::new(); - let mut env_b = CtwTest::new(); - - let mut obs_a = env_a.drain_observations(); - let mut obs_b = env_b.drain_observations(); - let mut rew_a = env_a.get_reward(); - let mut rew_b = env_b.get_reward(); - let mut prev_a = 0u64; - let mut prev_b = 0u64; - - for step in 0..64usize { - assert_eq!(obs_a, obs_b, "observation mismatch at step {step}"); - assert_eq!(rew_a, rew_b, "reward mismatch at step {step}"); - - a.model_update_percept_stream(&obs_a, rew_a); - b.model_update_percept_stream(&obs_b, rew_b); - - let act_a = a.get_planned_action(&obs_a, rew_a, prev_a); - let act_b = b.get_planned_action(&obs_b, rew_b, prev_b); - assert_eq!(act_a, act_b, "action mismatch at step {step}"); - - a.model_update_action_external(act_a); - b.model_update_action_external(act_b); - - env_a.perform_action(act_a); - env_b.perform_action(act_b); - obs_a = env_a.drain_observations(); - obs_b = env_b.drain_observations(); - rew_a = env_a.get_reward(); - rew_b = env_b.get_reward(); - prev_a = act_a; - prev_b = act_b; - } -} - -#[test] -fn agent_config_allows_unknown_algorithm_when_rate_backend_overrides() { - let cfg = generic_agent_config(RateBackend::Ppmd { - order: 4, - memory_mb: 8, - }); - assert!(cfg.validate().is_ok()); - let mut agent = Agent::try_new(cfg).expect("rate_backend override should be valid"); - let action = agent.get_planned_action(&[0], 0, 0); - assert!(action < 2); -} - -#[test] -fn agent_config_allows_algorithm_zpaq_when_rate_backend_overrides() { - let mut cfg = generic_agent_config(RateBackend::Ctw { depth: 8 }); - cfg.algorithm = "zpaq".to_string(); - cfg.zpaq_method = Some("1".to_string()); - assert!(cfg.validate().is_ok()); - let mut agent = Agent::try_new(cfg).expect("rate_backend override should bypass legacy zpaq"); - let action = agent.get_planned_action(&[0], 0, 0); - assert!(action < 2); -} - -#[test] -fn agent_config_rejects_zpaq_rate_backend_in_strict_mode() { - let cfg = generic_agent_config(RateBackend::Mixture { - spec: Arc::new(MixtureSpec::new( - MixtureKind::Bayes, - vec![MixtureExpertSpec { - name: Some("bad-zpaq".to_string()), - log_prior: 0.0, - max_order: -1, - backend: RateBackend::Zpaq { - method: "1".to_string(), - }, - }], - )), - }); - let err = cfg - .validate() - .expect_err("zpaq-backed generic MC-AIXI should be rejected"); - assert!(err.contains("A Monte-Carlo AIXI Approximation")); - assert!(err.contains("zpaq")); -} - -#[test] -fn agent_config_rejects_invalid_programmatic_mixture_rate_backend() { - let cfg = generic_agent_config(RateBackend::Mixture { - spec: Arc::new(MixtureSpec::new(MixtureKind::Bayes, vec![])), - }); - let err = cfg - .validate() - .expect_err("empty mixture backend should be rejected"); - assert!(err.contains("invalid rate_backend")); - assert!(err.contains("must include at least one expert")); -} - -#[test] -fn agent_config_rejects_programmatic_mixture_nesting_overflow() { - let cfg = generic_agent_config(deeply_nested_bayes_backend(MAX_MIXTURE_NESTING + 1)); - let err = cfg - .validate() - .expect_err("overly deep nested mixture should be rejected"); - assert!(err.contains("invalid rate_backend")); - assert!(err.contains("nesting too deep")); -} - -#[test] -fn agent_with_generic_mixture_backends_smoke_runs() { - for (kind, label) in [ - (MixtureKind::Bayes, "bayes"), - (MixtureKind::Switching, "switching"), - (MixtureKind::Convex, "convex"), - ] { - let mut agent = - Agent::try_new(generic_agent_config(mixture_backend(kind))).expect("valid mixture"); - let total_reward = run_agent_env(&mut agent, CtwTest::new(), 48); - assert!( - total_reward > 16.0, - "{label} mixture backend reward too low on CtwTest: {total_reward}" - ); - } -} diff --git a/tests/api_surface.rs b/tests/api_surface.rs deleted file mode 100644 index 58a1c2c6..00000000 --- a/tests/api_surface.rs +++ /dev/null @@ -1,196 +0,0 @@ -use infotheory::{ - CompressionBackend, GenerationConfig, InfotheoryCtx, MixtureKind, MixtureSpec, RateBackend, - RateBackendSession, biased_entropy_rate_backend, biased_entropy_rate_bytes, - conditional_entropy_bytes, conditional_entropy_rate_bytes, cross_entropy_bytes, - cross_entropy_rate_backend, cross_entropy_rate_bytes, d_kl_bytes, entropy_rate_backend, - entropy_rate_bytes, get_default_ctx, intrinsic_dependence_bytes, joint_entropy_rate_backend, - joint_entropy_rate_bytes, joint_marginal_entropy_bytes, js_div_bytes, marginal_entropy_bytes, - mutual_information_bytes, mutual_information_marg_bytes, mutual_information_rate_backend, - mutual_information_rate_bytes, ned_bytes, ned_cons_bytes, ned_cons_marg_bytes, - ned_cons_rate_bytes, ned_marg_bytes, ned_rate_backend, ned_rate_bytes, nhd_bytes, nte_bytes, - nte_marg_bytes, nte_rate_backend, nte_rate_bytes, resistance_to_transformation_bytes, - set_default_ctx, tvd_bytes, -}; -#[cfg(feature = "backend-zpaq")] -use infotheory::{ - NcdVariant, compress_bytes_backend, compress_size_backend, compress_size_chain_backend, - conditional_entropy_paths, cross_entropy_paths, decompress_bytes_backend, get_bytes_from_paths, - get_compressed_size, get_compressed_size_parallel, get_compressed_sizes_from_paths, - get_parallel_compressed_sizes_from_parallel_paths, - get_parallel_compressed_sizes_from_sequential_paths, - get_sequential_compressed_sizes_from_parallel_paths, - get_sequential_compressed_sizes_from_sequential_paths, js_divergence_paths, - kl_divergence_paths, mutual_information_paths, ncd_bytes, ncd_bytes_backend, ncd_bytes_default, - ncd_cons, ncd_matrix_bytes, ncd_matrix_paths, ncd_paths, ncd_paths_backend, ncd_sym_cons, - ncd_sym_vitanyi, ncd_vitanyi, ned_paths, nhd_paths, nte_paths, tvd_paths, -}; -#[cfg(feature = "backend-zpaq")] -use std::fs; -#[cfg(feature = "backend-zpaq")] -use std::path::PathBuf; -use std::sync::Arc; -#[cfg(feature = "backend-zpaq")] -use std::time::{SystemTime, UNIX_EPOCH}; - -#[cfg(feature = "backend-zpaq")] -fn temp_file(name: &str, contents: &[u8]) -> PathBuf { - let ts = SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("clock should be monotonic") - .as_nanos(); - let path = std::env::temp_dir().join(format!("infotheory_api_{name}_{ts}.bin")); - fs::write(&path, contents).expect("temp fixture write should succeed"); - path -} - -#[test] -fn api_surface_entropy_and_distance_wrappers_are_callable() { - let x = b"alpha beta alpha beta alpha"; - let y = b"alpha gamma alpha gamma alpha"; - let backend = RateBackend::Ctw { depth: 8 }; - - let prev = get_default_ctx(); - set_default_ctx(InfotheoryCtx::new( - backend.clone(), - CompressionBackend::default(), - )); - - assert!(entropy_rate_backend(x, -1, &backend) >= 0.0); - assert!(biased_entropy_rate_backend(x, -1, &backend) >= 0.0); - assert!(cross_entropy_rate_backend(x, y, -1, &backend) >= 0.0); - assert!(joint_entropy_rate_backend(x, y, -1, &backend) >= 0.0); - assert!(mutual_information_rate_backend(x, y, -1, &backend) >= 0.0); - assert!((0.0..=1.0).contains(&ned_rate_backend(x, y, -1, &backend))); - assert!((0.0..=2.0).contains(&nte_rate_backend(x, y, -1, &backend))); - - assert!(marginal_entropy_bytes(x) >= 0.0); - assert!(joint_marginal_entropy_bytes(x, y) >= 0.0); - assert!(entropy_rate_bytes(x, -1) >= 0.0); - assert!(biased_entropy_rate_bytes(x, -1) >= 0.0); - assert!(joint_entropy_rate_bytes(x, y, -1) >= 0.0); - assert!(conditional_entropy_rate_bytes(x, y, -1) >= 0.0); - assert!(conditional_entropy_bytes(x, y, 0) >= 0.0); - assert!(mutual_information_bytes(x, y, 0) >= 0.0); - assert!(mutual_information_marg_bytes(x, y) >= 0.0); - assert!(mutual_information_rate_bytes(x, y, -1) >= 0.0); - assert!((0.0..=1.0).contains(&ned_bytes(x, y, 0))); - assert!((0.0..=1.0).contains(&ned_marg_bytes(x, y))); - assert!((0.0..=1.0).contains(&ned_rate_bytes(x, y, -1))); - assert!((0.0..=1.0).contains(&ned_cons_bytes(x, y, 0))); - assert!((0.0..=1.0).contains(&ned_cons_marg_bytes(x, y))); - assert!((0.0..=1.0).contains(&ned_cons_rate_bytes(x, y, -1))); - assert!((0.0..=2.0).contains(&nte_bytes(x, y, 0))); - assert!((0.0..=2.0).contains(&nte_marg_bytes(x, y))); - assert!((0.0..=2.0).contains(&nte_rate_bytes(x, y, -1))); - assert!((0.0..=1.0).contains(&tvd_bytes(x, y, 0))); - assert!((0.0..=1.0).contains(&nhd_bytes(x, y, 0))); - assert!(cross_entropy_bytes(x, y, 0) >= 0.0); - assert!(cross_entropy_rate_bytes(x, y, -1) >= 0.0); - assert!(d_kl_bytes(x, y) >= 0.0); - assert!(js_div_bytes(x, y) >= 0.0); - assert!((0.0..=1.0).contains(&intrinsic_dependence_bytes(x, -1))); - assert!((0.0..=1.0).contains(&resistance_to_transformation_bytes(x, y, -1))); - - set_default_ctx(prev); -} - -#[test] -fn api_surface_generation_session_and_config_are_callable() { - let prompt = b"If a frog is green, dogs are red.\nIf a toad is green, cats are red.\nIf a dog is green, frogs are red.\nIf a cat is green, toads are red.\nIf a frog is red, dogs are green.\nIf a toad is red, cats are green.\nIf a dog is red, frogs are green.\nIf a cat is red, toads are "; - let backend = RateBackend::RosaPlus; - let ctx = InfotheoryCtx::new(backend.clone(), CompressionBackend::default()); - let cfg = GenerationConfig::sampled_frozen(42); - - let direct = ctx.generate_bytes_with_config(prompt, 8, -1, cfg); - assert_eq!(direct.len(), 8); - - let mut session = - RateBackendSession::from_backend(backend, -1, Some((prompt.len() + direct.len()) as u64)) - .expect("session init"); - session.observe(prompt); - let from_session = session.generate_bytes(8, cfg); - session.finish().expect("session finish"); - - assert_eq!(from_session, direct); -} - -#[test] -fn api_surface_rate_backend_session_rejects_invalid_programmatic_mixture() { - let backend = RateBackend::Mixture { - spec: Arc::new(MixtureSpec::new(MixtureKind::Bayes, vec![])), - }; - let err = match RateBackendSession::from_backend(backend, -1, None) { - Ok(_) => panic!("invalid mixture backend should be rejected before runtime construction"), - Err(err) => err, - }; - assert!(err.contains("must include at least one expert")); -} - -#[cfg(all(feature = "backend-zpaq", not(target_env = "musl")))] -#[test] -fn api_surface_path_and_compression_helpers_are_callable() { - let x = b"lorem ipsum dolor sit amet"; - let y = b"lorem ipsum dolor"; - let px = temp_file("x", x); - let py = temp_file("y", y); - let sx = px.to_string_lossy().to_string(); - let sy = py.to_string_lossy().to_string(); - let paths = [sx.as_str(), sy.as_str()]; - - let backend = CompressionBackend::Zpaq { - method: "1".to_string(), - }; - - assert!(compress_size_backend(x, &backend) > 0); - assert!(compress_size_chain_backend(&[x.as_slice(), y.as_slice()], &backend) > 0); - let c = compress_bytes_backend(x, &backend).expect("zpaq compress"); - let d = decompress_bytes_backend(&c, &backend).expect("zpaq decompress"); - assert_eq!(d, x); - - assert!(get_compressed_size(&sx, "1") > 0); - assert!(get_compressed_size_parallel(&sx, "1", 2) > 0); - - let bytes = get_bytes_from_paths(&paths); - assert_eq!(bytes.len(), 2); - assert_eq!(bytes[0], x); - assert_eq!(bytes[1], y); - - let s1 = get_sequential_compressed_sizes_from_sequential_paths(&paths, "1"); - let s2 = get_parallel_compressed_sizes_from_sequential_paths(&paths, "1", 2); - let s3 = get_sequential_compressed_sizes_from_parallel_paths(&paths, "1"); - let s4 = get_parallel_compressed_sizes_from_parallel_paths(&paths, "1", 2); - let s5 = get_compressed_sizes_from_paths(&paths, "1"); - for sizes in [s1, s2, s3, s4, s5] { - assert_eq!(sizes.len(), 2); - assert!(sizes[0] > 0); - assert!(sizes[1] > 0); - } - - assert!(ncd_bytes(x, y, "1", NcdVariant::Vitanyi) >= 0.0); - assert!(ncd_bytes_default(x, y, NcdVariant::SymVitanyi) >= 0.0); - assert!(ncd_bytes_backend(x, y, &backend, NcdVariant::Cons) >= 0.0); - assert!(ncd_paths(&sx, &sy, "1", NcdVariant::SymCons) >= 0.0); - assert!(ncd_paths_backend(&sx, &sy, &backend, NcdVariant::Vitanyi) >= 0.0); - assert!(ncd_vitanyi(&sx, &sy, "1") >= 0.0); - assert!(ncd_sym_vitanyi(&sx, &sy, "1") >= 0.0); - assert!(ncd_cons(&sx, &sy, "1") >= 0.0); - assert!(ncd_sym_cons(&sx, &sy, "1") >= 0.0); - - let m = ncd_matrix_bytes(&[x.to_vec(), y.to_vec()], "1", NcdVariant::Vitanyi); - assert_eq!(m.len(), 4); - let mp = ncd_matrix_paths(&paths, "1", NcdVariant::Cons); - assert_eq!(mp.len(), 4); - - assert!(ned_paths(&sx, &sy, 0) >= 0.0); - assert!(nte_paths(&sx, &sy, 0) >= 0.0); - assert!(tvd_paths(&sx, &sy, 0) >= 0.0); - assert!(nhd_paths(&sx, &sy, 0) >= 0.0); - assert!(mutual_information_paths(&sx, &sy, 0) >= 0.0); - assert!(conditional_entropy_paths(&sx, &sy, 0) >= 0.0); - assert!(cross_entropy_paths(&sx, &sy, 0) >= 0.0); - assert!(kl_divergence_paths(&sx, &sy) >= 0.0); - assert!(js_divergence_paths(&sx, &sy) >= 0.0); - - let _ = fs::remove_file(px); - let _ = fs::remove_file(py); -} diff --git a/tests/benchmark_suite_specs.rs b/tests/benchmark_suite_specs.rs deleted file mode 100644 index 6b7b73eb..00000000 --- a/tests/benchmark_suite_specs.rs +++ /dev/null @@ -1,68 +0,0 @@ -use serde_json::Value; -use std::fs; -use std::path::PathBuf; - -fn load_example(name: &str) -> Value { - let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("examples") - .join(name); - let raw = fs::read_to_string(&path) - .unwrap_or_else(|e| panic!("failed to read {}: {e}", path.display())); - serde_json::from_str(&raw) - .unwrap_or_else(|e| panic!("failed to parse {} as JSON: {e}", path.display())) -} - -#[test] -fn extra_suite_includes_expected_uncovered_backends() { - let v = load_example("extra.json"); - assert_eq!(v["kind"], "neural"); - let experts = v["experts"] - .as_array() - .expect("extra.json must contain experts array"); - assert!( - !experts.is_empty(), - "extra.json experts array must be non-empty" - ); - - let mut saw_mamba = false; - let mut saw_particle_fast = false; - let mut saw_sparse_match = false; - - for expert in experts { - let kind = expert["kind"].as_str().unwrap_or_default(); - match kind { - "mamba" => { - saw_mamba = true; - let method = expert["method"] - .as_str() - .expect("mamba expert must define method"); - assert!( - method.contains("policy:schedule="), - "mamba method should include an explicit schedule policy" - ); - } - "particle" => { - let spec_path = expert["spec_path"] - .as_str() - .expect("particle expert must use spec_path"); - if spec_path == "particle_fast.json" { - saw_particle_fast = true; - } - } - "sparse-match" => { - saw_sparse_match = true; - } - _ => {} - } - } - - assert!(saw_mamba, "extra.json must include a mamba expert"); - assert!( - saw_particle_fast, - "extra.json must include particle_fast.json-backed particle expert" - ); - assert!( - saw_sparse_match, - "extra.json must include a sparse-match expert" - ); -} diff --git a/tests/mixture_rate_backend.rs b/tests/mixture_rate_backend.rs deleted file mode 100644 index 0867b931..00000000 --- a/tests/mixture_rate_backend.rs +++ /dev/null @@ -1,426 +0,0 @@ -use infotheory::{MixtureExpertSpec, MixtureKind, MixtureSpec, RateBackend, entropy_rate_backend}; -use std::sync::Arc; - -#[test] -fn mixture_single_expert_matches_backend() { - let data = b"abababababababababababababababab"; - let base = RateBackend::Ctw { depth: 8 }; - let base_rate = entropy_rate_backend(data, -1, &base); - - let spec = MixtureSpec::new( - MixtureKind::Bayes, - vec![MixtureExpertSpec { - name: None, - log_prior: 0.0, - max_order: -1, - backend: base.clone(), - }], - ); - let mix_backend = RateBackend::Mixture { - spec: Arc::new(spec), - }; - let mix_rate = entropy_rate_backend(data, -1, &mix_backend); - - assert!( - (mix_rate - base_rate).abs() < 1e-6, - "mix={mix_rate} base={base_rate}" - ); -} - -#[test] -fn mixture_single_sequitur_expert_matches_backend() { - let data = b"abcabcabcabcabcabc"; - let base = RateBackend::Sequitur { context_bytes: 32 }; - let base_rate = entropy_rate_backend(data, -1, &base); - - let spec = MixtureSpec::new( - MixtureKind::Bayes, - vec![MixtureExpertSpec { - name: Some("sequitur".to_string()), - log_prior: 0.0, - max_order: -1, - backend: base.clone(), - }], - ); - let mix_backend = RateBackend::Mixture { - spec: Arc::new(spec), - }; - let mix_rate = entropy_rate_backend(data, -1, &mix_backend); - - assert!( - (mix_rate - base_rate).abs() < 1e-6, - "mix={mix_rate} base={base_rate}" - ); -} - -#[cfg(feature = "backend-rwkv")] -#[test] -fn rwkv_mixture_single_expert_matches_backend_with_tbptt() { - let data = b"abcdefghij"; - let base = RateBackend::Rwkv7Method { - method: "cfg:hidden=64,layers=1,intermediate=64,decay_rank=8,a_rank=8,v_rank=8,g_rank=8,seed=37,train=adam,lr=0.0008,stride=1;policy:schedule=0..100:train(scope=all,opt=adam,lr=0.0008,stride=1,bptt=8,clip=0,momentum=0.9)".to_string(), - }; - let base_rate = entropy_rate_backend(data, -1, &base); - - let spec = MixtureSpec::new( - MixtureKind::Bayes, - vec![MixtureExpertSpec { - name: Some("rwkv".to_string()), - log_prior: 0.0, - max_order: -1, - backend: base.clone(), - }], - ); - let mix_backend = RateBackend::Mixture { - spec: Arc::new(spec), - }; - let mix_rate = entropy_rate_backend(data, -1, &mix_backend); - - assert!( - (mix_rate - base_rate).abs() < 1e-6, - "mix={mix_rate} base={base_rate}" - ); -} - -#[test] -fn mixture_recursive_expert_matches_backend() { - let data = b"01010101010101010101010101010101"; - let base = RateBackend::Ctw { depth: 8 }; - let base_rate = entropy_rate_backend(data, -1, &base); - - let inner = MixtureSpec::new( - MixtureKind::Bayes, - vec![MixtureExpertSpec { - name: Some("ctw".to_string()), - log_prior: 0.0, - max_order: -1, - backend: base.clone(), - }], - ); - let outer = MixtureSpec::new( - MixtureKind::Bayes, - vec![MixtureExpertSpec { - name: Some("inner".to_string()), - log_prior: 0.0, - max_order: -1, - backend: RateBackend::Mixture { - spec: Arc::new(inner), - }, - }], - ); - let mix_backend = RateBackend::Mixture { - spec: Arc::new(outer), - }; - let mix_rate = entropy_rate_backend(data, -1, &mix_backend); - - assert!( - (mix_rate - base_rate).abs() < 1e-6, - "mix={mix_rate} base={base_rate}" - ); -} - -#[test] -fn neural_mixture_single_expert_matches_backend() { - let data = b"abababababababababababababababab"; - let base = RateBackend::Ctw { depth: 8 }; - let base_rate = entropy_rate_backend(data, -1, &base); - - let spec = MixtureSpec::new( - MixtureKind::Neural, - vec![MixtureExpertSpec { - name: None, - log_prior: 0.0, - max_order: -1, - backend: base.clone(), - }], - ) - .with_alpha(0.05); - let mix_backend = RateBackend::Mixture { - spec: Arc::new(spec), - }; - let mix_rate = entropy_rate_backend(data, -1, &mix_backend); - - assert!( - (mix_rate - base_rate).abs() < 1e-6, - "mix={mix_rate} base={base_rate}" - ); -} - -#[test] -fn convex_mixture_single_expert_matches_backend() { - let data = b"abababababababababababababababab"; - let base = RateBackend::Ctw { depth: 8 }; - let base_rate = entropy_rate_backend(data, -1, &base); - - let spec = MixtureSpec::new( - MixtureKind::Convex, - vec![MixtureExpertSpec { - name: None, - log_prior: 0.0, - max_order: -1, - backend: base.clone(), - }], - ) - .with_alpha(1.25); - let mix_backend = RateBackend::Mixture { - spec: Arc::new(spec), - }; - let mix_rate = entropy_rate_backend(data, -1, &mix_backend); - - assert!( - (mix_rate - base_rate).abs() < 1e-6, - "mix={mix_rate} base={base_rate}" - ); -} - -#[test] -fn switching_theorem_schedule_backend_executes() { - let data = b"abababababababababababababababab"; - let spec = MixtureSpec::new( - MixtureKind::Switching, - vec![ - MixtureExpertSpec { - name: Some("ctw".to_string()), - log_prior: 0.0, - max_order: -1, - backend: RateBackend::Ctw { depth: 8 }, - }, - MixtureExpertSpec { - name: Some("match".to_string()), - log_prior: 0.0, - max_order: -1, - backend: RateBackend::Match { - hash_bits: 18, - min_len: 3, - max_len: 96, - base_mix: 0.03, - confidence_scale: 1.0, - }, - }, - ], - ) - .with_schedule(infotheory::MixtureScheduleMode::Theorem) - .with_alpha(0.99); - let backend = RateBackend::Mixture { - spec: Arc::new(spec), - }; - let rate = entropy_rate_backend(data, -1, &backend); - assert!(rate.is_finite() && rate >= 0.0, "rate={rate}"); -} - -#[test] -fn convex_theorem_schedule_backend_executes() { - let data = b"abababababababababababababababab"; - let spec = MixtureSpec::new( - MixtureKind::Convex, - vec![ - MixtureExpertSpec { - name: Some("ctw".to_string()), - log_prior: 0.0, - max_order: -1, - backend: RateBackend::Ctw { depth: 8 }, - }, - MixtureExpertSpec { - name: Some("fac".to_string()), - log_prior: 0.0, - max_order: -1, - backend: RateBackend::FacCtw { - base_depth: 8, - num_percept_bits: 8, - encoding_bits: 8, - }, - }, - ], - ) - .with_schedule(infotheory::MixtureScheduleMode::Theorem) - .with_alpha(7.5); - let backend = RateBackend::Mixture { - spec: Arc::new(spec), - }; - let rate = entropy_rate_backend(data, -1, &backend); - assert!(rate.is_finite() && rate >= 0.0, "rate={rate}"); -} - -#[test] -fn neural_mixture_supports_nested_mixture_expert() { - let data = b"abracadabra abracadabra abracadabra"; - let inner = MixtureSpec::new( - MixtureKind::Bayes, - vec![ - MixtureExpertSpec { - name: Some("ctw".to_string()), - log_prior: 0.0, - max_order: -1, - backend: RateBackend::Ctw { depth: 8 }, - }, - MixtureExpertSpec { - name: Some("fac".to_string()), - log_prior: 0.0, - max_order: -1, - backend: RateBackend::FacCtw { - base_depth: 8, - num_percept_bits: 8, - encoding_bits: 8, - }, - }, - ], - ); - - let outer = MixtureSpec::new( - MixtureKind::Neural, - vec![ - MixtureExpertSpec { - name: Some("nested".to_string()), - log_prior: 0.0, - max_order: -1, - backend: RateBackend::Mixture { - spec: Arc::new(inner), - }, - }, - MixtureExpertSpec { - name: Some("zpaq".to_string()), - log_prior: 0.0, - max_order: -1, - backend: RateBackend::Zpaq { - method: "1".to_string(), - }, - }, - ], - ) - .with_alpha(0.03); - - let backend = RateBackend::Mixture { - spec: Arc::new(outer), - }; - let rate = entropy_rate_backend(data, -1, &backend); - assert!(rate.is_finite() && rate >= 0.0, "rate={rate}"); -} - -#[test] -fn convex_mixture_supports_nested_mixture_expert() { - let data = b"abracadabra abracadabra abracadabra"; - let inner = MixtureSpec::new( - MixtureKind::Bayes, - vec![ - MixtureExpertSpec { - name: Some("ctw".to_string()), - log_prior: 0.0, - max_order: -1, - backend: RateBackend::Ctw { depth: 8 }, - }, - MixtureExpertSpec { - name: Some("fac".to_string()), - log_prior: 0.0, - max_order: -1, - backend: RateBackend::FacCtw { - base_depth: 8, - num_percept_bits: 8, - encoding_bits: 8, - }, - }, - ], - ); - - let outer = MixtureSpec::new( - MixtureKind::Convex, - vec![ - MixtureExpertSpec { - name: Some("nested".to_string()), - log_prior: 0.0, - max_order: -1, - backend: RateBackend::Mixture { - spec: Arc::new(inner), - }, - }, - MixtureExpertSpec { - name: Some("ppmd".to_string()), - log_prior: 0.0, - max_order: -1, - backend: RateBackend::Ppmd { - order: 6, - memory_mb: 8, - }, - }, - ], - ) - .with_alpha(1.25); - - let backend = RateBackend::Mixture { - spec: Arc::new(outer), - }; - let rate = entropy_rate_backend(data, -1, &backend); - assert!(rate.is_finite() && rate >= 0.0, "rate={rate}"); -} - -#[test] -fn new_backends_have_finite_entropy_rates() { - let data = b"match match match sparse sparse sparse payload"; - let backends = [ - RateBackend::Match { - hash_bits: 20, - min_len: 4, - max_len: 255, - base_mix: 0.02, - confidence_scale: 1.0, - }, - RateBackend::SparseMatch { - hash_bits: 19, - min_len: 3, - max_len: 64, - gap_min: 1, - gap_max: 2, - base_mix: 0.05, - confidence_scale: 1.0, - }, - RateBackend::Ppmd { - order: 8, - memory_mb: 8, - }, - ]; - for backend in backends { - let rate = entropy_rate_backend(data, -1, &backend); - assert!(rate.is_finite() && rate >= 0.0, "rate={rate}"); - } -} - -#[test] -fn neural_mixture_supports_calibrated_expert() { - let data = b"calibrated ctw expert payload calibrated ctw expert payload"; - let spec = MixtureSpec::new( - MixtureKind::Neural, - vec![ - MixtureExpertSpec { - name: Some("cal".to_string()), - log_prior: 0.0, - max_order: -1, - backend: RateBackend::Calibrated { - spec: Arc::new(infotheory::CalibratedSpec { - base: RateBackend::Ctw { depth: 8 }, - context: infotheory::CalibrationContextKind::Text, - bins: 33, - learning_rate: 0.02, - bias_clip: 4.0, - }), - }, - }, - MixtureExpertSpec { - name: Some("match".to_string()), - log_prior: 0.0, - max_order: -1, - backend: RateBackend::Match { - hash_bits: 20, - min_len: 4, - max_len: 255, - base_mix: 0.02, - confidence_scale: 1.0, - }, - }, - ], - ) - .with_alpha(0.03); - let backend = RateBackend::Mixture { - spec: Arc::new(spec), - }; - let rate = entropy_rate_backend(data, -1, &backend); - assert!(rate.is_finite() && rate >= 0.0, "rate={rate}"); -} diff --git a/tests/zpaq_rate_backend.rs b/tests/zpaq_rate_backend.rs deleted file mode 100644 index 5a1bfbec..00000000 --- a/tests/zpaq_rate_backend.rs +++ /dev/null @@ -1,20 +0,0 @@ -#[cfg(feature = "backend-zpaq")] -use infotheory::{RateBackend, entropy_rate_backend}; - -#[test] -#[cfg(feature = "backend-zpaq")] -fn zpaq_rate_backend_compresses_copy_data() { - let mut data = Vec::new(); - let pattern = b"copy-like-pattern-"; - for _ in 0..512 { - data.extend_from_slice(pattern); - } - let backend = RateBackend::Zpaq { - method: "2".to_string(), - }; - let rate = entropy_rate_backend(&data, -1, &backend); - assert!( - rate < 0.5, - "expected low entropy rate for copy-like data, got {rate:.4}" - ); -} diff --git a/uv.lock b/uv.lock index 8082133b..d06a056c 100644 --- a/uv.lock +++ b/uv.lock @@ -143,7 +143,7 @@ wheels = [ [[package]] name = "infotheory-rs" -version = "1.1.1" +version = "1.2.0" source = { editable = "." } [package.dev-dependencies] diff --git a/vendor/gameengine b/vendor/gameengine new file mode 160000 index 00000000..36e0f10b --- /dev/null +++ b/vendor/gameengine @@ -0,0 +1 @@ +Subproject commit 36e0f10b43329926eaacd6268e94be6c7e394d27 diff --git a/vendor/nyx-lite b/vendor/nyx-lite new file mode 160000 index 00000000..3dc54897 --- /dev/null +++ b/vendor/nyx-lite @@ -0,0 +1 @@ +Subproject commit 3dc548979ac16994a31adc8d3a1a0e2086305a17 diff --git a/vendor/zpaq_rs b/vendor/zpaq_rs new file mode 160000 index 00000000..42d3becf --- /dev/null +++ b/vendor/zpaq_rs @@ -0,0 +1 @@ +Subproject commit 42d3becfb42b490f369d5c07e13f2510ca62c23c diff --git a/zpaq_rs b/zpaq_rs deleted file mode 160000 index cd5ec4e1..00000000 --- a/zpaq_rs +++ /dev/null @@ -1 +0,0 @@ -Subproject commit cd5ec4e1df78902958153e36d242a04bb08f7e58